def collect_commit_dependencies(self, key, file_name, share_ID):
if not (share_ID in self.share_machine_dict):
print 'Error: share ID not associated with any machines. Call add_share_to_machine()'
return
machine_proxy = self.machine_proxy_dict[(
self.share_machine_dict[share_ID]
)[0]] #use first machine proxy in list. TODO# use all proxies at once
bm = local_blob_manager()
while True:
files_needed = bm.blobs_to_restore_blob(
key, os.path.join(self.storage_directory, share_ID), file_name)
if files_needed == None:
return
for need_file_name in files_needed:
logging.debug('requesting file to fill dependency: %s',
need_file_name)
file_xmlrpc_data = machine_proxy.get_file(
share_ID, need_file_name)
f = open(
os.path.join(self.storage_directory, share_ID,
need_file_name), 'wb')
f.write(file_xmlrpc_data.data)
f.close()
def commit(self, params, rpc_id, packet):
"""
"""
#@TODO: Ensure only authenticated users can access this command
[ver, congest, args] = params
[
key, working_directory, share_ID, user_name, commit_msg,
parent_commit_hash, other_parent_commit_hash
] = args
logging.debug(
'[key, working_directory, share_ID, user_name, commit_msg, parent_commit_hash, other_parent_commit_hash]: '
% ([
key, working_directory, share_ID, user_name, commit_msg,
parent_commit_hash, other_parent_commit_hash
]))
#@TODO: If share_ID is None, look through all shares for the parent_commit_hash and use its share
bm = local_blob_manager()
commit_hash = bm.commit_directory(
key, working_directory,
os.path.join(self.storage_directory, share_ID), user_name,
commit_msg, parent_commit_hash, other_parent_commit_hash)
#@TODO: packet response is to myself for it is stored in shared object json_response_dict for use with autosync. Allow responding to other machines?
return_address = ('localhost', self.command_port)
packet.json_RPC_object = dict(jsonrpc="2.0",
result=commit_hash,
id=rpc_id)
packet.to_address = return_address
return packet
def on_any_event(self, event):
logging.debug('auto sync directory event triggered')
time.sleep(0.1) #ignore any transient changes
bm = local_blob_manager()
bm.commit_directory(self.key, self.monitoring_directory, os.path.join(self.storage_directory, self.share_ID),
self.user_name, 'auto', None, None) #TODO: real value for parent commit
def test_delta_commit(self):
logging.debug('Testing commiting with a known parent')
reset_storage()
setup_test_dirs()
bm = local_blob_manager()
commit_hash_1 = bm.commit_directory(key, '../resource/test_dir_3/root',
os.path.join(peer_A_storage, 'test_share'), 'joe.keur', 'first commit msg')
commit_hash_2 = bm.commit_directory(key, '../resource/test_dir_1/root',
os.path.join(peer_A_storage, 'test_share'), 'joe.keur', 'first commit msg', commit_hash_1)
def on_any_event(self, event):
logging.debug('auto sync directory event triggered')
time.sleep(0.1) #ignore any transient changes
bm = local_blob_manager()
bm.commit_directory(self.key, self.monitoring_directory,
os.path.join(self.storage_directory,
self.share_ID), self.user_name,
'auto', None,
None) #TODO: real value for parent commit
def restore(self, params, rpc_id, packet): """ """ #@TODO: Ensure only authenticated users can access this command [ver, congest, args] = params [key, working_directory, commit_hash] = args bm = local_blob_manager() #@TODO: look for version already in working_directory and do differential update bm.restore_directory(key, working_directory, self.storage_directory, commit_hash)
def test_initial_commit(self):
logging.debug('Testing recoring a tree')
reset_storage()
setup_test_dirs()
bm = local_blob_manager()
commit_hash_1 = bm.commit_directory(key, '../resource/test_dir_1/root',
os.path.join(peer_A_storage, 'test_share'), 'joe.keur', 'first commit msg')
bm.restore_directory(key,'../resource/restore_directory_1', os.path.join(peer_A_storage, 'test_share'),
commit_hash_1)
def restore(self, params, rpc_id, packet):
"""
"""
#@TODO: Ensure only authenticated users can access this command
[ver, congest, args] = params
[key, working_directory, commit_hash] = args
bm = local_blob_manager()
#@TODO: look for version already in working_directory and do differential update
bm.restore_directory(key, working_directory, self.storage_directory,
commit_hash)
def test_initial_commit(self):
logging.debug('Testing recoring a tree')
reset_storage()
setup_test_dirs()
bm = local_blob_manager()
commit_hash_1 = bm.commit_directory(
key, '../resource/test_dir_1/root',
os.path.join(peer_A_storage, 'test_share'), 'joe.keur',
'first commit msg')
bm.restore_directory(key, '../resource/restore_directory_1',
os.path.join(peer_A_storage, 'test_share'),
commit_hash_1)
def test_delta_commit(self):
logging.debug('Testing commiting with a known parent')
reset_storage()
setup_test_dirs()
bm = local_blob_manager()
commit_hash_1 = bm.commit_directory(
key, '../resource/test_dir_3/root',
os.path.join(peer_A_storage, 'test_share'), 'joe.keur',
'first commit msg')
commit_hash_2 = bm.commit_directory(
key, '../resource/test_dir_1/root',
os.path.join(peer_A_storage, 'test_share'), 'joe.keur',
'first commit msg', commit_hash_1)
def collect_commit_dependencies(self, key, file_name, peer_ID, url = None):
if (not (peer_ID in self.peer_dict)):
self.connect_peer(peer_ID, url)
bm = local_blob_manager()
while True:
files_needed = bm.blobs_to_restore_blob(key, self.storage_directory, file_name)
if files_needed==None:
return
for need_file_name in files_needed:
logging.debug('requesting file to fill dependency: %s', need_file_name)
file_xmlrpc_data = (self.peer_dict[peer_ID]).get_file(need_file_name)
f=open(os.path.join(self.storage_directory, need_file_name),'wb')
f.write(file_xmlrpc_data.data)
f.close()
def sync_new_commit(self, params, rpc_id, packet):
"""
"""
[ver, congest, args] = params
[commit_hash, share_ID] = args
logging.debug('[commit_hash, share_ID]: %s' %
([commit_hash, share_ID]))
key = b'Sixteen byte key' #@TODO: what do I do about this key?
#@TODO: this should sync by updating directory, not overwriting it.
#@TODO: should commit this directory before updating it
if share_ID not in self.autosync_share_working_directory_dict:
logging.error('No autosync directories with share_ID: %s' %
(share_ID))
return
if commit_hash[0] != '_':
commit_hash = '_' + commit_hash
full_file_name = os.path.join(self.storage_directory, share_ID,
commit_hash)
if not os.path.isfile(full_file_name):
logging.error('commit does not exist. path: %s' %
(full_file_name))
return
bm = local_blob_manager()
working_directory_list = self.autosync_share_working_directory_dict[
share_ID]
logging.debug('directories to sync: %s' % (working_directory_list))
for working_directory in working_directory_list:
file_list, mod_times, last_commit_hash = local_blob_manager.read_commit_meta(
working_directory)
logging.debug('parent sync hash: %s' % (last_commit_hash))
if commit_hash == last_commit_hash:
logging.debug(
'Working directory already contains desired commit')
continue
if working_directory in self.autosync_working_directory_dict: #disable watchdog while writing to directory
self.autosyncmanager.stop(working_directory)
(head, tail) = os.path.split(
working_directory) #TODO: this seems like a hack
bm.restore_directory(
key, head, os.path.join(self.storage_directory, share_ID),
commit_hash)
if working_directory in self.autosync_working_directory_dict:
self.autosyncmanager.start(working_directory)
def commit(self, params, rpc_id, packet):
"""
"""
#@TODO: Ensure only authenticated users can access this command
[ver, congest, args] = params
[key, working_directory, share_ID, user_name, commit_msg, parent_commit_hash, other_parent_commit_hash] = args
logging.debug('[key, working_directory, share_ID, user_name, commit_msg, parent_commit_hash, other_parent_commit_hash]: ' %([key, working_directory, share_ID, user_name, commit_msg, parent_commit_hash, other_parent_commit_hash]))
#@TODO: If share_ID is None, look through all shares for the parent_commit_hash and use its share
bm = local_blob_manager()
commit_hash = bm.commit_directory(key, working_directory, os.path.join(self.storage_directory, share_ID),
user_name, commit_msg, parent_commit_hash, other_parent_commit_hash)
#@TODO: packet response is to myself for it is stored in shared object json_response_dict for use with autosync. Allow responding to other machines?
return_address = ('localhost', self.command_port)
packet.json_RPC_object = dict(jsonrpc="2.0", result=commit_hash, id=rpc_id)
packet.to_address = return_address
return packet
def collect_commit_dependencies(self, key, file_name, share_ID):
if not (share_ID in self.share_machine_dict):
print 'Error: share ID not associated with any machines. Call add_share_to_machine()'
return
machine_proxy = self.machine_proxy_dict[(self.share_machine_dict[share_ID])[0]] #use first machine proxy in list. TODO# use all proxies at once
bm = local_blob_manager()
while True:
files_needed = bm.blobs_to_restore_blob(key, os.path.join(self.storage_directory, share_ID), file_name)
if files_needed==None:
return
for need_file_name in files_needed:
logging.debug('requesting file to fill dependency: %s', need_file_name)
file_xmlrpc_data = machine_proxy.get_file(share_ID, need_file_name)
f=open(os.path.join(self.storage_directory, share_ID, need_file_name),'wb')
f.write(file_xmlrpc_data.data)
f.close()
def collect_commit_dependencies(self, key, file_name, peer_ID, url=None):
if (not (peer_ID in self.peer_dict)):
self.connect_peer(peer_ID, url)
bm = local_blob_manager()
while True:
files_needed = bm.blobs_to_restore_blob(key,
self.storage_directory,
file_name)
if files_needed == None:
return
for need_file_name in files_needed:
logging.debug('requesting file to fill dependency: %s',
need_file_name)
file_xmlrpc_data = (
self.peer_dict[peer_ID]).get_file(need_file_name)
f = open(os.path.join(self.storage_directory, need_file_name),
'wb')
f.write(file_xmlrpc_data.data)
f.close()
def sync_new_commit(self, params, rpc_id, packet):
"""
"""
[ver, congest, args] = params
[commit_hash, share_ID] = args
logging.debug('[commit_hash, share_ID]: %s'%([commit_hash, share_ID]))
key=b'Sixteen byte key' #@TODO: what do I do about this key?
#@TODO: this should sync by updating directory, not overwriting it.
#@TODO: should commit this directory before updating it
if share_ID not in self.autosync_share_working_directory_dict:
logging.error('No autosync directories with share_ID: %s' %(share_ID))
return
if commit_hash[0] != '_':
commit_hash = '_' + commit_hash
full_file_name = os.path.join(self.storage_directory, share_ID, commit_hash)
if not os.path.isfile(full_file_name):
logging.error('commit does not exist. path: %s' %(full_file_name))
return
bm = local_blob_manager()
working_directory_list = self.autosync_share_working_directory_dict[share_ID]
logging.debug('directories to sync: %s' %(working_directory_list))
for working_directory in working_directory_list:
file_list, mod_times, last_commit_hash = local_blob_manager.read_commit_meta(working_directory)
logging.debug('parent sync hash: %s'%(last_commit_hash))
if commit_hash == last_commit_hash:
logging.debug('Working directory already contains desired commit')
continue
if working_directory in self.autosync_working_directory_dict: #disable watchdog while writing to directory
self.autosyncmanager.stop(working_directory)
(head, tail) = os.path.split(working_directory) #TODO: this seems like a hack
bm.restore_directory(key, head, os.path.join(self.storage_directory, share_ID), commit_hash)
if working_directory in self.autosync_working_directory_dict:
self.autosyncmanager.start(working_directory)
print "************************************************************************"
# encrypt and store the first file blob, then decrypt and load
fb_hash = fb.store(key, os.path.join(peer_A_storage, "test_share"))
fb3 = file_blob()
fb3.my_hash = "3"
fb3.load(key, os.path.join(peer_A_storage, "test_share"), fb_hash)
fb3.display()
print "\n\n"
print "************************************************************************"
print "***Testing loading a whole directory as an initial commit"
logging.debug("Testing loading a whole directory as an initial commit")
print "************************************************************************"
# load a whole directory as an initial commit
bm = local_blob_manager()
commit_hash_1 = bm.commit_directory(
key, "../resource/test_directory_1/root", os.path.join(peer_A_storage, "test_share"), "joe.keur", "first commit msg"
)
bm.restore_directory(key, "../resource/restore_directory_1", os.path.join(peer_A_storage, "test_share"), commit_hash_1)
print "\n\n"
print "************************************************************************"
print "***Testing adding a second commit"
logging.debug("Testing adding a second commit")
print "************************************************************************"
bm = local_blob_manager()
commit_hash_2 = bm.commit_directory(
key,
"../resource/test_directory_2/root",
logging.debug('Testing storing and loading a simple file blob')
print '************************************************************************'
#encrypt and store the first file blob, then decrypt and load
fb_hash = fb.store(key, os.path.join(peer_A_storage, 'test_share'))
fb3 = file_blob()
fb3.my_hash = '3'
fb3.load(key, os.path.join(peer_A_storage, 'test_share'), fb_hash)
fb3.display()
print '\n\n'
print '************************************************************************'
print '***Testing loading a whole directory as an initial commit'
logging.debug('Testing loading a whole directory as an initial commit')
print '************************************************************************'
#load a whole directory as an initial commit
bm = local_blob_manager()
commit_hash_1 = bm.commit_directory(key, '../resource/test_directory_1/root',
os.path.join(peer_A_storage, 'test_share'),
'joe.keur', 'first commit msg')
bm.restore_directory(key, '../resource/restore_directory_1',
os.path.join(peer_A_storage, 'test_share'), commit_hash_1)
print '\n\n'
print '************************************************************************'
print '***Testing adding a second commit'
logging.debug('Testing adding a second commit')
print '************************************************************************'
bm = local_blob_manager()
commit_hash_2 = bm.commit_directory(key, '../resource/test_directory_2/root',
os.path.join(peer_A_storage,
'test_share'), 'joe.keur',
