def __init__(self): self.json_call_queue = None self.json_response_dict = None #JSON responses are pushed here indexed by rpc ID self.json_out_queue = None #packets of JSON responses are pushed here awaiting postprocessing and transmission self.process_pool = [] self.congest_connect_state = None self.storage_directory = "" self.a_s_list = [] #list of autoSync objects self.command_port = None #Objects shared across processes self.my_machine_ID = "" self.manager = Manager() self.share_machine_dict = self.manager.dict() #maps share IDs to machine IDs #self.share_key_dict = self.manager.dict() #maps share IDs to corresponding public keys (or certificates?) #TODO: implement self.machine_address_dict = self.manager.dict() #maps machine IDs to url (IP:port) self.autosync_share_working_directory_dict = self.manager.dict() #lists all autosynced working directories for a given share self.autosync_working_directory_dict = self.manager.dict() #given a working directory return the auto sync observer self.autosyncmanager = AutoSyncManager(self.json_response_dict) #register JSONRPC functions self.method_dict = dict() #function pointers to registered JSONRPC functions self.register_func('ping', self.ping) self.register_func('get_file', self.get_file) self.register_func('save_file_block', self.save_file_block) self.register_func('update_machine_address', self.update_machine_address) self.register_func('add_share_to_machine', self.add_share_to_machine) self.register_func('has_file', self.has_file) self.register_func('push_update_to_peer', self.push_update_to_peer) self.register_func('register_auto_sync', self.register_auto_sync) self.register_func('commit', self.commit) self.register_func('restore', self.restore) self.register_func('sync_new_commit', self.sync_new_commit)
def __init__(self): self.json_call_queue = None self.json_response_dict = None #JSON responses are pushed here indexed by rpc ID self.json_out_queue = None #packets of JSON responses are pushed here awaiting postprocessing and transmission self.process_pool = [] self.congest_connect_state = None self.storage_directory = "" self.a_s_list = [] #list of autoSync objects self.command_port = None #Objects shared across processes self.my_machine_ID = "" self.manager = Manager() self.share_machine_dict = self.manager.dict( ) #maps share IDs to machine IDs #self.share_key_dict = self.manager.dict() #maps share IDs to corresponding public keys (or certificates?) #TODO: implement self.machine_address_dict = self.manager.dict( ) #maps machine IDs to url (IP:port) self.autosync_share_working_directory_dict = self.manager.dict( ) #lists all autosynced working directories for a given share self.autosync_working_directory_dict = self.manager.dict( ) #given a working directory return the auto sync observer self.autosyncmanager = AutoSyncManager(self.json_response_dict) #register JSONRPC functions self.method_dict = dict( ) #function pointers to registered JSONRPC functions self.register_func('ping', self.ping) self.register_func('get_file', self.get_file) self.register_func('save_file_block', self.save_file_block) self.register_func('update_machine_address', self.update_machine_address) self.register_func('add_share_to_machine', self.add_share_to_machine) self.register_func('has_file', self.has_file) self.register_func('push_update_to_peer', self.push_update_to_peer) self.register_func('register_auto_sync', self.register_auto_sync) self.register_func('commit', self.commit) self.register_func('restore', self.restore) self.register_func('sync_new_commit', self.sync_new_commit)
class JSONServer(): def __init__(self): self.json_call_queue = None self.json_response_dict = None #JSON responses are pushed here indexed by rpc ID self.json_out_queue = None #packets of JSON responses are pushed here awaiting postprocessing and transmission self.process_pool = [] self.congest_connect_state = None self.storage_directory = "" self.a_s_list = [] #list of autoSync objects self.command_port = None #Objects shared across processes self.my_machine_ID = "" self.manager = Manager() self.share_machine_dict = self.manager.dict( ) #maps share IDs to machine IDs #self.share_key_dict = self.manager.dict() #maps share IDs to corresponding public keys (or certificates?) #TODO: implement self.machine_address_dict = self.manager.dict( ) #maps machine IDs to url (IP:port) self.autosync_share_working_directory_dict = self.manager.dict( ) #lists all autosynced working directories for a given share self.autosync_working_directory_dict = self.manager.dict( ) #given a working directory return the auto sync observer self.autosyncmanager = AutoSyncManager(self.json_response_dict) #register JSONRPC functions self.method_dict = dict( ) #function pointers to registered JSONRPC functions self.register_func('ping', self.ping) self.register_func('get_file', self.get_file) self.register_func('save_file_block', self.save_file_block) self.register_func('update_machine_address', self.update_machine_address) self.register_func('add_share_to_machine', self.add_share_to_machine) self.register_func('has_file', self.has_file) self.register_func('push_update_to_peer', self.push_update_to_peer) self.register_func('register_auto_sync', self.register_auto_sync) self.register_func('commit', self.commit) self.register_func('restore', self.restore) self.register_func('sync_new_commit', self.sync_new_commit) def start_pool(self, processes=None): if processes == None: processes = multiprocessing.cpu_count() for i in range(0, processes): proc = Process(target=self.serve_forever) proc.start() self.process_pool.append(proc) logging.info('JSONServer pool %d processes started' % (processes)) def terminate(self): for p in self.process_pool: p.terminate() #TODO: the autosync processes don't get cleaned up def serve_forever(self): Random.atfork() while True: if not self.serve_once(): time.sleep(0.1) #if no work was done, poll 10 times per sec def serve_once(self): if self.json_call_queue.empty(): return False p_call = self.json_call_queue.get() #block here name = p_call.json_RPC_object["method"] params = p_call.json_RPC_object["params"] rpc_id = p_call.json_RPC_object.get("id") logging.debug( 'JSONRPC call. self.my_machine_ID: %s, method: %s, params: %s, rpc_id: %s, packet: %s' % (self.my_machine_ID, name, params, rpc_id, p_call)) if name not in self.method_dict: logging.error('JSONRPC method does not exist: ' % (name)) return True p_response = self.method_dict[name](params, rpc_id, p_call) #call JSONRPC function if rpc_id == None or p_response == None: #don't respond if RPC notify return True self.json_out_queue.put(p_response) #queue response return True def register_func(self, name, func): """Register a JSONRPC function """ self.method_dict.update({name: func}) ################################# #public JSONRPC functions ################################# def ping(self, params, rpc_id, packet): [ver, congest, args] = params [reply_machine_ID] = args logging.info('jsonrpc call. [reply_machine_ID]: %s' % (args)) return_address = self.machine_address_dict[reply_machine_ID] packet.json_RPC_object = dict(jsonrpc="2.0", result="pong", id=rpc_id) packet.to_address = return_address return packet def get_file(self, params, rpc_id, packet): [ver, congest, args] = params [reply_machine_ID, share_ID, file_name] = args logging.info( 'jsonrpc call. [reply_machine_ID, share_ID, file_name]: %s' % (args)) return_address = self.machine_address_dict[reply_machine_ID] #send ack packet.json_RPC_object = dict( jsonrpc="2.0", result=True, id=rpc_id ) #@TODO: send file size #@TODO: send new rpc_id for file chuncks packet.to_address = return_address self.json_out_queue.put(packet) self.transfer_file(share_ID, file_name, return_address) def save_file_block(self, params, rpc_id, packet): """Instructs peer to save this block of the file. File blocks are always sent in sizes divisible by 64 bytes. The maximum size of a UDP packet has to be less than 512MB, so, accounting for the header info the maximum binary payload should be less than 380 bytes. Bytes are sent in sets of 64 to reduce the accounting overhead. params: share_ID file_name file_offset (offset from beginning of file in multiples of 64 bytes) packet binary payload contains file block data """ [ver, congest, args] = params [reply_machine_ID, share_ID, file_name, file_offset] = args logging.info( 'jsonrpc call. [reply_machine_ID, share_ID, file_name, file_offset]: %s' % (args)) full_file_name = os.path.join(self.storage_directory, share_ID, file_name) if os.path.isfile(full_file_name): fp = open(full_file_name, 'r+b') else: fp = open(full_file_name, 'wb') fp.seek(64 * file_offset) fp.write(packet.binary_blob) fp.close() if rpc_id == None: return None packet.json_RPC_object = dict(jsonrpc="2.0", result=True, id=rpc_id) return_address = self.machine_address_dict[reply_machine_ID] packet.to_address = return_address packet.binary_blob = None return packet def update_machine_address(self, params, rpc_id, packet): [ver, congest, args] = params [machine_ID, url, port] = args self.machine_address_dict.update({machine_ID: (url, port)}) logging.info('Machine address dict updated: %s' % (self.machine_address_dict)) #TODO: packet response def add_share_to_machine( self, params, rpc_id, packet): #TODO: needs to be validated by a certificate each call? [ver, congest, args] = params [share_ID, machine_ID] = args if not share_ID in self.share_machine_dict: new_m_ID = [machine_ID] else: new_m_ID = self.share_machine_dict[share_ID] new_m_ID.append(machine_ID) self.share_machine_dict.update({share_ID: new_m_ID}) logging.info('Share %s added to machine %s' % (share_ID, machine_ID)) if not (machine_ID == self.my_machine_ID): return if not os.path.exists(os.path.join( self.storage_directory, share_ID)): #make local storage directories if necessary os.makedirs(os.path.join(self.storage_directory, share_ID)) logging.info('New share on localhost: %s' % (share_ID)) #TODO: packet response def has_file(self, params, rpc_id, packet): """ Polls the peer to see if they have a specific file. Returns true if they do. """ [ver, congest, args] = params [reply_machine_ID, share_ID, file_name] = args full_file_name = os.path.join(self.storage_directory, share_ID, file_name) response = os.path.isfile(full_file_name) packet.json_RPC_object = dict(jsonrpc="2.0", result=response, id=rpc_id) packet.to_address = self.machine_address_dict[reply_machine_ID] packet.binary_blob = None return packet def push_update_to_peer(self, params, rpc_id, packet): ''' Loop through all blobs in storage directory and poll peer to see if they have it. If the peer does not have the file, send it to them. If machine_ID is None then send updates to all machines with share. If current_commit is not None, then a sync_new_commit is sent to each peer ''' [ver, congest, args] = params [share_ID, machine_ID, current_commit] = args logging.info('Pushing update for %s to %s and update to commit: %s' % (share_ID, machine_ID, current_commit)) #Find all machines to push this update to to_transfer = dict() if machine_ID == None: logging.debug('dict: %s' % (self.share_machine_dict)) if share_ID not in self.share_machine_dict: logging.error('No machines with this share') return logging.debug(self.share_machine_dict[share_ID]) to_machines = self.share_machine_dict[share_ID] else: to_machines = [machine_ID] #delete my_machine_ID from to_machines if (self.my_machine_ID in to_machines): to_machines.remove(self.my_machine_ID) if len(to_machines) == 0: logging.info( 'No other machines have this share. Updates will not be pushed.' ) return logging.debug('Machines to push to: %s' % (to_machines)) logging.debug('share path: %s' % (os.path.join(self.storage_directory, share_ID))) #send has_file query for all files. for root, dirs, files in os.walk( os.path.join(self.storage_directory, share_ID)): logging.debug('files: %s' % (files)) for name in files: logging.debug('name: %s' % (name)) for machine in to_machines: address = self.machine_address_dict[machine] rpc_id = random.randint( 1, 1000000) #TODO: rpc_id collisions could occur p = Packet() p.json_RPC_object = dict( jsonrpc="2.0", method="has_file", params=[ 1.0, None, [self.my_machine_ID, share_ID, name] ], id=rpc_id) p.to_address = address self.send_block_choke(p, address, 3) to_transfer.update( {rpc_id: [name, machine]}) #TODO: rpc_id collisions could occur logging.info('Sent has_file to %s with following RPC_IDs: %s' % (to_machines, to_transfer.keys())) time.sleep(3) #wait for responses from peer logging.debug('json_response_dict: %s' % (self.json_response_dict)) #send the file if the query comes back negative for item in to_transfer.items(): [rpc_id, [name, machine]] = item if rpc_id not in self.json_response_dict: continue resp_packet = self.json_response_dict.pop(rpc_id) resp_value = resp_packet.json_RPC_object["result"] if not resp_value: #peer doesn't have this file, so send it self.transfer_file(share_ID, name, self.machine_address_dict[machine]) del to_transfer[rpc_id] logging.info('No response for following files: %s' % (to_transfer.items())) if current_commit == None: return time.sleep(3) #wait for responses from peer #Tell machines to update to current commit for machine in to_machines: address = self.machine_address_dict[machine] rpc_id = random.randint( 1, 1000000) #TODO: rpc_id collisions could occur p = Packet() p.json_RPC_object = dict( jsonrpc="2.0", method="sync_new_commit", params=[1.0, None, [current_commit, share_ID]], id=rpc_id) p.to_address = address self.send_block_choke(p, address, 3) def register_auto_sync(self, params, rpc_id, packet): """Sets up a daemon to perform commits whenever files have been changed in a given directory. Commits are then pushed to all relevant machines. """ [ver, congest, args] = params [key, monitoring_directory, share_ID, user_name, min_update_interval] = args logging.debug('Initializing auto sync') logging.debug( '[key, monitoring_directory, share_ID, user_name, min_update_interval]: %s' % (args)) self.autosyncmanager.create(key, monitoring_directory, self.storage_directory, share_ID, user_name, self.my_machine_ID, self.command_port, self.json_response_dict, min_update_interval) #self.autosyncmanager.start(monitoring_directory) #update the list of all autosynced working directories for this share working_directory_list = [monitoring_directory] if share_ID in self.autosync_share_working_directory_dict: working_directory_list.extend( self.autosync_share_working_directory_dict[share_ID]) working_directory_list = list( set(working_directory_list)) #make list unique self.autosync_share_working_directory_dict.update( {share_ID: working_directory_list}) def commit(self, params, rpc_id, packet): """ """ #@TODO: Ensure only authenticated users can access this command [ver, congest, args] = params [ key, working_directory, share_ID, user_name, commit_msg, parent_commit_hash, other_parent_commit_hash ] = args logging.debug( '[key, working_directory, share_ID, user_name, commit_msg, parent_commit_hash, other_parent_commit_hash]: ' % ([ key, working_directory, share_ID, user_name, commit_msg, parent_commit_hash, other_parent_commit_hash ])) #@TODO: If share_ID is None, look through all shares for the parent_commit_hash and use its share bm = local_blob_manager() commit_hash = bm.commit_directory( key, working_directory, os.path.join(self.storage_directory, share_ID), user_name, commit_msg, parent_commit_hash, other_parent_commit_hash) #@TODO: packet response is to myself for it is stored in shared object json_response_dict for use with autosync. Allow responding to other machines? return_address = ('localhost', self.command_port) packet.json_RPC_object = dict(jsonrpc="2.0", result=commit_hash, id=rpc_id) packet.to_address = return_address return packet def restore(self, params, rpc_id, packet): """ """ #@TODO: Ensure only authenticated users can access this command [ver, congest, args] = params [key, working_directory, commit_hash] = args bm = local_blob_manager() #@TODO: look for version already in working_directory and do differential update bm.restore_directory(key, working_directory, self.storage_directory, commit_hash) #@TODO: packet response? def sync_new_commit(self, params, rpc_id, packet): """ """ [ver, congest, args] = params [commit_hash, share_ID] = args logging.debug('[commit_hash, share_ID]: %s' % ([commit_hash, share_ID])) key = b'Sixteen byte key' #@TODO: what do I do about this key? #@TODO: this should sync by updating directory, not overwriting it. #@TODO: should commit this directory before updating it if share_ID not in self.autosync_share_working_directory_dict: logging.error('No autosync directories with share_ID: %s' % (share_ID)) return if commit_hash[0] != '_': commit_hash = '_' + commit_hash full_file_name = os.path.join(self.storage_directory, share_ID, commit_hash) if not os.path.isfile(full_file_name): logging.error('commit does not exist. path: %s' % (full_file_name)) return bm = local_blob_manager() working_directory_list = self.autosync_share_working_directory_dict[ share_ID] logging.debug('directories to sync: %s' % (working_directory_list)) for working_directory in working_directory_list: file_list, mod_times, last_commit_hash = local_blob_manager.read_commit_meta( working_directory) logging.debug('parent sync hash: %s' % (last_commit_hash)) if commit_hash == last_commit_hash: logging.debug( 'Working directory already contains desired commit') continue if working_directory in self.autosync_working_directory_dict: #disable watchdog while writing to directory self.autosyncmanager.stop(working_directory) (head, tail) = os.path.split( working_directory) #TODO: this seems like a hack bm.restore_directory( key, head, os.path.join(self.storage_directory, share_ID), commit_hash) if working_directory in self.autosync_working_directory_dict: self.autosyncmanager.start(working_directory) ################################# #JSONRPC helper functions ################################# def transfer_file(self, share_ID, file_name, return_address, start_offset=0, end_offset=999999999999, block_skip=1): """Sends a piece of a file to a peer in pieces. Initiated by JSONRPC: get_file() start_offset: start sending file from this offset end_offset: if this bytes is reached, send no more. block_skip: send a block, then skip this many blocks, then send the next etc. """ #continue to send all file blocks full_file_path = os.path.join(self.storage_directory, share_ID, file_name) f = open(full_file_path, 'rb') f.seek(start_offset) block_size = 5 file_offset = start_offset logging.debug('file_name: %s start_offset: %d, end_offset: %d' % (file_name, start_offset, end_offset)) while (file_offset < end_offset ): #@TODO: stop sending if given signal from return_address logging.debug('file_offset: %d' % (file_offset)) block_bytes = f.read(64 * block_size) if block_bytes == "": logging.debug('no bytes read from file') break p_out_block = Packet() p_out_block.json_RPC_object = dict( jsonrpc="2.0", method="save_file_block", params=[ 1.0, None, [self.my_machine_ID, share_ID, file_name, file_offset] ], ) #id=rpc_id p_out_block.binary_blob = block_bytes p_out_block.to_address = return_address self.send_block_choke(p_out_block, return_address, 3) time.sleep(0.002) file_offset += block_size logging.debug('finished file transfer') f.close() def send_block_choke(self, p, address, timeout=0): """Attempt to queue packet p for sending over the network. If the connection is choked this call will block until timeout, or until the connection becomes un-choked after which the packet will be queued. Returns True if packet was sent, False otherwise. """ #@TODO: Create a similar function (or same function with new parameters) that #blocks until the JSONRPC response is received, and then returns the result end_time = timeout + time.time() if not address in self.congest_connect_state: self.json_out_queue.put( p) #no congestion info on connection, so just send packet return True congest_state = self.congest_connect_state[address] while (congest_state == 'choke' ): #don't send packets if connection is choked if time.time() > end_time: return False time.sleep(0.01) congest_state = self.congest_connect_state[address] #connection has become unchoked self.json_out_queue.put(p) return True
class JSONServer(): def __init__(self): self.json_call_queue = None self.json_response_dict = None #JSON responses are pushed here indexed by rpc ID self.json_out_queue = None #packets of JSON responses are pushed here awaiting postprocessing and transmission self.process_pool = [] self.congest_connect_state = None self.storage_directory = "" self.a_s_list = [] #list of autoSync objects self.command_port = None #Objects shared across processes self.my_machine_ID = "" self.manager = Manager() self.share_machine_dict = self.manager.dict() #maps share IDs to machine IDs #self.share_key_dict = self.manager.dict() #maps share IDs to corresponding public keys (or certificates?) #TODO: implement self.machine_address_dict = self.manager.dict() #maps machine IDs to url (IP:port) self.autosync_share_working_directory_dict = self.manager.dict() #lists all autosynced working directories for a given share self.autosync_working_directory_dict = self.manager.dict() #given a working directory return the auto sync observer self.autosyncmanager = AutoSyncManager(self.json_response_dict) #register JSONRPC functions self.method_dict = dict() #function pointers to registered JSONRPC functions self.register_func('ping', self.ping) self.register_func('get_file', self.get_file) self.register_func('save_file_block', self.save_file_block) self.register_func('update_machine_address', self.update_machine_address) self.register_func('add_share_to_machine', self.add_share_to_machine) self.register_func('has_file', self.has_file) self.register_func('push_update_to_peer', self.push_update_to_peer) self.register_func('register_auto_sync', self.register_auto_sync) self.register_func('commit', self.commit) self.register_func('restore', self.restore) self.register_func('sync_new_commit', self.sync_new_commit) def start_pool(self, processes=None): if processes == None: processes = multiprocessing.cpu_count() for i in range(0,processes): proc = Process(target=self.serve_forever) proc.start() self.process_pool.append(proc) logging.info('JSONServer pool %d processes started' %(processes)) def terminate(self): for p in self.process_pool: p.terminate() #TODO: the autosync processes don't get cleaned up def serve_forever(self): Random.atfork() while True: if not self.serve_once(): time.sleep(0.1) #if no work was done, poll 10 times per sec def serve_once(self): if self.json_call_queue.empty(): return False p_call = self.json_call_queue.get() #block here name = p_call.json_RPC_object["method"] params = p_call.json_RPC_object["params"] rpc_id = p_call.json_RPC_object.get("id") logging.debug('JSONRPC call. self.my_machine_ID: %s, method: %s, params: %s, rpc_id: %s, packet: %s'%(self.my_machine_ID, name, params, rpc_id, p_call)) if name not in self.method_dict: logging.error('JSONRPC method does not exist: ' %(name)) return True p_response = self.method_dict[name](params, rpc_id, p_call) #call JSONRPC function if rpc_id == None or p_response == None: #don't respond if RPC notify return True self.json_out_queue.put(p_response) #queue response return True def register_func(self, name, func): """Register a JSONRPC function """ self.method_dict.update({name: func}) ################################# #public JSONRPC functions ################################# def ping(self, params, rpc_id, packet): [ver, congest, args] = params [reply_machine_ID] = args logging.info('jsonrpc call. [reply_machine_ID]: %s' %(args)) return_address = self.machine_address_dict[reply_machine_ID] packet.json_RPC_object = dict(jsonrpc="2.0", result="pong", id=rpc_id) packet.to_address = return_address return packet def get_file(self, params, rpc_id, packet): [ver, congest, args] = params [reply_machine_ID, share_ID, file_name] = args logging.info('jsonrpc call. [reply_machine_ID, share_ID, file_name]: %s' %(args)) return_address = self.machine_address_dict[reply_machine_ID] #send ack packet.json_RPC_object = dict(jsonrpc="2.0", result=True, id=rpc_id) #@TODO: send file size #@TODO: send new rpc_id for file chuncks packet.to_address = return_address self.json_out_queue.put(packet) self.transfer_file(share_ID, file_name, return_address) def save_file_block(self, params, rpc_id, packet): """Instructs peer to save this block of the file. File blocks are always sent in sizes divisible by 64 bytes. The maximum size of a UDP packet has to be less than 512MB, so, accounting for the header info the maximum binary payload should be less than 380 bytes. Bytes are sent in sets of 64 to reduce the accounting overhead. params: share_ID file_name file_offset (offset from beginning of file in multiples of 64 bytes) packet binary payload contains file block data """ [ver, congest, args] = params [reply_machine_ID, share_ID, file_name, file_offset] = args logging.info('jsonrpc call. [reply_machine_ID, share_ID, file_name, file_offset]: %s' %(args)) full_file_name = os.path.join(self.storage_directory, share_ID, file_name) if os.path.isfile(full_file_name): fp = open(full_file_name,'r+b') else: fp = open(full_file_name,'wb') fp.seek(64*file_offset) fp.write(packet.binary_blob) fp.close() if rpc_id == None: return None packet.json_RPC_object = dict(jsonrpc="2.0", result=True, id=rpc_id) return_address = self.machine_address_dict[reply_machine_ID] packet.to_address = return_address packet.binary_blob = None return packet def update_machine_address(self, params, rpc_id, packet): [ver, congest, args] = params [machine_ID, url, port] = args self.machine_address_dict.update({machine_ID:(url,port)}) logging.info('Machine address dict updated: %s' %(self.machine_address_dict)) #TODO: packet response def add_share_to_machine(self, params, rpc_id, packet): #TODO: needs to be validated by a certificate each call? [ver, congest, args] = params [share_ID, machine_ID] = args if not share_ID in self.share_machine_dict: new_m_ID = [machine_ID] else: new_m_ID = self.share_machine_dict[share_ID] new_m_ID.append(machine_ID) self.share_machine_dict.update({share_ID:new_m_ID}) logging.info('Share %s added to machine %s' %(share_ID, machine_ID)) if not (machine_ID == self.my_machine_ID): return if not os.path.exists(os.path.join(self.storage_directory, share_ID)): #make local storage directories if necessary os.makedirs(os.path.join(self.storage_directory, share_ID)) logging.info('New share on localhost: %s' %(share_ID)) #TODO: packet response def has_file(self, params, rpc_id, packet): """ Polls the peer to see if they have a specific file. Returns true if they do. """ [ver, congest, args] = params [reply_machine_ID, share_ID, file_name] = args full_file_name = os.path.join(self.storage_directory, share_ID, file_name) response = os.path.isfile(full_file_name) packet.json_RPC_object = dict(jsonrpc="2.0", result=response, id=rpc_id) packet.to_address = self.machine_address_dict[reply_machine_ID] packet.binary_blob = None return packet def push_update_to_peer(self, params, rpc_id, packet): ''' Loop through all blobs in storage directory and poll peer to see if they have it. If the peer does not have the file, send it to them. If machine_ID is None then send updates to all machines with share. If current_commit is not None, then a sync_new_commit is sent to each peer ''' [ver, congest, args] = params [share_ID, machine_ID, current_commit] = args logging.info('Pushing update for %s to %s and update to commit: %s' %(share_ID, machine_ID, current_commit)) #Find all machines to push this update to to_transfer = dict() if machine_ID == None: logging.debug('dict: %s' %(self.share_machine_dict)) if share_ID not in self.share_machine_dict: logging.error('No machines with this share') return logging.debug(self.share_machine_dict[share_ID]) to_machines = self.share_machine_dict[share_ID] else: to_machines = [machine_ID] #delete my_machine_ID from to_machines if (self.my_machine_ID in to_machines): to_machines.remove(self.my_machine_ID) if len(to_machines)==0: logging.info('No other machines have this share. Updates will not be pushed.') return logging.debug('Machines to push to: %s'%(to_machines)) logging.debug('share path: %s' %(os.path.join(self.storage_directory, share_ID))) #send has_file query for all files. for root, dirs, files in os.walk(os.path.join(self.storage_directory, share_ID)): logging.debug('files: %s' %(files)) for name in files: logging.debug('name: %s' %(name)) for machine in to_machines: address = self.machine_address_dict[machine] rpc_id=random.randint(1,1000000) #TODO: rpc_id collisions could occur p = Packet() p.json_RPC_object = dict(jsonrpc="2.0", method="has_file", params=[1.0, None, [self.my_machine_ID, share_ID, name]], id=rpc_id) p.to_address = address self.send_block_choke(p, address, 3) to_transfer.update({rpc_id:[name, machine]}) #TODO: rpc_id collisions could occur logging.info('Sent has_file to %s with following RPC_IDs: %s' %(to_machines, to_transfer.keys())) time.sleep(3) #wait for responses from peer logging.debug('json_response_dict: %s'%(self.json_response_dict)) #send the file if the query comes back negative for item in to_transfer.items(): [rpc_id, [name, machine]] = item if rpc_id not in self.json_response_dict: continue resp_packet = self.json_response_dict.pop(rpc_id) resp_value = resp_packet.json_RPC_object["result"] if not resp_value: #peer doesn't have this file, so send it self.transfer_file(share_ID, name, self.machine_address_dict[machine]) del to_transfer[rpc_id] logging.info('No response for following files: %s' %(to_transfer.items())) if current_commit == None: return time.sleep(3) #wait for responses from peer #Tell machines to update to current commit for machine in to_machines: address = self.machine_address_dict[machine] rpc_id=random.randint(1,1000000) #TODO: rpc_id collisions could occur p = Packet() p.json_RPC_object = dict(jsonrpc="2.0", method="sync_new_commit", params=[1.0, None, [current_commit, share_ID]], id=rpc_id) p.to_address = address self.send_block_choke(p, address, 3) def register_auto_sync(self, params, rpc_id, packet): """Sets up a daemon to perform commits whenever files have been changed in a given directory. Commits are then pushed to all relevant machines. """ [ver, congest, args] = params [key, monitoring_directory, share_ID, user_name, min_update_interval] = args logging.debug('Initializing auto sync') logging.debug('[key, monitoring_directory, share_ID, user_name, min_update_interval]: %s' %(args)) self.autosyncmanager.create(key, monitoring_directory, self.storage_directory, share_ID, user_name, self.my_machine_ID, self.command_port, self.json_response_dict, min_update_interval) #self.autosyncmanager.start(monitoring_directory) #update the list of all autosynced working directories for this share working_directory_list = [monitoring_directory] if share_ID in self.autosync_share_working_directory_dict: working_directory_list.extend(self.autosync_share_working_directory_dict[share_ID]) working_directory_list = list(set(working_directory_list)) #make list unique self.autosync_share_working_directory_dict.update({share_ID:working_directory_list}) def commit(self, params, rpc_id, packet): """ """ #@TODO: Ensure only authenticated users can access this command [ver, congest, args] = params [key, working_directory, share_ID, user_name, commit_msg, parent_commit_hash, other_parent_commit_hash] = args logging.debug('[key, working_directory, share_ID, user_name, commit_msg, parent_commit_hash, other_parent_commit_hash]: ' %([key, working_directory, share_ID, user_name, commit_msg, parent_commit_hash, other_parent_commit_hash])) #@TODO: If share_ID is None, look through all shares for the parent_commit_hash and use its share bm = local_blob_manager() commit_hash = bm.commit_directory(key, working_directory, os.path.join(self.storage_directory, share_ID), user_name, commit_msg, parent_commit_hash, other_parent_commit_hash) #@TODO: packet response is to myself for it is stored in shared object json_response_dict for use with autosync. Allow responding to other machines? return_address = ('localhost', self.command_port) packet.json_RPC_object = dict(jsonrpc="2.0", result=commit_hash, id=rpc_id) packet.to_address = return_address return packet def restore(self, params, rpc_id, packet): """ """ #@TODO: Ensure only authenticated users can access this command [ver, congest, args] = params [key, working_directory, commit_hash] = args bm = local_blob_manager() #@TODO: look for version already in working_directory and do differential update bm.restore_directory(key, working_directory, self.storage_directory, commit_hash) #@TODO: packet response? def sync_new_commit(self, params, rpc_id, packet): """ """ [ver, congest, args] = params [commit_hash, share_ID] = args logging.debug('[commit_hash, share_ID]: %s'%([commit_hash, share_ID])) key=b'Sixteen byte key' #@TODO: what do I do about this key? #@TODO: this should sync by updating directory, not overwriting it. #@TODO: should commit this directory before updating it if share_ID not in self.autosync_share_working_directory_dict: logging.error('No autosync directories with share_ID: %s' %(share_ID)) return if commit_hash[0] != '_': commit_hash = '_' + commit_hash full_file_name = os.path.join(self.storage_directory, share_ID, commit_hash) if not os.path.isfile(full_file_name): logging.error('commit does not exist. path: %s' %(full_file_name)) return bm = local_blob_manager() working_directory_list = self.autosync_share_working_directory_dict[share_ID] logging.debug('directories to sync: %s' %(working_directory_list)) for working_directory in working_directory_list: file_list, mod_times, last_commit_hash = local_blob_manager.read_commit_meta(working_directory) logging.debug('parent sync hash: %s'%(last_commit_hash)) if commit_hash == last_commit_hash: logging.debug('Working directory already contains desired commit') continue if working_directory in self.autosync_working_directory_dict: #disable watchdog while writing to directory self.autosyncmanager.stop(working_directory) (head, tail) = os.path.split(working_directory) #TODO: this seems like a hack bm.restore_directory(key, head, os.path.join(self.storage_directory, share_ID), commit_hash) if working_directory in self.autosync_working_directory_dict: self.autosyncmanager.start(working_directory) ################################# #JSONRPC helper functions ################################# def transfer_file(self, share_ID, file_name, return_address, start_offset=0, end_offset=999999999999, block_skip=1): """Sends a piece of a file to a peer in pieces. Initiated by JSONRPC: get_file() start_offset: start sending file from this offset end_offset: if this bytes is reached, send no more. block_skip: send a block, then skip this many blocks, then send the next etc. """ #continue to send all file blocks full_file_path = os.path.join(self.storage_directory, share_ID, file_name) f = open(full_file_path, 'rb') f.seek(start_offset) block_size = 5 file_offset = start_offset logging.debug('file_name: %s start_offset: %d, end_offset: %d' %(file_name, start_offset, end_offset)) while (file_offset < end_offset): #@TODO: stop sending if given signal from return_address logging.debug('file_offset: %d' %(file_offset)) block_bytes = f.read(64*block_size) if block_bytes == "": logging.debug('no bytes read from file') break p_out_block = Packet() p_out_block.json_RPC_object = dict(jsonrpc="2.0", method="save_file_block", params=[1.0, None, [self.my_machine_ID, share_ID, file_name, file_offset]], ) #id=rpc_id p_out_block.binary_blob = block_bytes p_out_block.to_address = return_address self.send_block_choke(p_out_block, return_address, 3) time.sleep(0.002) file_offset+=block_size logging.debug('finished file transfer') f.close() def send_block_choke(self, p, address, timeout=0): """Attempt to queue packet p for sending over the network. If the connection is choked this call will block until timeout, or until the connection becomes un-choked after which the packet will be queued. Returns True if packet was sent, False otherwise. """ #@TODO: Create a similar function (or same function with new parameters) that #blocks until the JSONRPC response is received, and then returns the result end_time = timeout + time.time() if not address in self.congest_connect_state: self.json_out_queue.put(p) #no congestion info on connection, so just send packet return True congest_state = self.congest_connect_state[address] while(congest_state == 'choke'): #don't send packets if connection is choked if time.time()>end_time: return False time.sleep(0.01) congest_state = self.congest_connect_state[address] #connection has become unchoked self.json_out_queue.put(p) return True