def doVerifyDHTRecords(self, *args, **kwargs): """ Action method. """ if _Debug: lg.args(_DebugLevel, my=self.my_position, desired=self.desired_position, dht=self.dht_brokers) if not self.dht_brokers.get(self.desired_position): # desired position is empty in DHT if self.desired_position == 0: self.automat('top-record-empty') else: prev_position = self.desired_position - 1 if not self.dht_brokers.get(prev_position): # also previous position in DHT is empty as well self.automat('prev-record-empty') else: if id_url.is_the_same(self.dht_brokers[prev_position], self.my_broker_idurl): # found my DHT record on a previous position - request was done on another position self.automat('prev-record-own', dht_brokers=self.dht_brokers) else: # found another broker on the previous position in DHT self.automat('prev-record-busy') return if id_url.is_the_same(self.dht_brokers[self.desired_position], self.my_broker_idurl): # my own record is present in DHT on that position if self.desired_position == 0: self.automat('top-record-own') else: prev_position = self.desired_position - 1 if not self.dht_brokers.get(prev_position): # but the previous position in DHT is empty self.automat('prev-record-empty') else: if id_url.is_the_same(self.dht_brokers[prev_position], self.my_broker_idurl): # found my DHT record on a previous position - this is wrong! self.automat('prev-record-own', dht_brokers=self.dht_brokers) else: # found another broker on the previous position in DHT self.automat('prev-record-busy') # self.automat('record-own') return # desired position is occupied by another broker in DHT records if self.desired_position == 0: self.automat('top-record-busy', dht_brokers=self.dht_brokers) else: self.automat('record-busy')
def _on_rotate_broker_connected(self, response_info, broker_pos, event, *args, **kwargs): try: # skip leading "accepted:" marker cooperated_brokers = jsn.loads( strng.to_text(response_info[0].Payload)[9:]) cooperated_brokers.pop('archive_folder_path', None) cooperated_brokers = { int(k): id_url.field(v) for k, v in cooperated_brokers.items() } except: lg.exc() self.automat('broker-rotate-failed') return if _Debug: lg.args(_DebugLevel, cooperated=cooperated_brokers, pos=broker_pos, e=event) if id_url.is_the_same(cooperated_brokers.get(broker_pos), self.my_broker_idurl): self.automat('broker-rotate-accepted', cooperated_brokers=cooperated_brokers) return self.automat('broker-rotate-rejected', cooperated_brokers=cooperated_brokers)
def _do_dht_push_state(self, event=None, **kwargs): if _Debug: lg.args(_DebugLevel, known_position=self.known_position, cooperated_brokers=self.cooperated_brokers, e=event, kw=kwargs) desired_position = self.known_position if desired_position is None or desired_position == -1: for pos, idurl in self.cooperated_brokers.items(): if idurl and id_url.is_the_same(idurl, self.broker_idurl): desired_position = pos if desired_position is None or desired_position == -1: raise Exception('not able to write record into DHT, my position is unknown') archive_folder_path = None if self.current_connect_request and self.current_connect_request.get('archive_folder_path'): archive_folder_path = self.current_connect_request['archive_folder_path'] prev_revision = self.latest_dht_records.get(desired_position, {}).get('revision', None) if prev_revision is None: prev_revision = 0 if _Debug: lg.args(_DebugLevel, c=self.customer_id, p=desired_position, b=self.broker_id, r=prev_revision+1) self._do_dht_write( desired_position=desired_position, archive_folder_path=archive_folder_path, revision=prev_revision+1, retry=True, event=event, **kwargs, )
def _on_new_broker_hired(self, response_info, broker_pos, my_pos, desired_pos, *args, **kwargs): try: # skip leading "accepted:" marker cooperated_brokers = jsn.loads( strng.to_text(response_info[0].Payload)[9:]) cooperated_brokers.pop('archive_folder_path', None) cooperated_brokers = { int(k): id_url.field(v) for k, v in cooperated_brokers.items() } except: lg.exc() self.automat('hire-broker-failed') return if _Debug: lg.args(_DebugLevel, cooperated=cooperated_brokers, target=broker_pos, my=my_pos, desired=desired_pos, args=args, kwargs=kwargs) if my_pos >= 0: if id_url.is_the_same(cooperated_brokers.get(my_pos), self.my_broker_idurl): self.automat('hire-broker-ok', cooperated_brokers=cooperated_brokers) return if desired_pos >= 0: if id_url.is_the_same(cooperated_brokers.get(desired_pos), self.my_broker_idurl): self.automat('hire-broker-ok', cooperated_brokers=cooperated_brokers) return lg.warn( 'new broker is not cooperative, my idurl is not found in the cooperation on the right place' ) self.automat('new-broker-rejected', cooperated_brokers=cooperated_brokers)
def verify_broker(self, broker_idurl, position, known_brokers, known_streams): if not self.InSync or self.state != 'CONNECTED': lg.warn('not able to verify another broker because %r is not in sync' % self) return None if self.cooperated_brokers: if id_url.is_not_in(broker_idurl, self.cooperated_brokers.values()): return 'unknown broker' if position not in self.cooperated_brokers.keys(): return 'unknown position' if not id_url.is_the_same(self.cooperated_brokers[position], broker_idurl): return 'position mismatch' if len(self.cooperated_brokers) != len(known_brokers): return 'brokers count mismatch' for i, b_idurl in self.cooperated_brokers.items(): if not id_url.is_the_same(known_brokers[i], b_idurl): return 'broker mismatch' if self.known_streams and known_streams: for my_queue_alias, my_archive_folder_path in self.known_streams.items(): for other_queue_alias, other_archive_folder_path in self.known_streams.items(): # TODO: verify streams pass return None
def doRememberOwnPosition(self, *args, **kwargs): """ Action method. """ my_new_position = -1 for pos, idurl in self.cooperated_brokers.items(): if idurl and id_url.is_the_same(idurl, self.broker_idurl): my_new_position = pos archive_folder_path = kwargs.get('archive_folder_path') if _Debug: lg.args(_DebugLevel, known=self.known_position, new=my_new_position, cooperated=self.cooperated_brokers, af_path=archive_folder_path) if my_new_position >= 0: self.known_position = my_new_position if self.current_connect_request and self.current_connect_request.get('request') == 'connect' and archive_folder_path: try: self.known_streams[self.current_connect_request['group_key_info']['alias']] = archive_folder_path except: lg.exc(str(self.current_connect_request))
def isAnyFilesShared(self, *args, **kwargs): """ Condition method. """ if id_url.is_the_same(self.customer_idurl, my_id.getIDURL()): if _Debug: lg.args(_DebugLevel, c=self.customer_idurl, key_alias=self.key_alias, ret='my own share') # no need to wait for incoming list files # TODO: check what is going to happen when new file is uploaded to the share by another user (not creator of the share) return True ret = backup_fs.HasChilds('', iter=backup_fs.fs(self.customer_idurl, self.key_alias)) if _Debug: lg.args(_DebugLevel, customer=self.customer_idurl, key_alias=self.key_alias, ret=ret) return ret
def on_files_received(newpacket, info): list_files_global_id = global_id.ParseGlobalID(newpacket.PacketID) if not list_files_global_id['idurl']: lg.warn('invalid PacketID: %s' % newpacket.PacketID) return False if not id_url.is_the_same(list_files_global_id['idurl'], my_id.getIDURL()): # ignore Files() if this is another customer if _Debug: lg.dbg(_DebugLevel, 'ignore incoming %r which is owned by another customer' % newpacket) return False if not contactsdb.is_supplier(newpacket.OwnerID): # ignore Files() if this is not my supplier if _Debug: lg.dbg(_DebugLevel, 'incoming %r received, but %r is not my supplier' % (newpacket, newpacket.OwnerID, )) return False if _Debug: lg.args(_DebugLevel, "service_backups._on_inbox_packet_received: %r for us from %s at %s" % ( newpacket, newpacket.CreatorID, info)) if IncomingSupplierListFiles(newpacket, list_files_global_id): p2p_service.SendAck(newpacket) else: p2p_service.SendFail(newpacket) return True
def on_identity_url_changed(evt): from access import group_member service_dir = settings.ServiceDir('service_private_groups') groups_dir = os.path.join(service_dir, 'groups') brokers_dir = os.path.join(service_dir, 'brokers') old_idurl = id_url.field(evt.data['old_idurl']) new_idurl = id_url.field(evt.data['new_idurl']) active_group_keys = list(active_groups()) to_be_reconnected = [] for group_key_id in active_group_keys: if not group_key_id: continue group_creator_idurl = global_id.glob2idurl(group_key_id) if id_url.is_the_same(group_creator_idurl, old_idurl): old_group_path = os.path.join(groups_dir, group_key_id) latest_group_key_id = my_keys.latest_key_id(group_key_id) latest_group_path = os.path.join(groups_dir, latest_group_key_id) lg.info('going to rename rotated group file: %r -> %r' % (old_group_path, latest_group_path, )) if os.path.isfile(old_group_path): try: os.rename(old_group_path, latest_group_path) except: lg.exc() continue else: lg.warn('key file %r was not found, key was not renamed' % old_group_path) active_groups()[latest_group_key_id] = active_groups().pop(group_key_id) group_member.rotate_active_group_memeber(group_key_id, latest_group_key_id) gm = group_member.get_active_group_member(group_key_id) if gm and gm.connected_brokers and id_url.is_in(old_idurl, gm.connected_brokers.values()): lg.info('connected broker %r IDURL is rotated, going to reconnect %r' % (old_idurl, gm, )) if group_key_id not in to_be_reconnected: to_be_reconnected.append(group_key_id) known_customers = list(known_brokers().keys()) for customer_id in known_customers: latest_customer_id = global_id.idurl2glob(new_idurl) customer_idurl = global_id.glob2idurl(customer_id) if id_url.is_the_same(customer_idurl, old_idurl): latest_customer_dir = os.path.join(brokers_dir, latest_customer_id) lg.info('going to rename rotated customer id: %r -> %r' % (customer_id, latest_customer_id, )) old_customer_dir = os.path.join(brokers_dir, customer_id) if os.path.isdir(old_customer_dir): try: bpio.move_dir_recursive(old_customer_dir, latest_customer_dir) bpio.rmdir_recursive(old_customer_dir) except: lg.exc() continue known_brokers()[latest_customer_id] = known_brokers().pop(customer_id) for broker_pos, broker_id in enumerate(known_brokers(latest_customer_id)): if not broker_id: continue broker_idurl = global_id.glob2idurl(broker_id) if broker_idurl == old_idurl: latest_broker_id = global_id.idurl2glob(new_idurl) latest_broker_path = os.path.join(latest_customer_dir, latest_broker_id) lg.info('going to rename rotated broker id: %r -> %r' % (broker_id, latest_broker_id, )) old_broker_path = os.path.join(latest_customer_dir, broker_id) if os.path.isfile(old_broker_path): try: os.rename(old_broker_path, latest_broker_path) except: lg.exc() continue if latest_broker_id in known_brokers(latest_customer_id): lg.warn('broker %r already exist' % latest_broker_id) continue known_brokers()[latest_customer_id][broker_pos] = latest_broker_id if _Debug: lg.args(_DebugLevel, to_be_reconnected=to_be_reconnected) for group_key_id in to_be_reconnected: gm = group_member.get_active_group_member(group_key_id) if gm: gm.automat('reconnect')
def doVerifyKnownBrokers(self, *args, **kwargs): """ Action method. """ if _Debug: lg.args(_DebugLevel, my=self.my_position, desired=self.desired_position, cooperated=self.cooperated_brokers) # cooperation was done before with other brokers and my own position is known already to me if self.desired_position != self.my_position: # but the request was done to a wrong position lg.warn( 'requester desired position %d mismatch, my current position is: %d' % ( self.desired_position, self.my_position, )) self.automat( 'request-invalid', Exception('position mismatch, current position is: %d' % self.my_position), cooperated_brokers=self.cooperated_brokers) return if not self.cooperated_brokers.get(self.my_position): # there is no broker present in the cooperation for my position lg.err( 'my current cooperated info is not valid, there is no broker on position %d' % self.my_position) self.automat( 'request-invalid', Exception( 'current cooperation is not valid, there is no broker on position %d' % self.my_position)) return if not id_url.is_the_same(self.cooperated_brokers[self.my_position], self.my_broker_idurl): # looks like my current deal is not correct - another broker is taking my position lg.err( 'my current cooperated info is not correct, another broker is taking my position' ) self.automat( 'request-invalid', Exception( 'current cooperation is not valid, another broker is taking position %d' % self.my_position)) return if self.my_position > 0: prev_position = self.my_position - 1 if not self.cooperated_brokers.get(prev_position): # but on the previous position there is no broker present in the cooperation lg.err( 'my current cooperated info is not valid, there is no broker on previous position %d' % prev_position) self.automat( 'request-invalid', Exception( 'current cooperation is not valid, there is no broker on previous position %d' % prev_position)) return # requester is trying to connect to me on the correct position if self.requestor_known_brokers.get(self.my_position): if not id_url.is_the_same( self.requestor_known_brokers[self.my_position], self.my_broker_idurl): # but there is a request to change the cooperation - it looks like a trigger for a brokers rotation lg.warn( 'received a request to change the cooperation, another broker %r going to replace me on position %d' % ( self.requestor_known_brokers[self.my_position], self.my_position, )) if not self.dht_brokers.get(self.my_position): # there is no record in DHT for my position # my info is not stored in DHT and another broker is going to replace me # but there is already a cooperation done before and my own position is known to me if self.my_position == 0: self.automat('my-top-record-empty-replace') else: self.automat('my-record-empty-replace') return # there is a record in DHT on my expected position while another broker is trying to replace me if not id_url.is_the_same(self.dht_brokers[self.my_position], self.my_broker_idurl): # DHT record on my expected position is occupied by another broker lg.warn( 'DHT record on my expected position %d is occupied by another broker %r and also another broker is trying to replace me: %r' % ( self.my_position, self.dht_brokers[self.my_position], self.requestor_known_brokers[self.my_position], )) if self.my_position == 0: self.automat('my-top-record-busy-replace') else: self.automat('my-record-busy-replace') return # found my own record on the expected position in DHT while another broker is trying to replace me if self.my_position == 0: self.automat('my-top-record-own-replace') else: self.automat('my-record-own-replace') return # requester is not going to change the existing cooperation for my own position if not self.dht_brokers.get(self.my_position): # but in DHT my own record is missing on the expected position lg.warn('DHT record on my expected position %d is empty' % self.my_position) if self.my_position == 0: self.automat('my-top-record-empty') else: self.automat('my-record-empty') return # there is a record in DHT on my expected position if not id_url.is_the_same(self.dht_brokers[self.my_position], self.my_broker_idurl): # DHT record on my expected position is occupied by another broker lg.warn( 'DHT record on my expected position %d is occupied by another broker: %r' % ( self.my_position, self.dht_brokers[self.my_position], )) if self.my_position == 0: self.automat('my-top-record-busy') else: self.automat('my-record-busy') return # found my own record on expected position in DHT if self.my_position == 0: self.automat('my-top-record-own') else: self.automat('my-record-own')
def _on_packet_request_result(self, NewPacketOrPacketID, result): if self.block_requests is None: return if _Debug: lg.args(_DebugLevel, packet=NewPacketOrPacketID, result=result) packet_id = None if strng.is_string(NewPacketOrPacketID): packet_id = NewPacketOrPacketID else: packet_id = getattr(NewPacketOrPacketID, 'PacketID', None) if not packet_id: raise Exception('packet ID is unknown from %r' % NewPacketOrPacketID) if packet_id not in self.block_requests: resp = global_id.NormalizeGlobalID(packet_id) for req_packet_id in self.block_requests: req = global_id.NormalizeGlobalID(req_packet_id) if resp['version'] == req['version'] and resp['path'] == req[ 'path']: if resp['key_alias'] == req['key_alias'] and resp[ 'user'] == req['user']: if id_url.is_the_same(resp['idurl'], req['idurl']): packet_id = req_packet_id lg.warn( 'found matching packet request %r for rotated idurl %r' % ( packet_id, resp['idurl'], )) break if packet_id not in self.block_requests: if _Debug: lg.args(_DebugLevel, block_requests=self.block_requests) raise Exception('packet ID not registered') if result == 'in queue': if self.block_requests[packet_id] is not None: raise Exception( 'packet is still in IO queue, but already unregistered') lg.warn('packet already in the request queue: %r' % packet_id) return if result in [ 'received', 'exist', ]: self.block_requests[packet_id] = True if result == 'exist': # reactor.callLater(0, self.automat, 'data-received', (None, packet_id, )) # @UndefinedVariable self.event('data-received', ( None, packet_id, )) else: # reactor.callLater(0, self.automat, 'data-received', (NewPacketOrPacketID, packet_id, )) # @UndefinedVariable self.event('data-received', ( NewPacketOrPacketID, packet_id, )) else: self.block_requests[packet_id] = False self.RequestFails.append(packet_id) # reactor.callLater(0, self.automat, 'request-failed', packet_id) # @UndefinedVariable self.event('request-failed', packet_id)
def contacts_remote(include_all=False, include_enabled=True): """ Return ID's list of all known peers. """ l = id_url.to_bin_list(contacts(include_all=include_all, include_enabled=include_enabled)) return [i for i in id_url.fields_list(l) if not id_url.is_the_same(i, my_id.getIDURL())]