def doRemoveSuppliers(self, *args, **kwargs): """ Action method. """ current_suppliers = contactsdb.suppliers() desired_suppliers = settings.getSuppliersNumberDesired() if len(current_suppliers) < desired_suppliers: lg.warn('must have more suppliers %d<%d' % ( len(current_suppliers), desired_suppliers)) removed_suppliers = [] for supplier_idurl in self.dismiss_list: if supplier_idurl not in current_suppliers: lg.warn('%s not a supplier' % supplier_idurl) continue pos = current_suppliers.index(supplier_idurl) # current_suppliers.remove(supplier_idurl) current_suppliers[pos] = '' removed_suppliers.append((pos, supplier_idurl,)) misc.writeSupplierData( supplier_idurl, 'disconnected', time.strftime('%d-%m-%Y %H:%M:%S'), my_id.getLocalID(), ) current_suppliers = current_suppliers[:desired_suppliers] contactsdb.update_suppliers(current_suppliers) contactsdb.save_suppliers() from main import control control.on_suppliers_changed(current_suppliers) for position, supplier_idurl in removed_suppliers: events.send('supplier-modified', dict( new_idurl=None, old_idurl=supplier_idurl, position=position, )) lg.out(2, '!!!!!!!!!!! REMOVE SUPPLIERS : %d' % len(self.dismiss_list))
def _do_save_customer_suppliers(id_cached_result, ret): if my_id.getIDURL() != id_url.field(ret['customer_idurl']): contactsdb.set_suppliers(ret['suppliers'], customer_idurl=ret['customer_idurl']) contactsdb.save_suppliers(customer_idurl=ret['customer_idurl']) if ret.get('ecc_map'): for supplier_idurl in ret['suppliers']: if supplier_idurl and id_url.is_cached(supplier_idurl): contactsdb.add_supplier_meta_info( supplier_idurl=supplier_idurl, info={ 'ecc_map': ret['ecc_map'], }, customer_idurl=ret['customer_idurl'], ) else: if _Debug: lg.out( _DebugLevel, 'dht_relations._do_save_customer_suppliers SKIP processing my own suppliers' ) if _Debug: lg.out( _DebugLevel, 'dht_relations._do_save_customer_suppliers OK for %r returned %d suppliers' % ( ret['customer_idurl'], len(ret['suppliers']), )) result.callback(ret) return ret
def doRemoveSuppliers(self, arg): """ Action method. """ current_suppliers = contactsdb.suppliers() desired_suppliers = settings.getSuppliersNumberDesired() if len(current_suppliers) < desired_suppliers: lg.warn('must have more suppliers %d<%d' % ( len(current_suppliers), desired_suppliers)) for supplier_idurl in self.dismiss_list: if supplier_idurl not in current_suppliers: lg.warn('%s not a supplier' % supplier_idurl) continue pos = current_suppliers.index(supplier_idurl) # current_suppliers.remove(supplier_idurl) current_suppliers[pos] = '' misc.writeSupplierData( supplier_idurl, 'disconnected', time.strftime('%d-%m-%Y %H:%M:%S')) current_suppliers = current_suppliers[:desired_suppliers] contactsdb.update_suppliers(current_suppliers) contactsdb.save_suppliers() if settings.NewWebGUI(): from web import control control.on_suppliers_changed(current_suppliers) else: from web import webcontrol webcontrol.OnListSuppliers() lg.out(2, '!!!!!!!!!!! REMOVE SUPPLIERS : %d' % len(self.dismiss_list))
def doSubstituteSupplier(self, arg): """ Action method. """ new_idurl = arg current_suppliers = list(contactsdb.suppliers()) if new_idurl in current_suppliers: raise Exception('%s is already supplier' % new_idurl) position = -1 old_idurl = None for i in range(len(current_suppliers)): if current_suppliers[i].strip() == '': position = i break if current_suppliers[i] in self.dismiss_list: # self.dismiss_list.remove(current_suppliers[i]) position = i old_idurl = current_suppliers[i] break lg.out(10, 'fire_hire.doSubstituteSupplier position=%d' % position) if position < 0: current_suppliers.append(new_idurl) else: current_suppliers[position] = new_idurl contactsdb.update_suppliers(current_suppliers) contactsdb.save_suppliers() misc.writeSupplierData( new_idurl, 'connected', time.strftime('%d-%m-%Y %H:%M:%S'), my_id.getLocalID(), ) if settings.NewWebGUI(): from web import control control.on_suppliers_changed(current_suppliers) else: from web import webcontrol webcontrol.OnListSuppliers() if position < 0: lg.out(2, '!!!!!!!!!!! ADDED NEW SUPPLIER : %s' % (new_idurl)) events.send('supplier-modified', dict( new_idurl=new_idurl, old_idurl=None, position=(len(current_suppliers) - 1), )) else: if old_idurl: lg.out(2, '!!!!!!!!!!! SUBSTITUTE EXISTING SUPPLIER %d : %s->%s' % (position, old_idurl, new_idurl)) events.send('supplier-modified', dict( new_idurl=new_idurl, old_idurl=old_idurl, position=position, )) else: lg.out(2, '!!!!!!!!!!! REPLACE EMPTY SUPPLIER %d : %s' % (position, new_idurl)) events.send('supplier-modified', dict( new_idurl=new_idurl, old_idurl=None, position=position, )) self.restart_interval = 1.0
def doSubstituteSupplier(self, arg): """ Action method. """ new_idurl = arg current_suppliers = list(contactsdb.suppliers()) if new_idurl in current_suppliers: raise Exception('%s is already supplier' % new_idurl) position = -1 old_idurl = None for i in range(len(current_suppliers)): if current_suppliers[i].strip() == '': position = i break if current_suppliers[i] in self.dismiss_list: # self.dismiss_list.remove(current_suppliers[i]) position = i old_idurl = current_suppliers[i] break lg.out(10, 'fire_hire.doSubstituteSupplier position=%d' % position) if position < 0: current_suppliers.append(new_idurl) else: current_suppliers[position] = new_idurl contactsdb.update_suppliers(current_suppliers) contactsdb.save_suppliers() misc.writeSupplierData( new_idurl, 'connected', time.strftime('%d-%m-%Y %H:%M:%S')) if settings.NewWebGUI(): from web import control # control.on_suppliers_changed(current_suppliers) else: from web import webcontrol webcontrol.OnListSuppliers() if position < 0: lg.out(2, '!!!!!!!!!!! ADD SUPPLIER : %s' % (new_idurl)) else: if old_idurl: lg.out( 2, '!!!!!!!!!!! SUBSTITUTE SUPPLIER %d : %s->%s' % (position, old_idurl, new_idurl)) else: lg.out( 2, '!!!!!!!!!!! REPLACE EMPTY SUPPLIER %d : %s' % (position, new_idurl)) self.restart_interval = 1.0
def doRemoveSuppliers(self, *args, **kwargs): """ Action method. """ current_suppliers = contactsdb.suppliers() desired_suppliers = settings.getSuppliersNumberDesired() if len(current_suppliers) < desired_suppliers: lg.warn('must have more suppliers %d<%d' % (len(current_suppliers), desired_suppliers)) removed_suppliers = [] for supplier_idurl in self.dismiss_list: if id_url.is_not_in(supplier_idurl, current_suppliers, as_field=False): lg.warn('%s not a supplier' % supplier_idurl) continue pos = current_suppliers.index(id_url.field(supplier_idurl)) current_suppliers[pos] = '' removed_suppliers.append(( pos, supplier_idurl, )) misc.writeSupplierData( supplier_idurl, 'disconnected', time.strftime('%d-%m-%Y %H:%M:%S'), my_id.getLocalID(), ) current_suppliers = current_suppliers[:desired_suppliers] contactsdb.update_suppliers(current_suppliers) contactsdb.save_suppliers() from main import control control.on_suppliers_changed(current_suppliers) for position, supplier_idurl in removed_suppliers: events.send( 'supplier-modified', dict( new_idurl=None, old_idurl=supplier_idurl, position=position, )) lg.info( 'removed some suppliers : %d desired_suppliers=%d current_suppliers=%d' % (len(self.dismiss_list), desired_suppliers, len(contactsdb.suppliers()))) if _Debug: lg.out(_DebugLevel, ' my current suppliers: %r' % contactsdb.suppliers())
def write_customer_suppliers( customer_idurl, suppliers_list, ecc_map=None, revision=None, publisher_idurl=None, ): if customer_idurl == my_id.getLocalIDURL(): lg.warn( 'skip writing my own suppliers list which suppose to be written to DHT' ) else: contactsdb.set_suppliers(suppliers_list, customer_idurl=customer_idurl) contactsdb.save_suppliers(customer_idurl=customer_idurl) return dht_records.set_suppliers( customer_idurl=customer_idurl, suppliers_list=suppliers_list, ecc_map=ecc_map, revision=revision, publisher_idurl=publisher_idurl, )
def _do_verify(dht_value): try: _ecc_map = dht_value['ecc_map'] _customer_idurl = strng.to_bin(dht_value['customer_idurl']) _publisher_idurl = dht_value.get('publisher_idurl') _suppliers_list = list(map(strng.to_bin, dht_value['suppliers'])) _revision = dht_value.get('revision') _timestamp = dht_value.get('timestamp') except: lg.exc() result.callback(None) return None ret = { 'suppliers': _suppliers_list, 'ecc_map': _ecc_map, 'customer_idurl': _customer_idurl, 'revision': _revision, 'publisher_idurl': _publisher_idurl, 'timestamp': _timestamp, } if customer_idurl == my_id.getLocalIDURL(): if _Debug: lg.out( _DebugLevel, 'dht_relations.read_customer_suppliers skip caching my own suppliers list received from DHT: %s' % ret) else: contactsdb.set_suppliers(_suppliers_list, customer_idurl=customer_idurl) contactsdb.save_suppliers(customer_idurl=customer_idurl) if _Debug: lg.out( _DebugLevel, 'dht_relations.read_customer_suppliers %r returned %r' % ( customer_idurl, ret, )) result.callback(ret) return None
def _on_my_dht_relations_discovered(self, dht_result): if not (dht_result and isinstance(dht_result, dict) and len(dht_result.get('suppliers', [])) > 0): lg.warn('no dht records found for my customer family') self.automat('suppliers-read-failed') return dht_suppliers = id_url.to_bin_list(dht_result['suppliers']) dht_ecc_map = dht_result.get('ecc_map', settings.DefaultEccMapName()) try: dht_desired_suppliers_number = eccmap.GetEccMapSuppliersNumber( dht_ecc_map) except: lg.exc() dht_desired_suppliers_number = eccmap.GetEccMapSuppliersNumber( settings.DefaultEccMapName()) settings.config.conf().setInt('services/customer/suppliers-number', dht_desired_suppliers_number) contactsdb.set_suppliers(dht_suppliers) contactsdb.save_suppliers() lg.info('found and restored list of %d suppliers from DHT' % dht_desired_suppliers_number) self.automat('suppliers-read-ok')
def doSubstituteSupplier(self, *args, **kwargs): """ Action method. """ new_idurl = id_url.field(args[0]) family_position = kwargs.get('family_position') current_suppliers = list(contactsdb.suppliers()) desired_suppliers = settings.getSuppliersNumberDesired() old_idurl = None if family_position in self.hire_list: self.hire_list.remove(family_position) lg.info( 'found position on which new supplier suppose to be hired: %d' % family_position) else: lg.warn('did not found position for new supplier to be hired on') if new_idurl in current_suppliers: raise Exception('%s is already supplier' % new_idurl) if family_position is None or family_position == -1: lg.warn( 'unknown family_position from supplier results, will pick first empty spot' ) position = -1 old_idurl = None for i in range(len(current_suppliers)): if not current_suppliers[i].strip(): position = i break if id_url.is_in(current_suppliers[i], self.dismiss_list, as_field=False): position = i old_idurl = current_suppliers[i] break family_position = position if _Debug: lg.out( _DebugLevel, 'fire_hire.doSubstituteSupplier family_position=%d' % family_position) contactsdb.add_supplier(idurl=new_idurl, position=family_position) contactsdb.save_suppliers() misc.writeSupplierData( new_idurl, 'connected', time.strftime('%d-%m-%Y %H:%M:%S'), my_id.getIDURL(), ) from main import control control.on_suppliers_changed(current_suppliers) if family_position < 0: lg.info( 'added new supplier, family position unknown: %s desired_suppliers=%d current_suppliers=%d' % (new_idurl, desired_suppliers, len(contactsdb.suppliers()))) events.send('supplier-modified', data=dict( new_idurl=new_idurl, old_idurl=None, position=family_position, ecc_map=eccmap.Current().name, family_snapshot=id_url.to_bin_list( contactsdb.suppliers()), )) else: if old_idurl: lg.info( 'hired new supplier and substitute existing supplier on position %d : %s->%s desired_suppliers=%d current_suppliers=%d' % (family_position, old_idurl, new_idurl, desired_suppliers, len(contactsdb.suppliers()))) events.send('supplier-modified', data=dict( new_idurl=new_idurl, old_idurl=old_idurl, position=family_position, ecc_map=eccmap.Current().name, family_snapshot=id_url.to_bin_list( contactsdb.suppliers()), )) else: lg.info( 'hired new supplier on empty position %d : %s desired_suppliers=%d current_suppliers=%d' % (family_position, new_idurl, desired_suppliers, len(contactsdb.suppliers()))) events.send('supplier-modified', data=dict( new_idurl=new_idurl, old_idurl=None, position=family_position, ecc_map=eccmap.Current().name, family_snapshot=id_url.to_bin_list( contactsdb.suppliers()), )) self.restart_interval = 1.0 if _Debug: lg.out(_DebugLevel, ' my current suppliers: %r' % contactsdb.suppliers())
def _on_files_received(self, newpacket, info): from logs import lg from lib import serialization from main import settings from main import events from p2p import p2p_service from storage import backup_fs from storage import backup_control from crypt import encrypted from crypt import my_keys from userid import my_id from userid import global_id from storage import backup_matrix from supplier import list_files from contacts import contactsdb list_files_global_id = global_id.ParseGlobalID(newpacket.PacketID) if not list_files_global_id['idurl']: lg.warn('invalid PacketID: %s' % newpacket.PacketID) return False trusted_customer_idurl = list_files_global_id['idurl'] incoming_key_id = list_files_global_id['key_id'] if trusted_customer_idurl == my_id.getGlobalID(): lg.warn('skip %s packet which seems to came from my own supplier' % newpacket) # only process list Files() from other users who granted me access return False if not my_keys.is_valid_key_id(incoming_key_id): lg.warn('ignore, invalid key id in packet %s' % newpacket) return False if not my_keys.is_key_private(incoming_key_id): lg.warn('private key is not registered : %s' % incoming_key_id) p2p_service.SendFail(newpacket, 'private key is not registered') return False try: block = encrypted.Unserialize( newpacket.Payload, decrypt_key=incoming_key_id, ) except: lg.exc(newpacket.Payload) return False if block is None: lg.warn('failed reading data from %s' % newpacket.RemoteID) return False # if block.CreatorID != trusted_customer_idurl: # lg.warn('invalid packet, creator ID must be present in packet ID : %s ~ %s' % ( # block.CreatorID, list_files_global_id['idurl'], )) # return False try: raw_files = block.Data() except: lg.exc() return False if block.CreatorID == trusted_customer_idurl: # this is a trusted guy sending some shared files to me try: json_data = serialization.BytesToDict(raw_files, keys_to_text=True) json_data['items'] except: lg.exc() return False count = backup_fs.Unserialize( raw_data=json_data, iter=backup_fs.fs(trusted_customer_idurl), iterID=backup_fs.fsID(trusted_customer_idurl), from_json=True, ) p2p_service.SendAck(newpacket) events.send( 'shared-list-files-received', dict( customer_idurl=trusted_customer_idurl, new_items=count, )) if count == 0: lg.warn('no files were imported during file sharing') else: backup_control.Save() lg.info('imported %d shared files from %s, key_id=%s' % ( count, trusted_customer_idurl, incoming_key_id, )) return True # otherwise this must be an external supplier sending us a files he stores for trusted customer external_supplier_idurl = block.CreatorID try: supplier_raw_list_files = list_files.UnpackListFiles( raw_files, settings.ListFilesFormat()) backup_matrix.SaveLatestRawListFiles( supplier_idurl=external_supplier_idurl, raw_data=supplier_raw_list_files, customer_idurl=trusted_customer_idurl, ) except: lg.exc() return False # need to detect supplier position from the list of packets # and place that supplier on the correct position in contactsdb real_supplier_pos = backup_matrix.DetectSupplierPosition( supplier_raw_list_files) known_supplier_pos = contactsdb.supplier_position( external_supplier_idurl, trusted_customer_idurl) if real_supplier_pos >= 0: if known_supplier_pos >= 0 and known_supplier_pos != real_supplier_pos: lg.warn( 'external supplier %s position is not matching to list files, rewriting for customer %s' % (external_supplier_idurl, trusted_customer_idurl)) contactsdb.erase_supplier( idurl=external_supplier_idurl, customer_idurl=trusted_customer_idurl, ) contactsdb.add_supplier( idurl=external_supplier_idurl, position=real_supplier_pos, customer_idurl=trusted_customer_idurl, ) contactsdb.save_suppliers(customer_idurl=trusted_customer_idurl) else: lg.warn( 'not possible to detect external supplier position for customer %s' % trusted_customer_idurl) # finally send ack packet back p2p_service.SendAck(newpacket) lg.info( 'received list of packets from external supplier %s for customer %s' % (external_supplier_idurl, trusted_customer_idurl)) return True
def doSubstituteSupplier(self, *args, **kwargs): """ Action method. """ new_idurl = strng.to_bin(args[0]) family_position = kwargs.get('family_position') current_suppliers = list(contactsdb.suppliers()) old_idurl = None if family_position in self.hire_list: self.hire_list.remove(family_position) lg.info('found position on which new supplier suppose to be hired: %d' % family_position) else: lg.warn('did not found position for new supplier to be hired on') if new_idurl in current_suppliers: raise Exception('%s is already supplier' % new_idurl) if not family_position: lg.warn('unknown family_position from supplier results, will pick first empty spot') position = -1 old_idurl = None for i in range(len(current_suppliers)): if not current_suppliers[i].strip(): position = i break if current_suppliers[i] in self.dismiss_list: # self.dismiss_list.remove(current_suppliers[i]) position = i old_idurl = current_suppliers[i] break family_position = position lg.out(10, 'fire_hire.doSubstituteSupplier family_position=%d' % family_position) contactsdb.add_supplier(idurl=new_idurl, position=family_position) contactsdb.save_suppliers() misc.writeSupplierData( new_idurl, 'connected', time.strftime('%d-%m-%Y %H:%M:%S'), my_id.getLocalID(), ) from main import control control.on_suppliers_changed(current_suppliers) if family_position < 0: lg.out(2, '!!!!!!!!!!! ADDED NEW SUPPLIER : %s' % new_idurl) events.send('supplier-modified', dict( new_idurl=new_idurl, old_idurl=None, position=family_position, ecc_map=eccmap.Current().name, family_snapshot=contactsdb.suppliers(), )) else: if old_idurl: lg.out(2, '!!!!!!!!!!! SUBSTITUTE EXISTING SUPPLIER %d : %s->%s' % (family_position, old_idurl, new_idurl)) events.send('supplier-modified', dict( new_idurl=new_idurl, old_idurl=old_idurl, position=family_position, ecc_map=eccmap.Current().name, family_snapshot=contactsdb.suppliers(), )) else: lg.out(2, '!!!!!!!!!!! REPLACE EMPTY SUPPLIER %d : %s' % (family_position, new_idurl)) events.send('supplier-modified', dict( new_idurl=new_idurl, old_idurl=None, position=family_position, ecc_map=eccmap.Current().name, family_snapshot=contactsdb.suppliers(), )) self.restart_interval = 1.0