def doDisconnectSuppliers(self, *args, **kwargs): """ Action method. """ from customer import supplier_connector from p2p import online_status if _Debug: lg.out(_DebugLevel, 'fire_hire.doDisconnectSuppliers %r' % self.dismiss_list) self.dismiss_results = [] for supplier_idurl in self.dismiss_list: sc = supplier_connector.by_idurl(supplier_idurl) if sc: sc.set_callback('fire_hire', self._on_supplier_connector_state_changed) sc.automat( 'disconnect', ecc_map=eccmap.Current().name, ) else: lg.warn('supplier_connector must exist, but not found %s' % supplier_idurl) online_status.remove_online_status_listener_callback( idurl=supplier_idurl, callback_method=self._on_supplier_online_status_state_changed, )
def doInit(self, *args, **kwargs): """ Action method. """ self._do_block_rebuilding() self.known_suppliers = [ _f for _f in contactsdb.suppliers(customer_idurl=self.customer_idurl) if _f ] if not self.EccMap: if self.customer_idurl == my_id.getIDURL(): self.EccMap = eccmap.Current() lg.info('ECC map %r set from local for my own suppliers' % self.EccMap) if not self.EccMap: known_eccmap_dict = {} for supplier_idurl in self.known_suppliers: known_ecc_map = contactsdb.get_supplier_meta_info( supplier_idurl=supplier_idurl, customer_idurl=self.customer_idurl, ).get('ecc_map', None) if known_ecc_map: if known_ecc_map not in known_eccmap_dict: known_eccmap_dict[known_ecc_map] = 0 known_eccmap_dict[known_ecc_map] += 1 if known_eccmap_dict: all_known_eccmaps = list(known_eccmap_dict.items()) all_known_eccmaps.sort(key=lambda i: i[1], reverse=True) self.EccMap = eccmap.eccmap(all_known_eccmaps[0][0]) lg.info('ECC map %r recognized from suppliers meta info' % self.EccMap) else: known_ecc_map = None if driver.is_on('service_shared_data'): from access import shared_access_coordinator active_share = shared_access_coordinator.get_active_share( self.key_id) if active_share: known_ecc_map = active_share.known_ecc_map if known_ecc_map: self.EccMap = eccmap.eccmap(known_ecc_map) lg.info('ECC map %r recognized from active share %r' % ( self.EccMap, active_share, )) else: num_suppliers = len(self.known_suppliers) if num_suppliers not in eccmap.GetPossibleSuppliersCount(): num_suppliers = settings.DefaultDesiredSuppliers() self.EccMap = eccmap.eccmap( eccmap.GetEccMapName(num_suppliers)) lg.warn( 'no meta info found, guessed ECC map %r from %d known suppliers' % (self.EccMap, len(self.known_suppliers))) self.max_errors = eccmap.GetCorrectableErrors( self.EccMap.NumSuppliers()) if data_receiver.A(): data_receiver.A().addStateChangedCallback( self._on_data_receiver_state_changed)
def doSupplierConnect(self, *args, **kwargs): """ Action method. """ from customer import supplier_connector from customer import fire_hire from raid import eccmap position = self.family_position if not position: lg.warn('position for new supplier is unknown, will "guess"') current_suppliers = list(contactsdb.suppliers()) for i in range(len(current_suppliers)): if not current_suppliers[i].strip(): position = i break if current_suppliers[i] in fire_hire.A().dismiss_list: position = i break sc = supplier_connector.by_idurl(self.target_idurl) if not sc: sc = supplier_connector.create( supplier_idurl=self.target_idurl, customer_idurl=my_id.getLocalID(), ) sc.automat( 'connect', family_position=position, ecc_map=(self.ecc_map or eccmap.Current().name), family_snapshot=self.family_snapshot, ) sc.set_callback('supplier_finder', self._supplier_connector_state)
def doRequestService(self, arg): """ Action method. """ service_info = { 'needed_bytes': self.needed_bytes, 'customer_id': global_id.UrlToGlobalID(self.customer_idurl), } my_customer_key_id = my_id.getGlobalID(key_alias='customer') if my_keys.is_key_registered(my_customer_key_id): service_info['customer_public_key'] = my_keys.get_key_info( key_id=my_customer_key_id, include_private=False, ) if self.key_id: service_info['key_id'] = self.key_id if self.customer_idurl == my_id.getLocalIDURL(): service_info['ecc_map'] = eccmap.Current().name request = p2p_service.SendRequestService( remote_idurl=self.supplier_idurl, service_name='service_supplier', json_payload=service_info, callbacks={ commands.Ack(): self._supplier_acked, commands.Fail(): self._supplier_failed, }, ) self.request_packet_id = request.PacketID
def doConnectSuppliers(self, *args, **kwargs): """ Action method. """ self.connect_list = [] my_current_family = list(contactsdb.suppliers()) for pos, supplier_idurl in enumerate(my_current_family): if not supplier_idurl: continue sc = supplier_connector.by_idurl(supplier_idurl) if sc is None: sc = supplier_connector.create( supplier_idurl=supplier_idurl, customer_idurl=my_id.getLocalID(), ) sc.set_callback('fire_hire', self._on_supplier_connector_state_changed) self.connect_list.append(supplier_idurl) sc.automat( 'connect', family_position=pos, ecc_map=eccmap.Current().name, family_snapshot=my_current_family, ) supplier_contact_status = contact_status.getInstance( supplier_idurl) if supplier_contact_status: supplier_contact_status.addStateChangedCallback( self._on_supplier_contact_status_state_changed, newstate='OFFLINE', )
def health_check(self): from raid import eccmap from contacts import contactsdb from userid import id_url missed_suppliers = id_url.empty_count(contactsdb.suppliers()) # to have that service running minimum amount of suppliers must be already in the family return missed_suppliers <= eccmap.Current().CorrectableErrors
def doSupplierConnect(self, *args, **kwargs): """ Action method. """ from customer import supplier_connector from customer import fire_hire from raid import eccmap position = self.family_position if position is None or position == -1: lg.warn('position for new supplier is unknown, will "guess"') current_suppliers = list(contactsdb.suppliers()) for i in range(len(current_suppliers)): supplier_idurl = current_suppliers[i].to_bin() if not supplier_idurl: position = i break if id_url.is_in(supplier_idurl, fire_hire.A().dismiss_list, as_field=False): position = i break sc = supplier_connector.by_idurl(self.target_idurl) if not sc: sc = supplier_connector.create( supplier_idurl=self.target_idurl, customer_idurl=my_id.getIDURL(), ) sc.set_callback('supplier_finder', self._supplier_connector_state) sc.automat( 'connect', family_position=position, ecc_map=(self.ecc_map or eccmap.Current().name), family_snapshot=self.family_snapshot, )
def doFindNewSupplier(self, *args, **kwargs): """ Action method. """ if _Debug: lg.out(_DebugLevel, 'fire_hire.doFindNewSupplier') position_for_new_supplier = None for pos in range(settings.getSuppliersNumberDesired()): if pos in self.hire_list: continue supplier_idurl = contactsdb.supplier(pos) if not supplier_idurl: lg.info('found empty supplier at position %d and going to find new supplier on that position' % pos) position_for_new_supplier = pos break if supplier_idurl in self.dismiss_list: lg.info('going to find new supplier on existing position %d to replace supplier %s' % ( pos, supplier_idurl, )) position_for_new_supplier = pos break if position_for_new_supplier is None: lg.err('did not found position for new supplier') self.automat('search-failed') return self.hire_list.append(position_for_new_supplier) supplier_finder.A( 'start', family_position=position_for_new_supplier, ecc_map=eccmap.Current().name, family_snapshot=contactsdb.suppliers(), )
def _start_one_block(self): from storage import backup_matrix if self.blockIndex < 0: lg.out( 10, 'backup_rebuilder._start_one_block finish all blocks blockIndex=%d' % self.blockIndex) reactor.callLater(0, self._finish_rebuilding) return BlockNumber = self.workingBlocksQueue[self.blockIndex] lg.out( 10, 'backup_rebuilder._start_one_block %d to rebuild, blockIndex=%d, other blocks: %s' % ((BlockNumber, self.blockIndex, str(self.workingBlocksQueue)))) task_params = ( self.currentBackupID, BlockNumber, eccmap.Current(), backup_matrix.GetActiveArray(), backup_matrix.GetRemoteMatrix(self.currentBackupID, BlockNumber), backup_matrix.GetLocalMatrix(self.currentBackupID, BlockNumber), ) raid_worker.add_task( 'rebuild', task_params, lambda cmd, params, result: self._block_finished(result, params))
def doConnectSuppliers(self, *args, **kwargs): """ Action method. """ from customer import supplier_connector from p2p import online_status self.connect_list = [] my_current_family = contactsdb.suppliers() for pos, supplier_idurl in enumerate(my_current_family): if not supplier_idurl: continue if self.configs[0] and pos >= self.configs[0]: continue sc = supplier_connector.by_idurl(supplier_idurl) if sc is None: sc = supplier_connector.create( supplier_idurl=supplier_idurl, customer_idurl=my_id.getIDURL(), ) else: sc.needed_bytes = None sc.do_calculate_needed_bytes() sc.set_callback('fire_hire', self._on_supplier_connector_state_changed) self.connect_list.append(supplier_idurl) sc.automat( 'connect', family_position=pos, ecc_map=eccmap.Current().name, family_snapshot=id_url.to_bin_list(my_current_family), ) online_status.add_online_status_listener_callback( idurl=supplier_idurl, callback_method=self._on_supplier_online_status_state_changed, )
def __init__(self, BackupID, OutputFile, KeyID=None): # OutputFileName self.CreatorID = my_id.getLocalID() self.BackupID = BackupID _parts = packetid.SplitBackupID(self.BackupID) self.CustomerGlobalID = _parts[0] self.CustomerIDURL = global_id.GlobalUserToIDURL(self.CustomerGlobalID) self.PathID = _parts[1] self.Version = _parts[2] self.File = OutputFile self.KeyID = KeyID # is current active block - so when add 1 we get to first, which is 0 self.BlockNumber = -1 self.BytesWritten = 0 self.OnHandData = [] self.OnHandParity = [] self.AbortState = False self.Done = False self.EccMap = eccmap.Current() self.Started = time.time() self.LastAction = time.time() self.InboxPacketsQueue = [] self.InboxQueueWorker = None self.RequestFails = [] self.InboxQueueDelay = 1 # For anyone who wants to know when we finish self.MyDeferred = Deferred() self.packetInCallback = None self.blockRestoredCallback = None automat.Automat.__init__(self, 'restore_%s' % self.BackupID, 'AT_STARTUP', _DebugLevel, _Debug) events.send('restore-started', dict(backup_id=self.BackupID))
def doPrepare(self, *args, **kwargs): """ Action method. """ self.restored_count = 0 self.saved_count = 0 self.deleted_count = 0 self.stored_keys = {} self.not_stored_keys = {} self.unreliable_keys = {} self.keys_to_upload = set() self.keys_to_erase = {} self.keys_to_rename = {} lookup = backup_fs.ListChildsByPath( path='.keys', recursive=False, ) if isinstance(lookup, list): minimum_reliable_percent = eccmap.GetCorrectablePercent( eccmap.Current().suppliers_number) for i in lookup: if i['path'].endswith('.public'): stored_key_id = i['path'].replace('.public', '').replace( '.keys/', '') is_private = False else: stored_key_id = i['path'].replace('.private', '').replace( '.keys/', '') is_private = True stored_key_id = my_keys.latest_key_id(stored_key_id) is_reliable = False for v in i['versions']: try: reliable = float(v['reliable'].replace('%', '')) except: lg.exc() reliable = 0.0 if reliable >= minimum_reliable_percent: is_reliable = True break if is_reliable: self.stored_keys[stored_key_id] = is_private else: if is_private and my_keys.is_key_private(stored_key_id): self.not_stored_keys[stored_key_id] = is_private elif not is_private and my_keys.is_key_registered( stored_key_id): self.not_stored_keys[stored_key_id] = is_private else: self.unreliable_keys[stored_key_id] = is_private if _Debug: lg.args(_DebugLevel, stored_keys=len(self.stored_keys), not_stored_keys=list(self.not_stored_keys.keys()), unreliable_keys=len(self.unreliable_keys))
def doFindNewSupplier(self, *args, **kwargs): """ Action method. """ if _Debug: lg.out( _DebugLevel, 'fire_hire.doFindNewSupplier desired_suppliers=%d current_suppliers=%r' % (settings.getSuppliersNumberDesired(), contactsdb.suppliers())) from p2p import network_connector if network_connector.A().state != 'CONNECTED': if _Debug: lg.out( _DebugLevel, ' network_connector is not CONNECTED at the moment, SKIP' ) self.automat('search-failed') return position_for_new_supplier = None for pos in range(settings.getSuppliersNumberDesired()): if pos in self.hire_list: continue supplier_idurl = contactsdb.supplier(pos) if not supplier_idurl: lg.info( 'found empty supplier at position %d and going to find new supplier on that position' % pos) position_for_new_supplier = pos break if id_url.is_in(supplier_idurl, self.dismiss_list, as_field=False): lg.info( 'going to find new supplier on existing position %d to replace supplier %s' % ( pos, supplier_idurl, )) position_for_new_supplier = pos break if position_for_new_supplier is None: lg.err('did not found position for new supplier') self.automat('search-failed') return from customer import supplier_finder for idurl_txt in strng.to_text(config.conf().getData( 'services/employer/candidates')).split(','): if idurl_txt.strip(): supplier_finder.AddSupplierToHire(idurl_txt) self.hire_list.append(position_for_new_supplier) supplier_finder.A( 'start', family_position=position_for_new_supplier, ecc_map=eccmap.Current().name, family_snapshot=id_url.to_bin_list(contactsdb.suppliers()), )
def isEnoughListFilesReceived(self, *args, **kwargs): """ Condition method. """ global _ReceivedListFilesCounter lg.out( 6, 'list_files_orator.isSomeListFilesReceived %d list files was received' % _ReceivedListFilesCounter) from raid import eccmap critical_suppliers_number = eccmap.GetCorrectableErrors( eccmap.Current().suppliers_number) return _ReceivedListFilesCounter >= critical_suppliers_number
def __init__( self, backupID, pipe, finishCallback=None, blockResultCallback=None, notifyNewDataCallback=None, blockSize=None, sourcePath=None, keyID=None, ecc_map=None, creatorIDURL=None, ): self.backupID = backupID self.creatorIDURL = creatorIDURL or my_id.getIDURL() _parts = packetid.SplitBackupID(self.backupID) self.customerGlobalID = _parts[0] self.pathID = _parts[1] self.version = _parts[2] self.customerIDURL = global_id.GlobalUserToIDURL(self.customerGlobalID) self.sourcePath = sourcePath self.keyID = keyID self.eccmap = ecc_map or eccmap.Current() self.pipe = pipe self.blockSize = blockSize if self.blockSize is None: self.blockSize = settings.getBackupBlockSize() self.ask4abort = False self.terminating = False self.stateEOF = False self.stateReading = False self.closed = False self.currentBlockData = BytesIO() self.currentBlockSize = 0 self.workBlocks = {} self.blockNumber = 0 self.dataSent = 0 self.blocksSent = 0 self.totalSize = -1 self.resultDefer = Deferred() self.finishCallback = finishCallback self.blockResultCallback = blockResultCallback self.notifyNewDataCallback = notifyNewDataCallback automat.Automat.__init__( self, name='backup_%s' % self.version, state='AT_STARTUP', debug_level=_DebugLevel, log_events=_Debug, log_transitions=_Debug, )
def _do_notify_supplier_position(self, supplier_idurl, supplier_position): from p2p import p2p_service from raid import eccmap from userid import my_id p2p_service.SendContacts( remote_idurl=supplier_idurl, json_payload={ 'space': 'family_member', 'type': 'supplier_position', 'customer_idurl': my_id.getLocalIDURL(), 'customer_ecc_map': eccmap.Current().name, 'supplier_idurl': supplier_idurl, 'supplier_position': supplier_position, }, )
def doSendRejectService(self, *args, **kwargs): """ Action method. """ space_dict, free_space, spent_bytes, current_customers, removed_customers = args[ 0] for customer_idurl in removed_customers: p2p_service.SendFailNoRequest(customer_idurl, packetid.UniqueID(), 'service rejected') events.send( 'existing-customer-terminated', dict( idurl=customer_idurl, ecc_map=eccmap.Current().name, )) self.automat('packets-sent')
def isChanceToRebuild(self, arg): """ Condition method. """ from storage import backup_matrix # supplierSet = backup_matrix.suppliers_set() # start checking in reverse order, see below for explanation for blockIndex in range(len(self.workingBlocksQueue) - 1, -1, -1): blockNumber = self.workingBlocksQueue[blockIndex] if eccmap.Current().CanMakeProgress( backup_matrix.GetLocalDataArray(self.currentBackupID, blockNumber), backup_matrix.GetLocalParityArray(self.currentBackupID, blockNumber)): return True return False
def __init__(self, BackupID, OutputFile, KeyID=None, debug_level=_DebugLevel, log_events=_Debug, log_transitions=_Debug, publish_events=False, **kwargs): """ Builds `restore_worker()` state machine. """ self.CreatorID = my_id.getLocalID() self.BackupID = BackupID _parts = packetid.SplitBackupID(self.BackupID) self.CustomerGlobalID = _parts[0] self.CustomerIDURL = global_id.GlobalUserToIDURL(self.CustomerGlobalID) self.PathID = _parts[1] self.Version = _parts[2] self.File = OutputFile self.KeyID = KeyID # is current active block - so when add 1 we get to first, which is 0 self.BlockNumber = -1 self.BytesWritten = 0 self.OnHandData = [] self.OnHandParity = [] self.AbortState = False self.Done = False self.EccMap = eccmap.Current() self.Started = time.time() self.LastAction = time.time() self.RequestFails = [] # For anyone who wants to know when we finish self.MyDeferred = Deferred() self.packetInCallback = None self.blockRestoredCallback = None super(RestoreWorker, self).__init__(name='restore_worker_%s' % self.Version, state="AT_STARTUP", debug_level=debug_level, log_events=log_events, log_transitions=log_transitions, publish_events=publish_events, **kwargs) events.send('restore-started', dict(backup_id=self.BackupID))
def __init__( self, backupID, pipe, finishCallback=None, blockResultCallback=None, blockSize=None, sourcePath=None, keyID=None, ): self.backupID = backupID _parts = packetid.SplitBackupID(self.backupID) self.customerGlobalID = _parts[0] self.pathID = _parts[1] self.version = _parts[2] self.customerIDURL = global_id.GlobalUserToIDURL(self.customerGlobalID) self.sourcePath = sourcePath self.keyID = keyID self.eccmap = eccmap.Current() self.pipe = pipe self.blockSize = blockSize if self.blockSize is None: self.blockSize = settings.getBackupBlockSize() self.ask4abort = False self.terminating = False self.stateEOF = False self.stateReading = False self.closed = False self.currentBlockData = BytesIO() self.currentBlockSize = 0 self.workBlocks = {} self.blockNumber = 0 self.dataSent = 0 self.blocksSent = 0 self.totalSize = -1 self.finishCallback = finishCallback self.blockResultCallback = blockResultCallback automat.Automat.__init__(self, 'backup_%s' % self.version, 'AT_STARTUP', _DebugLevel)
def health_check(self): from contacts import contactsdb from raid import eccmap missed_suppliers = contactsdb.suppliers().count('') # critical amount of suppliers must be already in the family to have that service running return missed_suppliers <= eccmap.Current().CorrectableErrors
def doSubstituteSupplier(self, *args, **kwargs): """ Action method. """ new_idurl = id_url.field(args[0]) family_position = kwargs.get('family_position') current_suppliers = list(contactsdb.suppliers()) desired_suppliers = settings.getSuppliersNumberDesired() old_idurl = None if family_position in self.hire_list: self.hire_list.remove(family_position) lg.info( 'found position on which new supplier suppose to be hired: %d' % family_position) else: lg.warn('did not found position for new supplier to be hired on') if new_idurl in current_suppliers: raise Exception('%s is already supplier' % new_idurl) if family_position is None or family_position == -1: lg.warn( 'unknown family_position from supplier results, will pick first empty spot' ) position = -1 old_idurl = None for i in range(len(current_suppliers)): if not current_suppliers[i].strip(): position = i break if id_url.is_in(current_suppliers[i], self.dismiss_list, as_field=False): position = i old_idurl = current_suppliers[i] break family_position = position if _Debug: lg.out( _DebugLevel, 'fire_hire.doSubstituteSupplier family_position=%d' % family_position) contactsdb.add_supplier(idurl=new_idurl, position=family_position) contactsdb.save_suppliers() misc.writeSupplierData( new_idurl, 'connected', time.strftime('%d-%m-%Y %H:%M:%S'), my_id.getIDURL(), ) from main import control control.on_suppliers_changed(current_suppliers) if family_position < 0: lg.info( 'added new supplier, family position unknown: %s desired_suppliers=%d current_suppliers=%d' % (new_idurl, desired_suppliers, len(contactsdb.suppliers()))) events.send('supplier-modified', data=dict( new_idurl=new_idurl, old_idurl=None, position=family_position, ecc_map=eccmap.Current().name, family_snapshot=id_url.to_bin_list( contactsdb.suppliers()), )) else: if old_idurl: lg.info( 'hired new supplier and substitute existing supplier on position %d : %s->%s desired_suppliers=%d current_suppliers=%d' % (family_position, old_idurl, new_idurl, desired_suppliers, len(contactsdb.suppliers()))) events.send('supplier-modified', data=dict( new_idurl=new_idurl, old_idurl=old_idurl, position=family_position, ecc_map=eccmap.Current().name, family_snapshot=id_url.to_bin_list( contactsdb.suppliers()), )) else: lg.info( 'hired new supplier on empty position %d : %s desired_suppliers=%d current_suppliers=%d' % (family_position, new_idurl, desired_suppliers, len(contactsdb.suppliers()))) events.send('supplier-modified', data=dict( new_idurl=new_idurl, old_idurl=None, position=family_position, ecc_map=eccmap.Current().name, family_snapshot=id_url.to_bin_list( contactsdb.suppliers()), )) self.restart_interval = 1.0 if _Debug: lg.out(_DebugLevel, ' my current suppliers: %r' % contactsdb.suppliers())
def doSubstituteSupplier(self, *args, **kwargs): """ Action method. """ new_idurl = strng.to_bin(args[0]) family_position = kwargs.get('family_position') current_suppliers = list(contactsdb.suppliers()) old_idurl = None if family_position in self.hire_list: self.hire_list.remove(family_position) lg.info('found position on which new supplier suppose to be hired: %d' % family_position) else: lg.warn('did not found position for new supplier to be hired on') if new_idurl in current_suppliers: raise Exception('%s is already supplier' % new_idurl) if not family_position: lg.warn('unknown family_position from supplier results, will pick first empty spot') position = -1 old_idurl = None for i in range(len(current_suppliers)): if not current_suppliers[i].strip(): position = i break if current_suppliers[i] in self.dismiss_list: # self.dismiss_list.remove(current_suppliers[i]) position = i old_idurl = current_suppliers[i] break family_position = position lg.out(10, 'fire_hire.doSubstituteSupplier family_position=%d' % family_position) contactsdb.add_supplier(idurl=new_idurl, position=family_position) contactsdb.save_suppliers() misc.writeSupplierData( new_idurl, 'connected', time.strftime('%d-%m-%Y %H:%M:%S'), my_id.getLocalID(), ) from main import control control.on_suppliers_changed(current_suppliers) if family_position < 0: lg.out(2, '!!!!!!!!!!! ADDED NEW SUPPLIER : %s' % new_idurl) events.send('supplier-modified', dict( new_idurl=new_idurl, old_idurl=None, position=family_position, ecc_map=eccmap.Current().name, family_snapshot=contactsdb.suppliers(), )) else: if old_idurl: lg.out(2, '!!!!!!!!!!! SUBSTITUTE EXISTING SUPPLIER %d : %s->%s' % (family_position, old_idurl, new_idurl)) events.send('supplier-modified', dict( new_idurl=new_idurl, old_idurl=old_idurl, position=family_position, ecc_map=eccmap.Current().name, family_snapshot=contactsdb.suppliers(), )) else: lg.out(2, '!!!!!!!!!!! REPLACE EMPTY SUPPLIER %d : %s' % (family_position, new_idurl)) events.send('supplier-modified', dict( new_idurl=new_idurl, old_idurl=None, position=family_position, ecc_map=eccmap.Current().name, family_snapshot=contactsdb.suppliers(), )) self.restart_interval = 1.0