def doSaveConfig(self, arg): """ Action method. """ self.configs = (settings.getSuppliersNumberDesired(), diskspace.GetBytesFromString( settings.getNeededString()))
def isConfigChanged(self, *args, **kwargs): """ Condition method. """ curconfigs = (settings.getSuppliersNumberDesired(), diskspace.GetBytesFromString(settings.getNeededString())) if None in self.configs: return True return self.configs[0] != curconfigs[0] or self.configs[1] != curconfigs[1]
def __init__(self, supplier_idurl, customer_idurl, needed_bytes, key_id=None, queue_subscribe=True): """ """ self.supplier_idurl = supplier_idurl self.customer_idurl = customer_idurl self.needed_bytes = needed_bytes self.key_id = key_id self.queue_subscribe = queue_subscribe if self.needed_bytes is None: total_bytes_needed = diskspace.GetBytesFromString( settings.getNeededString(), 0) num_suppliers = -1 if self.customer_idurl == my_id.getLocalIDURL(): num_suppliers = settings.getSuppliersNumberDesired() else: known_ecc_map = contactsdb.get_customer_meta_info( customer_idurl).get('ecc_map') if known_ecc_map: num_suppliers = eccmap.GetEccMapSuppliersNumber( known_ecc_map) if num_suppliers > 0: self.needed_bytes = int( math.ceil(2.0 * total_bytes_needed / float(num_suppliers))) else: raise Exception( 'not possible to determine needed_bytes value to be requested from that supplier' ) # self.needed_bytes = int(math.ceil(2.0 * settings.MinimumNeededBytes() / float(settings.DefaultDesiredSuppliers()))) name = 'supplier_%s_%s' % ( nameurl.GetName(self.supplier_idurl), diskspace.MakeStringFromBytes(self.needed_bytes).replace(' ', ''), ) self.request_packet_id = None self.callbacks = {} try: st = bpio.ReadTextFile( settings.SupplierServiceFilename( idurl=self.supplier_idurl, customer_idurl=self.customer_idurl, )).strip() except: st = 'DISCONNECTED' automat.Automat.__init__( self, name, state=st, debug_level=_DebugLevel, log_events=_Debug, log_transitions=_Debug, ) for cb in self.callbacks.values(): cb(self.supplier_idurl, self.state, self.state)
def do_calculate_needed_bytes(self): if self.needed_bytes is None: total_bytes_needed = diskspace.GetBytesFromString(settings.getNeededString(), 0) num_suppliers = -1 if self.customer_idurl == my_id.getLocalIDURL(): num_suppliers = settings.getSuppliersNumberDesired() else: known_ecc_map = contactsdb.get_customer_meta_info(self.customer_idurl).get('ecc_map') if known_ecc_map: num_suppliers = eccmap.GetEccMapSuppliersNumber(known_ecc_map) if num_suppliers > 0: self.needed_bytes = int(math.ceil(2.0 * total_bytes_needed / float(num_suppliers))) else: raise Exception('not possible to determine needed_bytes value to be requested from that supplier')
def __init__(self, supplier_idurl, customer_idurl, needed_bytes, key_id=None, queue_subscribe=True): """ """ self.supplier_idurl = supplier_idurl self.customer_idurl = customer_idurl self.needed_bytes = needed_bytes self.key_id = key_id self.queue_subscribe = queue_subscribe if self.needed_bytes is None: total_bytes_needed = diskspace.GetBytesFromString( settings.getNeededString(), 0) num_suppliers = settings.getSuppliersNumberDesired() if num_suppliers > 0: self.needed_bytes = int( math.ceil(2.0 * total_bytes_needed / float(num_suppliers))) else: self.needed_bytes = int( math.ceil(2.0 * settings.MinimumNeededBytes() / float(settings.DefaultDesiredSuppliers()))) name = 'supplier_%s_%s' % ( nameurl.GetName(self.supplier_idurl), diskspace.MakeStringFromBytes(self.needed_bytes).replace(' ', ''), ) self.request_packet_id = None self.callbacks = {} try: st = bpio.ReadTextFile( settings.SupplierServiceFilename( idurl=self.supplier_idurl, customer_idurl=self.customer_idurl, )).strip() except: st = 'DISCONNECTED' automat.Automat.__init__( self, name, state=st, debug_level=_DebugLevel, log_events=_Debug, log_transitions=_Debug, ) for cb in self.callbacks.values(): cb(self.supplier_idurl, self.state, self.state)
def __init__(self, supplier_idurl, customer_idurl, needed_bytes): """ """ self.supplier_idurl = supplier_idurl self.customer_idurl = customer_idurl self.needed_bytes = needed_bytes if self.needed_bytes is None: total_bytes_needed = diskspace.GetBytesFromString( settings.getNeededString(), 0) num_suppliers = settings.getSuppliersNumberDesired() if num_suppliers > 0: self.needed_bytes = int( math.ceil(2.0 * total_bytes_needed / float(num_suppliers))) else: self.needed_bytes = int( math.ceil(2.0 * settings.MinimumNeededBytes() / float(settings.DefaultDesiredSuppliers()))) name = 'supplier_%s_%s' % ( nameurl.GetName(self.supplier_idurl), diskspace.MakeStringFromBytes(self.needed_bytes).replace(' ', ''), ) self.request_packet_id = None self.callbacks = {} try: st = bpio.ReadTextFile( settings.SupplierServiceFilename( idurl=self.supplier_idurl, customer_idurl=self.customer_idurl, )).strip() except: st = 'DISCONNECTED' if st == 'CONNECTED': automat.Automat.__init__(self, name, 'CONNECTED', _DebugLevel, _Debug) elif st == 'NO_SERVICE': automat.Automat.__init__(self, name, 'NO_SERVICE', _DebugLevel, _Debug) else: automat.Automat.__init__(self, name, 'DISCONNECTED', _DebugLevel, _Debug) for cb in self.callbacks.values(): cb(self.supplier_idurl, self.state, self.state)
def doRequestService(self, arg): """ Action method. """ bytes_needed = diskspace.GetBytesFromString(settings.getNeededString(), 0) num_suppliers = settings.getSuppliersNumberDesired() if num_suppliers > 0: bytes_per_supplier = int( math.ceil(2.0 * bytes_needed / float(num_suppliers))) else: bytes_per_supplier = int( math.ceil(2.0 * settings.MinimumNeededBytes() / float(settings.DefaultDesiredSuppliers()))) service_info = 'service_supplier %d' % bytes_per_supplier request = p2p_service.SendRequestService(self.idurl, service_info, callbacks={ commands.Ack(): self._supplier_acked, commands.Fail(): self._supplier_failed, }) self.request_packet_id = request.PacketID
def doCleanUpBackups(self, *args, **kwargs): # here we check all backups we have and remove the old one # user can set how many versions of that file or folder to keep # other versions (older) will be removed here from storage import backup_rebuilder try: self.backups_progress_last_iteration = len( backup_rebuilder.A().backupsWasRebuilt) except: self.backups_progress_last_iteration = 0 versionsToKeep = settings.getBackupsMaxCopies() if not contactsdb.num_suppliers(): bytesUsed = 0 else: bytesUsed = backup_fs.sizebackups() / contactsdb.num_suppliers() bytesNeeded = diskspace.GetBytesFromString(settings.getNeededString(), 0) customerGlobID = my_id.getGlobalID() if _Debug: lg.out( _DebugLevel, 'backup_monitor.doCleanUpBackups backupsToKeep=%d used=%d needed=%d' % (versionsToKeep, bytesUsed, bytesNeeded)) delete_count = 0 if versionsToKeep > 0: for pathID, localPath, itemInfo in backup_fs.IterateIDs(): pathID = global_id.CanonicalID(pathID) if backup_control.IsPathInProcess(pathID): continue versions = itemInfo.list_versions() # TODO: do we need to sort the list? it comes from a set, so must be sorted may be while len(versions) > versionsToKeep: backupID = packetid.MakeBackupID(customerGlobID, pathID, versions.pop(0)) if _Debug: lg.out( _DebugLevel, 'backup_monitor.doCleanUpBackups %d of %d backups for %s, so remove older %s' % (len(versions), versionsToKeep, localPath, backupID)) backup_control.DeleteBackup(backupID, saveDB=False, calculate=False) delete_count += 1 # we need also to fit used space into needed space (given from other users) # they trust us - do not need to take extra space from our friends # so remove oldest backups, but keep at least one for every folder - at least locally! # still our suppliers will remove our "extra" files by their "local_tester" if bytesNeeded <= bytesUsed: sizeOk = False for pathID, localPath, itemInfo in backup_fs.IterateIDs(): if sizeOk: break pathID = global_id.CanonicalID(pathID) versions = itemInfo.list_versions(True, False) if len(versions) <= 1: continue for version in versions[1:]: backupID = packetid.MakeBackupID(customerGlobID, pathID, version) versionInfo = itemInfo.get_version_info(version) if versionInfo[1] > 0: if _Debug: lg.out( _DebugLevel, 'backup_monitor.doCleanUpBackups over use %d of %d, so remove %s of %s' % (bytesUsed, bytesNeeded, backupID, localPath)) backup_control.DeleteBackup(backupID, saveDB=False, calculate=False) delete_count += 1 bytesUsed -= versionInfo[1] if bytesNeeded > bytesUsed: sizeOk = True break if delete_count > 0: backup_fs.Scan() backup_fs.Calculate() backup_control.Save() from main import control control.request_update() collected = gc.collect() if self.backups_progress_last_iteration > 0: if _Debug: lg.out( _DebugLevel, 'backup_monitor.doCleanUpBackups sending "restart", backups_progress_last_iteration=%s' % self.backups_progress_last_iteration) reactor.callLater(1, self.automat, 'restart') # @UndefinedVariable if _Debug: lg.out( _DebugLevel, 'backup_monitor.doCleanUpBackups collected %d objects' % collected)
def getInBytes(dirpath, default=-1): """ Return directory size in bytes, must use ``ask`` first. """ return diskspace.GetBytesFromString(get(dirpath), default)
def renderWizardStoragePage(self, request): template = 'pages/wizard_storage.html' req = {} if request is not None: req = request.REQUEST self.data['customersdir'] = unicode(req.get('customersdir', settings.getCustomersFilesDir())) self.data['localbackupsdir'] = unicode(req.get('localbackupsdir', settings.getLocalBackupsDir())) self.data['restoredir'] = unicode(req.get('restoredir', settings.getRestoreDir())) self.data['needed'] = req.get('needed', self.data['needed']) neededV = diskspace.GetBytesFromString(self.data['needed'] + ' Mb', settings.DefaultNeededBytes()) self.data['donated'] = req.get('donated', self.data['donated']) donatedV = diskspace.GetBytesFromString(self.data['donated'] + ' Mb', settings.DefaultDonatedBytes()) self.data['suppliers'] = req.get('suppliers', self.data['suppliers']) mounts = [] freeSpaceIsOk = True if bpio.Windows(): for d in bpio.listLocalDrivesWindows(): free, total = diskusage.GetWinDriveSpace(d[0]) if free is None or total is None: continue color = '#ffffff' if self.data['customersdir'][0].upper() == d[0].upper(): color = '#60e060' if donatedV >= free: color = '#e06060' freeSpaceIsOk = False if self.data['localbackupsdir'][0].upper() == d[0].upper(): color = '#60e060' if neededV >= free: color = '#e06060' freeSpaceIsOk = False mounts.append((d[0:2], diskspace.MakeStringFromBytes(free), diskspace.MakeStringFromBytes(total), color,)) elif bpio.Linux() or bpio.Mac(): for mnt in bpio.listMountPointsLinux(): free, total = diskusage.GetLinuxDriveSpace(mnt) if free is None or total is None: continue color = '#ffffff' if bpio.getMountPointLinux(self.data['customersdir']) == mnt: color = '#60e060' if donatedV >= free: color = '#e06060' freeSpaceIsOk = False if bpio.getMountPointLinux(self.data['localbackupsdir']) == mnt: color = '#60e060' if neededV >= free: color = '#e06060' freeSpaceIsOk = False mounts.append((mnt, diskspace.MakeStringFromBytes(free), diskspace.MakeStringFromBytes(total), color,)) ok = True out = '' if not freeSpaceIsOk: out += '<font color=red>you do not have enough free space on the disk</font><br/>\n' ok = False if donatedV < settings.MinimumDonatedBytes(): out += '<font color=red>you must donate at least %f MB</font><br/>\n' % ( round(settings.MinimumDonatedBytes() / (1024.0 * 1024.0), 2)) ok = False if not os.path.isdir(self.data['customersdir']): out += '<font color=red>directory %s not exist</font><br/>\n' % self.data['customersdir'] ok = False if not os.access(self.data['customersdir'], os.W_OK): out += '<font color=red>folder %s does not have write permissions</font><br/>\n' % self.data['customersdir'] ok = False if not os.path.isdir(self.data['localbackupsdir']): out += '<font color=red>directory %s not exist</font><br/>\n' % self.data['localbackupsdir'] ok = False if not os.access(self.data['localbackupsdir'], os.W_OK): out += '<font color=red>folder %s does not have write permissions</font><br/>\n' % self.data['localbackupsdir'] ok = False if int(self.data['suppliers']) not in settings.getECCSuppliersNumbers(): out += '<font color=red>incorrect number of suppliers, correct values are: %s</font><br/>\n' % ( str(settings.getECCSuppliersNumbers()).strip('[]')) ok = False context = {'output': out, 'mounts': mounts, 'needed': self.data['needed'], 'donated': self.data['donated'], 'localbackupsdir': self.data['localbackupsdir'], 'customersdir': self.data['customersdir'], 'restoredir': self.data['restoredir'], 'suppliers': self.data['suppliers'], } if request is None: return template, context, request action = request.REQUEST.get('action', None) if action == 'next': if ok: install_wizard.A(action, self.data) return None if action == 'back': install_wizard.A(action) return None return template, context, request