Пример #1
0
 def doSaveConfig(self, arg):
     """
     Action method.
     """
     self.configs = (settings.getSuppliersNumberDesired(),
                     diskspace.GetBytesFromString(
                         settings.getNeededString()))
Пример #2
0
 def doSaveConfig(self, arg):
     """
     Action method.
     """
     self.configs = (
         settings.getSuppliersNumberDesired(),
         diskspace.GetBytesFromString(
             settings.getNeededString()))
Пример #3
0
 def __init__(self,
              supplier_idurl,
              customer_idurl,
              needed_bytes,
              key_id=None,
              queue_subscribe=True):
     """
     """
     self.supplier_idurl = supplier_idurl
     self.customer_idurl = customer_idurl
     self.needed_bytes = needed_bytes
     self.key_id = key_id
     self.queue_subscribe = queue_subscribe
     if self.needed_bytes is None:
         total_bytes_needed = diskspace.GetBytesFromString(
             settings.getNeededString(), 0)
         num_suppliers = -1
         if self.customer_idurl == my_id.getLocalIDURL():
             num_suppliers = settings.getSuppliersNumberDesired()
         else:
             known_ecc_map = contactsdb.get_customer_meta_info(
                 customer_idurl).get('ecc_map')
             if known_ecc_map:
                 num_suppliers = eccmap.GetEccMapSuppliersNumber(
                     known_ecc_map)
         if num_suppliers > 0:
             self.needed_bytes = int(
                 math.ceil(2.0 * total_bytes_needed / float(num_suppliers)))
         else:
             raise Exception(
                 'not possible to determine needed_bytes value to be requested from that supplier'
             )
             # self.needed_bytes = int(math.ceil(2.0 * settings.MinimumNeededBytes() / float(settings.DefaultDesiredSuppliers())))
     name = 'supplier_%s_%s' % (
         nameurl.GetName(self.supplier_idurl),
         diskspace.MakeStringFromBytes(self.needed_bytes).replace(' ', ''),
     )
     self.request_packet_id = None
     self.callbacks = {}
     try:
         st = bpio.ReadTextFile(
             settings.SupplierServiceFilename(
                 idurl=self.supplier_idurl,
                 customer_idurl=self.customer_idurl,
             )).strip()
     except:
         st = 'DISCONNECTED'
     automat.Automat.__init__(
         self,
         name,
         state=st,
         debug_level=_DebugLevel,
         log_events=_Debug,
         log_transitions=_Debug,
     )
     for cb in self.callbacks.values():
         cb(self.supplier_idurl, self.state, self.state)
Пример #4
0
 def isConfigChanged(self, *args, **kwargs):
     """
     Condition method.
     """
     curconfigs = (settings.getSuppliersNumberDesired(),
                   diskspace.GetBytesFromString(settings.getNeededString()))
     if None in self.configs:
         return True
     return self.configs[0] != curconfigs[0] or self.configs[1] != curconfigs[1]
Пример #5
0
 def isConfigChanged(self, arg):
     """
     Condition method.
     """
     curconfigs = (settings.getSuppliersNumberDesired(),
                   diskspace.GetBytesFromString(settings.getNeededString()))
     if None in self.configs:
         return True
     return self.configs[0] != curconfigs[
         0] or self.configs[1] != curconfigs[1]
Пример #6
0
 def doCleanUpBackups(self, arg):
     # here we check all backups we have and remove the old one
     # user can set how many versions of that file or folder to keep
     # other versions (older) will be removed here
     versionsToKeep = settings.getBackupsMaxCopies()
     bytesUsed = backup_fs.sizebackups() / contactsdb.num_suppliers()
     bytesNeeded = diskspace.GetBytesFromString(settings.getNeededString(), 0)
     lg.out(6, 'backup_monitor.doCleanUpBackups backupsToKeep=%d used=%d needed=%d' % (versionsToKeep, bytesUsed, bytesNeeded))
     delete_count = 0
     if versionsToKeep > 0:
         for pathID, localPath, itemInfo in backup_fs.IterateIDs():
             if backup_control.IsPathInProcess(pathID):
                 continue
             versions = itemInfo.list_versions()
             # TODO: do we need to sort the list? it comes from a set, so must be sorted may be
             while len(versions) > versionsToKeep:
                 backupID = pathID + '/' + versions.pop(0)
                 lg.out(6, 'backup_monitor.doCleanUpBackups %d of %d backups for %s, so remove older %s' % (len(versions), versionsToKeep, localPath, backupID))
                 backup_control.DeleteBackup(backupID, saveDB=False, calculate=False)
                 delete_count += 1
     # we need also to fit used space into needed space (given from other users)
     # they trust us - do not need to take extra space from our friends
     # so remove oldest backups, but keep at least one for every folder - at least locally!
     # still our suppliers will remove our "extra" files by their "local_tester"
     if bytesNeeded <= bytesUsed:
         sizeOk = False
         for pathID, localPath, itemInfo in backup_fs.IterateIDs():
             if sizeOk:
                 break
             versions = itemInfo.list_versions(True, False)
             if len(versions) <= 1:
                 continue
             for version in versions[1:]:
                 backupID = pathID + '/' + version
                 versionInfo = itemInfo.get_version_info(version)
                 if versionInfo[1] > 0:
                     lg.out(6, 'backup_monitor.doCleanUpBackups over use %d of %d, so remove %s of %s' % (
                         bytesUsed, bytesNeeded, backupID, localPath))
                     backup_control.DeleteBackup(backupID, saveDB=False, calculate=False)
                     delete_count += 1
                     bytesUsed -= versionInfo[1]
                     if bytesNeeded > bytesUsed:
                         sizeOk = True
                         break
     if delete_count > 0:
         backup_fs.Scan()
         backup_fs.Calculate()
         backup_control.Save()
         from web import control
         control.request_update()
     collected = gc.collect()
     lg.out(6, 'backup_monitor.doCleanUpBackups collected %d objects' % collected)
Пример #7
0
 def do_calculate_needed_bytes(self):
     if self.needed_bytes is None:
         total_bytes_needed = diskspace.GetBytesFromString(settings.getNeededString(), 0)
         num_suppliers = -1
         if self.customer_idurl == my_id.getLocalIDURL():
             num_suppliers = settings.getSuppliersNumberDesired()
         else:
             known_ecc_map = contactsdb.get_customer_meta_info(self.customer_idurl).get('ecc_map')
             if known_ecc_map:
                 num_suppliers = eccmap.GetEccMapSuppliersNumber(known_ecc_map)
         if num_suppliers > 0:
             self.needed_bytes = int(math.ceil(2.0 * total_bytes_needed / float(num_suppliers)))
         else:
             raise Exception('not possible to determine needed_bytes value to be requested from that supplier')
Пример #8
0
 def __init__(self,
              supplier_idurl,
              customer_idurl,
              needed_bytes,
              key_id=None,
              queue_subscribe=True):
     """
     """
     self.supplier_idurl = supplier_idurl
     self.customer_idurl = customer_idurl
     self.needed_bytes = needed_bytes
     self.key_id = key_id
     self.queue_subscribe = queue_subscribe
     if self.needed_bytes is None:
         total_bytes_needed = diskspace.GetBytesFromString(
             settings.getNeededString(), 0)
         num_suppliers = settings.getSuppliersNumberDesired()
         if num_suppliers > 0:
             self.needed_bytes = int(
                 math.ceil(2.0 * total_bytes_needed / float(num_suppliers)))
         else:
             self.needed_bytes = int(
                 math.ceil(2.0 * settings.MinimumNeededBytes() /
                           float(settings.DefaultDesiredSuppliers())))
     name = 'supplier_%s_%s' % (
         nameurl.GetName(self.supplier_idurl),
         diskspace.MakeStringFromBytes(self.needed_bytes).replace(' ', ''),
     )
     self.request_packet_id = None
     self.callbacks = {}
     try:
         st = bpio.ReadTextFile(
             settings.SupplierServiceFilename(
                 idurl=self.supplier_idurl,
                 customer_idurl=self.customer_idurl,
             )).strip()
     except:
         st = 'DISCONNECTED'
     automat.Automat.__init__(
         self,
         name,
         state=st,
         debug_level=_DebugLevel,
         log_events=_Debug,
         log_transitions=_Debug,
     )
     for cb in self.callbacks.values():
         cb(self.supplier_idurl, self.state, self.state)
Пример #9
0
 def doRequestService(self, arg):
     """
     Action method.
     """
     bytes_needed = diskspace.GetBytesFromString(settings.getNeededString(), 0)
     num_suppliers = settings.getSuppliersNumberDesired()
     if num_suppliers > 0:
         bytes_per_supplier = int(math.ceil(2.0 * bytes_needed / float(num_suppliers)))
     else:
         bytes_per_supplier = int(math.ceil(2.0 * settings.MinimumNeededBytes() / float(settings.DefaultDesiredSuppliers())))
     service_info = 'service_supplier %d' % bytes_per_supplier
     request = p2p_service.SendRequestService(
         self.idurl, service_info, callbacks={
             commands.Ack(): self._supplier_acked,
             commands.Fail(): self._supplier_failed})
     # commands.Ack(): lambda response, info: self.automat('ack', response),
     # commands.Fail(): lambda response, info: self.automat('fail', response)})
     self.request_packet_id = request.PacketID
Пример #10
0
 def __init__(self, supplier_idurl, customer_idurl, needed_bytes):
     """
     """
     self.supplier_idurl = supplier_idurl
     self.customer_idurl = customer_idurl
     self.needed_bytes = needed_bytes
     if self.needed_bytes is None:
         total_bytes_needed = diskspace.GetBytesFromString(
             settings.getNeededString(), 0)
         num_suppliers = settings.getSuppliersNumberDesired()
         if num_suppliers > 0:
             self.needed_bytes = int(
                 math.ceil(2.0 * total_bytes_needed / float(num_suppliers)))
         else:
             self.needed_bytes = int(
                 math.ceil(2.0 * settings.MinimumNeededBytes() /
                           float(settings.DefaultDesiredSuppliers())))
     name = 'supplier_%s_%s' % (
         nameurl.GetName(self.supplier_idurl),
         diskspace.MakeStringFromBytes(self.needed_bytes).replace(' ', ''),
     )
     self.request_packet_id = None
     self.callbacks = {}
     try:
         st = bpio.ReadTextFile(
             settings.SupplierServiceFilename(
                 idurl=self.supplier_idurl,
                 customer_idurl=self.customer_idurl,
             )).strip()
     except:
         st = 'DISCONNECTED'
     if st == 'CONNECTED':
         automat.Automat.__init__(self, name, 'CONNECTED', _DebugLevel,
                                  _Debug)
     elif st == 'NO_SERVICE':
         automat.Automat.__init__(self, name, 'NO_SERVICE', _DebugLevel,
                                  _Debug)
     else:
         automat.Automat.__init__(self, name, 'DISCONNECTED', _DebugLevel,
                                  _Debug)
     for cb in self.callbacks.values():
         cb(self.supplier_idurl, self.state, self.state)
Пример #11
0
 def doRequestService(self, arg):
     """
     Action method.
     """
     bytes_needed = diskspace.GetBytesFromString(settings.getNeededString(),
                                                 0)
     num_suppliers = settings.getSuppliersNumberDesired()
     if num_suppliers > 0:
         bytes_per_supplier = int(
             math.ceil(2.0 * bytes_needed / float(num_suppliers)))
     else:
         bytes_per_supplier = int(
             math.ceil(2.0 * settings.MinimumNeededBytes() /
                       float(settings.DefaultDesiredSuppliers())))
     service_info = 'service_supplier %d' % bytes_per_supplier
     request = p2p_service.SendRequestService(self.idurl,
                                              service_info,
                                              callbacks={
                                                  commands.Ack():
                                                  self._supplier_acked,
                                                  commands.Fail():
                                                  self._supplier_failed,
                                              })
     self.request_packet_id = request.PacketID
Пример #12
0
 def doCleanUpBackups(self, *args, **kwargs):
     # here we check all backups we have and remove the old one
     # user can set how many versions of that file or folder to keep
     # other versions (older) will be removed here
     from storage import backup_rebuilder
     try:
         self.backups_progress_last_iteration = len(
             backup_rebuilder.A().backupsWasRebuilt)
     except:
         self.backups_progress_last_iteration = 0
     versionsToKeep = settings.getBackupsMaxCopies()
     if not contactsdb.num_suppliers():
         bytesUsed = 0
     else:
         bytesUsed = backup_fs.sizebackups() / contactsdb.num_suppliers()
     bytesNeeded = diskspace.GetBytesFromString(settings.getNeededString(),
                                                0)
     customerGlobID = my_id.getGlobalID()
     if _Debug:
         lg.out(
             _DebugLevel,
             'backup_monitor.doCleanUpBackups backupsToKeep=%d used=%d needed=%d'
             % (versionsToKeep, bytesUsed, bytesNeeded))
     delete_count = 0
     if versionsToKeep > 0:
         for pathID, localPath, itemInfo in backup_fs.IterateIDs():
             pathID = global_id.CanonicalID(pathID)
             if backup_control.IsPathInProcess(pathID):
                 continue
             versions = itemInfo.list_versions()
             # TODO: do we need to sort the list? it comes from a set, so must be sorted may be
             while len(versions) > versionsToKeep:
                 backupID = packetid.MakeBackupID(customerGlobID, pathID,
                                                  versions.pop(0))
                 if _Debug:
                     lg.out(
                         _DebugLevel,
                         'backup_monitor.doCleanUpBackups %d of %d backups for %s, so remove older %s'
                         % (len(versions), versionsToKeep, localPath,
                            backupID))
                 backup_control.DeleteBackup(backupID,
                                             saveDB=False,
                                             calculate=False)
                 delete_count += 1
     # we need also to fit used space into needed space (given from other users)
     # they trust us - do not need to take extra space from our friends
     # so remove oldest backups, but keep at least one for every folder - at least locally!
     # still our suppliers will remove our "extra" files by their "local_tester"
     if bytesNeeded <= bytesUsed:
         sizeOk = False
         for pathID, localPath, itemInfo in backup_fs.IterateIDs():
             if sizeOk:
                 break
             pathID = global_id.CanonicalID(pathID)
             versions = itemInfo.list_versions(True, False)
             if len(versions) <= 1:
                 continue
             for version in versions[1:]:
                 backupID = packetid.MakeBackupID(customerGlobID, pathID,
                                                  version)
                 versionInfo = itemInfo.get_version_info(version)
                 if versionInfo[1] > 0:
                     if _Debug:
                         lg.out(
                             _DebugLevel,
                             'backup_monitor.doCleanUpBackups over use %d of %d, so remove %s of %s'
                             %
                             (bytesUsed, bytesNeeded, backupID, localPath))
                     backup_control.DeleteBackup(backupID,
                                                 saveDB=False,
                                                 calculate=False)
                     delete_count += 1
                     bytesUsed -= versionInfo[1]
                     if bytesNeeded > bytesUsed:
                         sizeOk = True
                         break
     if delete_count > 0:
         backup_fs.Scan()
         backup_fs.Calculate()
         backup_control.Save()
         from main import control
         control.request_update()
     collected = gc.collect()
     if self.backups_progress_last_iteration > 0:
         if _Debug:
             lg.out(
                 _DebugLevel,
                 'backup_monitor.doCleanUpBackups  sending "restart", backups_progress_last_iteration=%s'
                 % self.backups_progress_last_iteration)
         reactor.callLater(1, self.automat, 'restart')  # @UndefinedVariable
     if _Debug:
         lg.out(
             _DebugLevel,
             'backup_monitor.doCleanUpBackups collected %d objects' %
             collected)