def doSaveConfig(self, arg):
"""
Action method.
"""
self.configs = (settings.getSuppliersNumberDesired(),
diskspace.GetBytesFromString(
settings.getNeededString()))
def doSaveConfig(self, arg):
"""
Action method.
"""
self.configs = (
settings.getSuppliersNumberDesired(),
diskspace.GetBytesFromString(
settings.getNeededString()))
def __init__(self,
supplier_idurl,
customer_idurl,
needed_bytes,
key_id=None,
queue_subscribe=True):
"""
"""
self.supplier_idurl = supplier_idurl
self.customer_idurl = customer_idurl
self.needed_bytes = needed_bytes
self.key_id = key_id
self.queue_subscribe = queue_subscribe
if self.needed_bytes is None:
total_bytes_needed = diskspace.GetBytesFromString(
settings.getNeededString(), 0)
num_suppliers = -1
if self.customer_idurl == my_id.getLocalIDURL():
num_suppliers = settings.getSuppliersNumberDesired()
else:
known_ecc_map = contactsdb.get_customer_meta_info(
customer_idurl).get('ecc_map')
if known_ecc_map:
num_suppliers = eccmap.GetEccMapSuppliersNumber(
known_ecc_map)
if num_suppliers > 0:
self.needed_bytes = int(
math.ceil(2.0 * total_bytes_needed / float(num_suppliers)))
else:
raise Exception(
'not possible to determine needed_bytes value to be requested from that supplier'
)
# self.needed_bytes = int(math.ceil(2.0 * settings.MinimumNeededBytes() / float(settings.DefaultDesiredSuppliers())))
name = 'supplier_%s_%s' % (
nameurl.GetName(self.supplier_idurl),
diskspace.MakeStringFromBytes(self.needed_bytes).replace(' ', ''),
)
self.request_packet_id = None
self.callbacks = {}
try:
st = bpio.ReadTextFile(
settings.SupplierServiceFilename(
idurl=self.supplier_idurl,
customer_idurl=self.customer_idurl,
)).strip()
except:
st = 'DISCONNECTED'
automat.Automat.__init__(
self,
name,
state=st,
debug_level=_DebugLevel,
log_events=_Debug,
log_transitions=_Debug,
)
for cb in self.callbacks.values():
cb(self.supplier_idurl, self.state, self.state)
def isConfigChanged(self, *args, **kwargs):
"""
Condition method.
"""
curconfigs = (settings.getSuppliersNumberDesired(),
diskspace.GetBytesFromString(settings.getNeededString()))
if None in self.configs:
return True
return self.configs[0] != curconfigs[0] or self.configs[1] != curconfigs[1]
def isConfigChanged(self, arg):
"""
Condition method.
"""
curconfigs = (settings.getSuppliersNumberDesired(),
diskspace.GetBytesFromString(settings.getNeededString()))
if None in self.configs:
return True
return self.configs[0] != curconfigs[
0] or self.configs[1] != curconfigs[1]
def doCleanUpBackups(self, arg):
# here we check all backups we have and remove the old one
# user can set how many versions of that file or folder to keep
# other versions (older) will be removed here
versionsToKeep = settings.getBackupsMaxCopies()
bytesUsed = backup_fs.sizebackups() / contactsdb.num_suppliers()
bytesNeeded = diskspace.GetBytesFromString(settings.getNeededString(), 0)
lg.out(6, 'backup_monitor.doCleanUpBackups backupsToKeep=%d used=%d needed=%d' % (versionsToKeep, bytesUsed, bytesNeeded))
delete_count = 0
if versionsToKeep > 0:
for pathID, localPath, itemInfo in backup_fs.IterateIDs():
if backup_control.IsPathInProcess(pathID):
continue
versions = itemInfo.list_versions()
# TODO: do we need to sort the list? it comes from a set, so must be sorted may be
while len(versions) > versionsToKeep:
backupID = pathID + '/' + versions.pop(0)
lg.out(6, 'backup_monitor.doCleanUpBackups %d of %d backups for %s, so remove older %s' % (len(versions), versionsToKeep, localPath, backupID))
backup_control.DeleteBackup(backupID, saveDB=False, calculate=False)
delete_count += 1
# we need also to fit used space into needed space (given from other users)
# they trust us - do not need to take extra space from our friends
# so remove oldest backups, but keep at least one for every folder - at least locally!
# still our suppliers will remove our "extra" files by their "local_tester"
if bytesNeeded <= bytesUsed:
sizeOk = False
for pathID, localPath, itemInfo in backup_fs.IterateIDs():
if sizeOk:
break
versions = itemInfo.list_versions(True, False)
if len(versions) <= 1:
continue
for version in versions[1:]:
backupID = pathID + '/' + version
versionInfo = itemInfo.get_version_info(version)
if versionInfo[1] > 0:
lg.out(6, 'backup_monitor.doCleanUpBackups over use %d of %d, so remove %s of %s' % (
bytesUsed, bytesNeeded, backupID, localPath))
backup_control.DeleteBackup(backupID, saveDB=False, calculate=False)
delete_count += 1
bytesUsed -= versionInfo[1]
if bytesNeeded > bytesUsed:
sizeOk = True
break
if delete_count > 0:
backup_fs.Scan()
backup_fs.Calculate()
backup_control.Save()
from web import control
control.request_update()
collected = gc.collect()
lg.out(6, 'backup_monitor.doCleanUpBackups collected %d objects' % collected)
def do_calculate_needed_bytes(self):
if self.needed_bytes is None:
total_bytes_needed = diskspace.GetBytesFromString(settings.getNeededString(), 0)
num_suppliers = -1
if self.customer_idurl == my_id.getLocalIDURL():
num_suppliers = settings.getSuppliersNumberDesired()
else:
known_ecc_map = contactsdb.get_customer_meta_info(self.customer_idurl).get('ecc_map')
if known_ecc_map:
num_suppliers = eccmap.GetEccMapSuppliersNumber(known_ecc_map)
if num_suppliers > 0:
self.needed_bytes = int(math.ceil(2.0 * total_bytes_needed / float(num_suppliers)))
else:
raise Exception('not possible to determine needed_bytes value to be requested from that supplier')
def __init__(self,
supplier_idurl,
customer_idurl,
needed_bytes,
key_id=None,
queue_subscribe=True):
"""
"""
self.supplier_idurl = supplier_idurl
self.customer_idurl = customer_idurl
self.needed_bytes = needed_bytes
self.key_id = key_id
self.queue_subscribe = queue_subscribe
if self.needed_bytes is None:
total_bytes_needed = diskspace.GetBytesFromString(
settings.getNeededString(), 0)
num_suppliers = settings.getSuppliersNumberDesired()
if num_suppliers > 0:
self.needed_bytes = int(
math.ceil(2.0 * total_bytes_needed / float(num_suppliers)))
else:
self.needed_bytes = int(
math.ceil(2.0 * settings.MinimumNeededBytes() /
float(settings.DefaultDesiredSuppliers())))
name = 'supplier_%s_%s' % (
nameurl.GetName(self.supplier_idurl),
diskspace.MakeStringFromBytes(self.needed_bytes).replace(' ', ''),
)
self.request_packet_id = None
self.callbacks = {}
try:
st = bpio.ReadTextFile(
settings.SupplierServiceFilename(
idurl=self.supplier_idurl,
customer_idurl=self.customer_idurl,
)).strip()
except:
st = 'DISCONNECTED'
automat.Automat.__init__(
self,
name,
state=st,
debug_level=_DebugLevel,
log_events=_Debug,
log_transitions=_Debug,
)
for cb in self.callbacks.values():
cb(self.supplier_idurl, self.state, self.state)
def doRequestService(self, arg):
"""
Action method.
"""
bytes_needed = diskspace.GetBytesFromString(settings.getNeededString(), 0)
num_suppliers = settings.getSuppliersNumberDesired()
if num_suppliers > 0:
bytes_per_supplier = int(math.ceil(2.0 * bytes_needed / float(num_suppliers)))
else:
bytes_per_supplier = int(math.ceil(2.0 * settings.MinimumNeededBytes() / float(settings.DefaultDesiredSuppliers())))
service_info = 'service_supplier %d' % bytes_per_supplier
request = p2p_service.SendRequestService(
self.idurl, service_info, callbacks={
commands.Ack(): self._supplier_acked,
commands.Fail(): self._supplier_failed})
# commands.Ack(): lambda response, info: self.automat('ack', response),
# commands.Fail(): lambda response, info: self.automat('fail', response)})
self.request_packet_id = request.PacketID
def __init__(self, supplier_idurl, customer_idurl, needed_bytes):
"""
"""
self.supplier_idurl = supplier_idurl
self.customer_idurl = customer_idurl
self.needed_bytes = needed_bytes
if self.needed_bytes is None:
total_bytes_needed = diskspace.GetBytesFromString(
settings.getNeededString(), 0)
num_suppliers = settings.getSuppliersNumberDesired()
if num_suppliers > 0:
self.needed_bytes = int(
math.ceil(2.0 * total_bytes_needed / float(num_suppliers)))
else:
self.needed_bytes = int(
math.ceil(2.0 * settings.MinimumNeededBytes() /
float(settings.DefaultDesiredSuppliers())))
name = 'supplier_%s_%s' % (
nameurl.GetName(self.supplier_idurl),
diskspace.MakeStringFromBytes(self.needed_bytes).replace(' ', ''),
)
self.request_packet_id = None
self.callbacks = {}
try:
st = bpio.ReadTextFile(
settings.SupplierServiceFilename(
idurl=self.supplier_idurl,
customer_idurl=self.customer_idurl,
)).strip()
except:
st = 'DISCONNECTED'
if st == 'CONNECTED':
automat.Automat.__init__(self, name, 'CONNECTED', _DebugLevel,
_Debug)
elif st == 'NO_SERVICE':
automat.Automat.__init__(self, name, 'NO_SERVICE', _DebugLevel,
_Debug)
else:
automat.Automat.__init__(self, name, 'DISCONNECTED', _DebugLevel,
_Debug)
for cb in self.callbacks.values():
cb(self.supplier_idurl, self.state, self.state)
def doRequestService(self, arg):
"""
Action method.
"""
bytes_needed = diskspace.GetBytesFromString(settings.getNeededString(),
0)
num_suppliers = settings.getSuppliersNumberDesired()
if num_suppliers > 0:
bytes_per_supplier = int(
math.ceil(2.0 * bytes_needed / float(num_suppliers)))
else:
bytes_per_supplier = int(
math.ceil(2.0 * settings.MinimumNeededBytes() /
float(settings.DefaultDesiredSuppliers())))
service_info = 'service_supplier %d' % bytes_per_supplier
request = p2p_service.SendRequestService(self.idurl,
service_info,
callbacks={
commands.Ack():
self._supplier_acked,
commands.Fail():
self._supplier_failed,
})
self.request_packet_id = request.PacketID
def doCleanUpBackups(self, *args, **kwargs):
# here we check all backups we have and remove the old one
# user can set how many versions of that file or folder to keep
# other versions (older) will be removed here
from storage import backup_rebuilder
try:
self.backups_progress_last_iteration = len(
backup_rebuilder.A().backupsWasRebuilt)
except:
self.backups_progress_last_iteration = 0
versionsToKeep = settings.getBackupsMaxCopies()
if not contactsdb.num_suppliers():
bytesUsed = 0
else:
bytesUsed = backup_fs.sizebackups() / contactsdb.num_suppliers()
bytesNeeded = diskspace.GetBytesFromString(settings.getNeededString(),
0)
customerGlobID = my_id.getGlobalID()
if _Debug:
lg.out(
_DebugLevel,
'backup_monitor.doCleanUpBackups backupsToKeep=%d used=%d needed=%d'
% (versionsToKeep, bytesUsed, bytesNeeded))
delete_count = 0
if versionsToKeep > 0:
for pathID, localPath, itemInfo in backup_fs.IterateIDs():
pathID = global_id.CanonicalID(pathID)
if backup_control.IsPathInProcess(pathID):
continue
versions = itemInfo.list_versions()
# TODO: do we need to sort the list? it comes from a set, so must be sorted may be
while len(versions) > versionsToKeep:
backupID = packetid.MakeBackupID(customerGlobID, pathID,
versions.pop(0))
if _Debug:
lg.out(
_DebugLevel,
'backup_monitor.doCleanUpBackups %d of %d backups for %s, so remove older %s'
% (len(versions), versionsToKeep, localPath,
backupID))
backup_control.DeleteBackup(backupID,
saveDB=False,
calculate=False)
delete_count += 1
# we need also to fit used space into needed space (given from other users)
# they trust us - do not need to take extra space from our friends
# so remove oldest backups, but keep at least one for every folder - at least locally!
# still our suppliers will remove our "extra" files by their "local_tester"
if bytesNeeded <= bytesUsed:
sizeOk = False
for pathID, localPath, itemInfo in backup_fs.IterateIDs():
if sizeOk:
break
pathID = global_id.CanonicalID(pathID)
versions = itemInfo.list_versions(True, False)
if len(versions) <= 1:
continue
for version in versions[1:]:
backupID = packetid.MakeBackupID(customerGlobID, pathID,
version)
versionInfo = itemInfo.get_version_info(version)
if versionInfo[1] > 0:
if _Debug:
lg.out(
_DebugLevel,
'backup_monitor.doCleanUpBackups over use %d of %d, so remove %s of %s'
%
(bytesUsed, bytesNeeded, backupID, localPath))
backup_control.DeleteBackup(backupID,
saveDB=False,
calculate=False)
delete_count += 1
bytesUsed -= versionInfo[1]
if bytesNeeded > bytesUsed:
sizeOk = True
break
if delete_count > 0:
backup_fs.Scan()
backup_fs.Calculate()
backup_control.Save()
from main import control
control.request_update()
collected = gc.collect()
if self.backups_progress_last_iteration > 0:
if _Debug:
lg.out(
_DebugLevel,
'backup_monitor.doCleanUpBackups sending "restart", backups_progress_last_iteration=%s'
% self.backups_progress_last_iteration)
reactor.callLater(1, self.automat, 'restart') # @UndefinedVariable
if _Debug:
lg.out(
_DebugLevel,
'backup_monitor.doCleanUpBackups collected %d objects' %
collected)