def doScanExistingPackets(self, *args, **kwargs):
"""
Action method.
"""
for SupplierNumber in range(self.EccMap.datasegments):
PacketID = packetid.MakePacketID(self.backup_id, self.block_number, SupplierNumber, 'Data')
customerID, remotePath = packetid.SplitPacketID(PacketID)
self.OnHandData[SupplierNumber] = bool(os.path.exists(os.path.join(
settings.getLocalBackupsDir(), customerID, remotePath)))
for SupplierNumber in range(self.EccMap.paritysegments):
PacketID = packetid.MakePacketID(self.backup_id, self.block_number, SupplierNumber, 'Parity')
customerID, remotePath = packetid.SplitPacketID(PacketID)
self.OnHandParity[SupplierNumber] = bool(os.path.exists(os.path.join(
settings.getLocalBackupsDir(), customerID, remotePath)))
def doScanExistingPackets(self, arg):
for SupplierNumber in range(self.EccMap.datasegments):
PacketID = packetid.MakePacketID(self.BackupID, self.BlockNumber,
SupplierNumber, 'Data')
customer, remotePath = packetid.SplitPacketID(PacketID)
self.OnHandData[SupplierNumber] = os.path.exists(
os.path.join(settings.getLocalBackupsDir(), customer,
remotePath))
for SupplierNumber in range(self.EccMap.paritysegments):
PacketID = packetid.MakePacketID(self.BackupID, self.BlockNumber,
SupplierNumber, 'Parity')
customer, remotePath = packetid.SplitPacketID(PacketID)
self.OnHandParity[SupplierNumber] = os.path.exists(
os.path.join(settings.getLocalBackupsDir(), customer,
remotePath))
def QueueRequestFile(self, callOnReceived, creatorID, packetID, ownerID,
remoteID):
# make sure that we don't actually already have the file
# if packetID != settings.BackupInfoFileName():
remoteID = id_url.field(remoteID)
ownerID = id_url.field(ownerID)
creatorID = id_url.field(creatorID)
if packetID not in [
settings.BackupInfoFileName(),
settings.BackupInfoFileNameOld(),
settings.BackupInfoEncryptedFileName(),
]:
customer, pathID = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(), customer,
pathID)
if os.path.exists(filename):
lg.warn("%s already exist " % filename)
if callOnReceived:
reactor.callLater(0, callOnReceived, packetID,
'exist') # @UndefinedVariable
return False
if remoteID not in list(self.supplierQueues.keys()):
# made a new queue for this man
self.supplierQueues[remoteID] = SupplierQueue(
remoteID, self.creatorID)
lg.info("made a new receiving queue for %s" %
nameurl.GetName(remoteID))
# lg.out(10, "io_throttle.QueueRequestFile asking for %s from %s" % (packetID, nameurl.GetName(remoteID)))
return self.supplierQueues[remoteID].SupplierRequestFile(
callOnReceived, creatorID, packetID, ownerID)
def DeletePathBackups(pathID,
removeLocalFilesToo=True,
saveDB=True,
calculate=True):
"""
This removes all backups of given path ID
Doing same operations as ``DeleteBackup()``.
"""
from . import backup_rebuilder
from customer import io_throttle
pathID = global_id.CanonicalID(pathID)
# get the working item
customer, remotePath = packetid.SplitPacketID(pathID)
customer_idurl = global_id.GlobalUserToIDURL(customer)
item = backup_fs.GetByID(remotePath, iterID=backup_fs.fsID(customer_idurl))
if item is None:
return False
lg.out(8, 'backup_control.DeletePathBackups ' + pathID)
# this is a list of all known backups of this path
versions = item.list_versions()
for version in versions:
backupID = packetid.MakeBackupID(customer, remotePath, version)
lg.out(8, ' removing %s' % backupID)
# abort backup if it just started and is running at the moment
AbortRunningBackup(backupID)
# if we requested for files for this backup - we do not need it anymore
io_throttle.DeleteBackupRequests(backupID)
io_throttle.DeleteBackupSendings(backupID)
# remove interests in transport_control
# callback.delete_backup_interest(backupID)
# remove local files for this backupID
if removeLocalFilesToo:
backup_fs.DeleteLocalBackup(settings.getLocalBackupsDir(),
backupID)
# remove remote info for this backup from the memory
backup_matrix.EraseBackupRemoteInfo(backupID)
# also remove local info
backup_matrix.EraseBackupLocalInfo(backupID)
# finally remove this backup from the index
item.delete_version(version)
# lg.out(8, 'backup_control.DeletePathBackups ' + backupID)
# stop any rebuilding, we will restart it soon
backup_rebuilder.RemoveAllBackupsToWork()
backup_rebuilder.SetStoppedFlag()
# check and calculate used space
if calculate:
backup_fs.Scan()
backup_fs.Calculate()
# save the index if needed
if saveDB:
Save()
control.request_update()
return True
def doRemoveTempFile(self, *args, **kwargs):
"""
Action method.
"""
if not args or not len(args) > 1:
return
filename = args[1]
if filename:
tmpfile.throw_out(filename, 'block restored')
if settings.getBackupsKeepLocalCopies():
return
from storage import backup_matrix
from storage import backup_rebuilder
if not backup_rebuilder.ReadStoppedFlag():
if backup_rebuilder.A().currentBackupID is not None:
if backup_rebuilder.A().currentBackupID == self.backup_id:
if _Debug:
lg.out(
_DebugLevel,
'restore_worker.doRemoveTempFile SKIP because rebuilding in process'
)
return
count = 0
for supplierNum in range(
contactsdb.num_suppliers(customer_idurl=self.customer_idurl)):
supplierIDURL = contactsdb.supplier(
supplierNum, customer_idurl=self.customer_idurl)
if not supplierIDURL:
continue
for dataORparity in [
'Data',
'Parity',
]:
packetID = packetid.MakePacketID(self.backup_id,
self.block_number,
supplierNum, dataORparity)
customer, remotePath = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(),
customer, remotePath)
if os.path.isfile(filename):
try:
os.remove(filename)
except:
lg.exc()
continue
count += 1
backup_matrix.LocalBlockReport(self.backup_id, self.block_number,
*args, **kwargs)
if _Debug:
lg.out(
_DebugLevel,
'restore_worker.doRemoveTempFile %d files were removed' %
count)
def constructFilename(customerIDURL, packetID):
customerGlobID, packetID = packetid.SplitPacketID(packetID)
if customerGlobID:
customerIDURL_packet = global_id.GlobalUserToIDURL(customerGlobID)
if customerIDURL_packet != customerIDURL:
lg.warn('construct filename for another customer: %s != %s' %
(customerIDURL_packet, customerIDURL))
customerDirName = nameurl.UrlFilename(customerIDURL)
customersDir = settings.getCustomersFilesDir()
if not os.path.exists(customersDir):
bpio._dir_make(customersDir)
ownerDir = os.path.join(customersDir, customerDirName)
if not os.path.exists(ownerDir):
bpio._dir_make(ownerDir)
filename = os.path.join(ownerDir, packetID)
return filename
def RunRequest(self):
packetsToRemove = {}
for i in range(
0, min(self.fileRequestMaxLength, len(self.fileRequestQueue))):
packetID = self.fileRequestQueue[i]
# must never happen, but just in case
if packetID not in self.fileRequestDict:
packetsToRemove[packetID] = 'broken'
lg.err('file %r not found in downloading queue for %r' %
(packetID, self.remoteID))
continue
f_down = self.fileRequestDict[packetID]
if f_down.state == 'IN_QUEUE':
customer, pathID = packetid.SplitPacketID(packetID)
if os.path.exists(
os.path.join(settings.getLocalBackupsDir(), customer,
pathID)):
# we have the data file, no need to request it
packetsToRemove[packetID] = 'exist'
else:
f_down.event('start')
# remember requests results
result = len(packetsToRemove)
# remove finished requests
for packetID, why in packetsToRemove.items():
if _Debug:
lg.out(
_DebugLevel,
"io_throttle.RunRequest %r to be removed from [%s] downloading queue because %r, %d more items"
%
(packetID, self.remoteID, why, len(self.fileRequestQueue)))
if packetID in self.fileRequestQueue:
f_down = self.fileRequestDict[packetID]
if why == 'exist':
f_down.event('file-already-exists')
else:
lg.warn(
'unexpected result "%r" for %r in downloading queue for %s'
% (why, packetID, self.remoteID))
f_down.event('stop')
else:
lg.warn('packet %r not found in request queue for [%s]' %
(packetID, self.remoteID))
del packetsToRemove
if result:
self.DoRequest()
return result
def _file_received(self, newpacket, state):
if state in ['in queue', 'shutdown', 'exist', 'failed']:
return
if state != 'received':
lg.warn("incorrect state [%s] for packet %s" %
(str(state), str(newpacket)))
return
if not newpacket.Valid():
# TODO: if we didn't get a valid packet ... re-request it or delete
# it?
lg.warn("%s is not a valid packet: %r" %
(newpacket.PacketID, newpacket))
return
# packetID = newpacket.PacketID
packetID = global_id.CanonicalID(newpacket.PacketID)
customer, remotePath = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(), customer,
remotePath)
if os.path.isfile(filename):
lg.warn("found existed file" + filename)
self.automat('inbox-data-packet', packetID)
return
# try:
# os.remove(filename)
# except:
# lg.exc()
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
bpio._dirs_make(dirname)
except:
lg.out(
2,
"backup_rebuilder._file_received ERROR can not create sub dir "
+ dirname)
return
if not bpio.WriteFile(filename, newpacket.Payload):
lg.out(2,
"backup_rebuilder._file_received ERROR writing " + filename)
return
from storage import backup_matrix
backup_matrix.LocalFileReport(packetID)
lg.out(10, "backup_rebuilder._file_received and wrote to " + filename)
self.automat('inbox-data-packet', packetID)
def _bk_done(bid, result):
from crypt import signed
customer, remotePath = packetid.SplitPacketID(bid)
try:
os.mkdir(os.path.join(settings.getLocalBackupsDir(), customer, remotePath + '.out'))
except:
pass
for filename in os.listdir(os.path.join(settings.getLocalBackupsDir(), customer, remotePath)):
filepath = os.path.join(settings.getLocalBackupsDir(), customer, remotePath, filename)
payld = bpio.ReadBinaryFile(filepath)
newpacket = signed.Packet(
'Data',
my_id.getLocalID(),
my_id.getLocalID(),
filename,
payld,
'http://megafaq.ru/cvps1010.xml')
newfilepath = os.path.join(settings.getLocalBackupsDir(), customer, remotePath + '.out', filename)
bpio.WriteBinaryFile(newfilepath, newpacket.Serialize())
reactor.stop()
def doRemoveTempFile(self, arg):
try:
filename = arg[1]
except:
return
tmpfile.throw_out(filename, 'block restored')
if settings.getBackupsKeepLocalCopies():
return
import backup_rebuilder
import backup_matrix
if not backup_rebuilder.ReadStoppedFlag():
if backup_rebuilder.A().currentBackupID is not None:
if backup_rebuilder.A().currentBackupID == self.BackupID:
lg.out(
6,
'restore.doRemoveTempFile SKIP because rebuilding in process'
)
return
count = 0
for supplierNum in xrange(
contactsdb.num_suppliers(customer_idurl=self.CustomerIDURL)):
supplierIDURL = contactsdb.supplier(
supplierNum, customer_idurl=self.CustomerIDURL)
if not supplierIDURL:
continue
for dataORparity in ['Data', 'Parity']:
packetID = packetid.MakePacketID(self.BackupID,
self.BlockNumber, supplierNum,
dataORparity)
customer, remotePath = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(),
customer, remotePath)
if os.path.isfile(filename):
try:
os.remove(filename)
except:
lg.exc()
continue
count += 1
backup_matrix.LocalBlockReport(self.BackupID, self.BlockNumber, arg)
lg.out(6, 'restore.doRemoveTempFile %d files were removed' % count)
def OnFoundFolderSize(pth, sz, arg):
"""
This is a callback, fired from ``lib.dirsize.ask()`` method after finish
calculating of folder size.
"""
try:
pathID, version = arg
customerGlobID, pathID = packetid.SplitPacketID(pathID)
customerIDURL = global_id.GlobalUserToIDURL(customerGlobID)
item = backup_fs.GetByID(pathID, iterID=backup_fs.fsID(customerIDURL))
if item:
item.set_size(sz)
backup_fs.Calculate()
Save()
if version:
backupID = packetid.MakeBackupID(customerGlobID, pathID, version)
job = GetRunningBackupObject(backupID)
if job:
job.totalSize = sz
if _Debug:
lg.out(_DebugLevel, 'backup_control.OnFoundFolderSize %s %d' % (backupID, sz))
except:
lg.exc()
def makeFilename(customerIDURL, packetID):
"""
Must be a customer, and then we make full path filename for where this
packet is stored locally.
"""
customerGlobID, packetID = packetid.SplitPacketID(packetID)
if not packetid.Valid(packetID): # SECURITY
if packetID not in [
settings.BackupInfoFileName(),
settings.BackupInfoFileNameOld(),
settings.BackupInfoEncryptedFileName(),
settings.BackupIndexFileName()
]:
# lg.out(1, "p2p_service.makeFilename ERROR failed packetID format: " + packetID )
return ''
if not contactsdb.is_customer(customerIDURL): # SECURITY
lg.warn("%s is not a customer" % (customerIDURL))
return ''
if customerGlobID:
customerIDURL_packet = global_id.GlobalUserToIDURL(customerGlobID)
if customerIDURL_packet != customerIDURL:
lg.warn('making filename for another customer: %s != %s' %
(customerIDURL_packet, customerIDURL))
return constructFilename(customerIDURL, packetID)
def doScanAndQueue(self, *args, **kwargs):
"""
Action method.
"""
global _ShutdownFlag
if _ShutdownFlag:
if _Debug:
lg.out(_DebugLevel,
'data_sender.doScanAndQueue _ShutdownFlag is True\n')
self.automat('scan-done', 0)
return
from storage import backup_matrix
from storage import backup_fs
backup_matrix.ReadLocalFiles()
progress = 0
# if _Debug:
# lg.out(_DebugLevel, 'data_sender.doScanAndQueue with %d known customers' % len(contactsdb.known_customers()))
for customer_idurl in contactsdb.known_customers():
if customer_idurl != my_id.getIDURL():
# TODO: check that later
if _Debug:
lg.out(
_DebugLevel + 2,
'data_sender.doScanAndQueue skip sending to another customer: %r'
% customer_idurl)
continue
known_suppliers = contactsdb.suppliers(customer_idurl)
if not known_suppliers or id_url.is_some_empty(known_suppliers):
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue found empty supplier(s) for customer %r, SKIP'
% customer_idurl)
continue
known_backups = misc.sorted_backup_ids(
list(backup_matrix.local_files().keys()), True)
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue found %d known suppliers for customer %r with %d backups'
%
(len(known_suppliers), customer_idurl, len(known_backups)))
for backupID in known_backups:
this_customer_idurl = packetid.CustomerIDURL(backupID)
if this_customer_idurl != customer_idurl:
continue
customerGlobalID, pathID, _ = packetid.SplitBackupID(
backupID, normalize_key_alias=True)
keyAlias = packetid.KeyAlias(customerGlobalID)
item = backup_fs.GetByID(pathID,
iterID=backup_fs.fsID(
customer_idurl, keyAlias))
if not item:
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue skip sending backup %r path not exist in catalog'
% backupID)
continue
if item.key_id and customerGlobalID and customerGlobalID != item.key_id:
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue skip sending backup %r key is different in the catalog: %r ~ %r'
% (
backupID,
customerGlobalID,
item.key_id,
))
continue
packetsBySupplier = backup_matrix.ScanBlocksToSend(
backupID, limit_per_supplier=None)
total_for_customer = sum(
[len(v) for v in packetsBySupplier.values()])
if total_for_customer:
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue sending %r for customer %r with %d pieces'
%
(item.name(), customer_idurl, total_for_customer))
for supplierNum in packetsBySupplier.keys():
# supplier_idurl = contactsdb.supplier(supplierNum, customer_idurl=customer_idurl)
if supplierNum >= 0 and supplierNum < len(
known_suppliers):
supplier_idurl = known_suppliers[supplierNum]
else:
supplier_idurl = None
if not supplier_idurl:
lg.warn(
'skip sending, unknown supplier_idurl supplierNum=%s for %s, customer_idurl=%r'
% (supplierNum, backupID, customer_idurl))
continue
for packetID in packetsBySupplier[supplierNum]:
backupID_, _, supplierNum_, _ = packetid.BidBnSnDp(
packetID)
if backupID_ != backupID:
lg.warn(
'skip sending, unexpected backupID supplierNum=%s for %s, customer_idurl=%r'
% (packetID, backupID, customer_idurl))
continue
if supplierNum_ != supplierNum:
lg.warn(
'skip sending, unexpected supplierNum %s for %s, customer_idurl=%r'
% (packetID, backupID, customer_idurl))
continue
if io_throttle.HasPacketInSendQueue(
supplier_idurl, packetID):
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue %s already in sending queue for %r'
% (packetID, supplier_idurl))
continue
latest_progress = self.statistic.get(
supplier_idurl, {}).get('latest', '')
if len(latest_progress
) >= 3 and latest_progress.endswith('---'):
if _Debug:
lg.out(
_DebugLevel + 2,
'data_sender.doScanAndQueue skip sending to supplier %r because multiple packets already failed'
% supplier_idurl)
continue
if not io_throttle.OkToSend(supplier_idurl):
if _Debug:
lg.out(
_DebugLevel + 2,
'data_sender.doScanAndQueue skip sending, queue is busy for %r'
% supplier_idurl)
continue
customerGlobalID, pathID = packetid.SplitPacketID(
packetID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customerGlobalID,
pathID,
)
if not os.path.isfile(filename):
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue %s is not a file'
% filename)
continue
itemInfo = item.to_json()
if io_throttle.QueueSendFile(
filename,
packetID,
supplier_idurl,
my_id.getIDURL(),
lambda packet, ownerID,
packetID: self._packetAcked(
packet, ownerID, packetID, itemInfo),
lambda remoteID, packetID,
why: self._packetFailed(
remoteID, packetID, why, itemInfo),
):
progress += 1
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue for %r put %s in the queue progress=%d'
% (
item.name(),
packetID,
progress,
))
else:
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue io_throttle.QueueSendFile FAILED %s'
% packetID)
if _Debug:
lg.out(_DebugLevel,
'data_sender.doScanAndQueue progress=%s' % progress)
self.automat('scan-done', progress)
def doRemoveUnusedFiles(self, arg):
# we want to remove files for this block
# because we only need them during rebuilding
if settings.getBackupsKeepLocalCopies() is True:
# if user set this in settings - he want to keep the local files
return
# ... user do not want to keep local backups
if settings.getGeneralWaitSuppliers() is True:
from customer import fire_hire
# but he want to be sure - all suppliers are green for a long time
if len(contact_status.listOfflineSuppliers(
)) > 0 or time.time() - fire_hire.GetLastFireTime() < 24 * 60 * 60:
# some people are not there or we do not have stable team yet
# do not remove the files because we need it to rebuild
return
count = 0
from storage import backup_matrix
from storage import restore_monitor
from storage import backup_rebuilder
if _Debug:
lg.out(_DebugLevel, 'data_sender.doRemoveUnusedFiles')
for backupID in misc.sorted_backup_ids(
backup_matrix.local_files().keys()):
if restore_monitor.IsWorking(backupID):
if _Debug:
lg.out(_DebugLevel,
' %s : SKIP, because restoring' % backupID)
continue
if backup_rebuilder.IsBackupNeedsWork(backupID):
if _Debug:
lg.out(
_DebugLevel,
' %s : SKIP, because needs rebuilding' %
backupID)
continue
if not backup_rebuilder.ReadStoppedFlag():
if backup_rebuilder.A().currentBackupID is not None:
if backup_rebuilder.A().currentBackupID == backupID:
if _Debug:
lg.out(
_DebugLevel,
' %s : SKIP, because rebuilding is in process'
% backupID)
continue
packets = backup_matrix.ScanBlocksToRemove(
backupID, settings.getGeneralWaitSuppliers())
for packetID in packets:
customer, pathID = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(),
customer, pathID)
if os.path.isfile(filename):
try:
os.remove(filename)
# lg.out(6, ' ' + os.path.basename(filename))
except:
lg.exc()
continue
count += 1
if _Debug:
lg.out(_DebugLevel, ' %d files were removed' % count)
backup_matrix.ReadLocalFiles()
def doScanAndQueue(self, arg):
global _ShutdownFlag
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue _ShutdownFlag=%r' % _ShutdownFlag)
if _Debug:
log = open(os.path.join(settings.LogsDir(), 'data_sender.log'),
'w')
log.write('doScanAndQueue %s\n' % time.asctime())
if _ShutdownFlag:
if _Debug:
log.write('doScanAndQueue _ShutdownFlag is True\n')
self.automat('scan-done')
if _Debug:
log.flush()
log.close()
return
for customer_idurl in contactsdb.known_customers():
if '' not in contactsdb.suppliers(customer_idurl):
from storage import backup_matrix
for backupID in misc.sorted_backup_ids(
backup_matrix.local_files().keys(), True):
packetsBySupplier = backup_matrix.ScanBlocksToSend(
backupID)
if _Debug:
log.write('%s\n' % packetsBySupplier)
for supplierNum in packetsBySupplier.keys():
supplier_idurl = contactsdb.supplier(
supplierNum, customer_idurl=customer_idurl)
if not supplier_idurl:
lg.warn('?supplierNum? %s for %s' %
(supplierNum, backupID))
continue
for packetID in packetsBySupplier[supplierNum]:
backupID_, _, supplierNum_, _ = packetid.BidBnSnDp(
packetID)
if backupID_ != backupID:
lg.warn('?backupID? %s for %s' %
(packetID, backupID))
continue
if supplierNum_ != supplierNum:
lg.warn('?supplierNum? %s for %s' %
(packetID, backupID))
continue
if io_throttle.HasPacketInSendQueue(
supplier_idurl, packetID):
if _Debug:
log.write(
'%s already in sending queue for %s\n'
% (packetID, supplier_idurl))
continue
if not io_throttle.OkToSend(supplier_idurl):
if _Debug:
log.write('ok to send %s ? - NO!\n' %
supplier_idurl)
continue
customerGlobalID, pathID = packetid.SplitPacketID(
packetID)
# tranByID = gate.transfers_out_by_idurl().get(supplier_idurl, [])
# if len(tranByID) > 3:
# log.write('transfers by %s: %d\n' % (supplier_idurl, len(tranByID)))
# continue
customerGlobalID, pathID = packetid.SplitPacketID(
packetID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customerGlobalID,
pathID,
)
if not os.path.isfile(filename):
if _Debug:
log.write('%s is not a file\n' % filename)
continue
if io_throttle.QueueSendFile(
filename,
packetID,
supplier_idurl,
my_id.getLocalID(),
self._packetAcked,
self._packetFailed,
):
if _Debug:
log.write(
'io_throttle.QueueSendFile %s\n' %
packetID)
else:
if _Debug:
log.write(
'io_throttle.QueueSendFile FAILED %s\n'
% packetID)
# lg.out(6, ' %s for %s' % (packetID, backupID))
# DEBUG
# break
self.automat('scan-done')
if _Debug:
log.flush()
log.close()
def doScanAndQueue(self, *args, **kwargs):
"""
Action method.
"""
global _ShutdownFlag
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue _ShutdownFlag=%r' % _ShutdownFlag)
if _ShutdownFlag:
if _Debug:
lg.out(_DebugLevel, ' _ShutdownFlag is True\n')
self.automat('scan-done')
return
from storage import backup_matrix
backup_matrix.ReadLocalFiles()
progress = 0
for customer_idurl in contactsdb.known_customers():
if customer_idurl != my_id.getLocalIDURL():
# TODO: check that later
if _Debug:
lg.out(
_DebugLevel + 6,
' skip sending to another customer: %r' %
customer_idurl)
continue
known_suppliers = contactsdb.suppliers(customer_idurl)
if b'' in known_suppliers or '' in known_suppliers:
if _Debug:
lg.out(
_DebugLevel,
' found empty supplier for customer %r, SKIP' %
customer_idurl)
continue
known_backups = misc.sorted_backup_ids(
list(backup_matrix.local_files().keys()), True)
if _Debug:
lg.out(
_DebugLevel,
' found %d known suppliers for customer %r with %d backups'
%
(len(known_suppliers), customer_idurl, len(known_backups)))
for backupID in known_backups:
this_customer_idurl = packetid.CustomerIDURL(backupID)
if this_customer_idurl != customer_idurl:
continue
packetsBySupplier = backup_matrix.ScanBlocksToSend(backupID)
if _Debug:
lg.out(
_DebugLevel, ' packets for customer %r : %s' %
(customer_idurl, packetsBySupplier))
for supplierNum in packetsBySupplier.keys():
# supplier_idurl = contactsdb.supplier(supplierNum, customer_idurl=customer_idurl)
try:
supplier_idurl = known_suppliers[supplierNum]
except:
lg.exc()
continue
if not supplier_idurl:
lg.warn(
'unknown supplier_idurl supplierNum=%s for %s, customer_idurl=%r'
% (supplierNum, backupID, customer_idurl))
continue
for packetID in packetsBySupplier[supplierNum]:
backupID_, _, supplierNum_, _ = packetid.BidBnSnDp(
packetID)
if backupID_ != backupID:
lg.warn(
'unexpected backupID supplierNum=%s for %s, customer_idurl=%r'
% (packetID, backupID, customer_idurl))
continue
if supplierNum_ != supplierNum:
lg.warn(
'unexpected supplierNum %s for %s, customer_idurl=%r'
% (packetID, backupID, customer_idurl))
continue
if io_throttle.HasPacketInSendQueue(
supplier_idurl, packetID):
if _Debug:
lg.out(
_DebugLevel,
' %s already in sending queue for %r'
% (packetID, supplier_idurl))
continue
if not io_throttle.OkToSend(supplier_idurl):
if _Debug:
lg.out(
_DebugLevel + 6,
' skip, not ok to send %s\n' %
supplier_idurl)
continue
customerGlobalID, pathID = packetid.SplitPacketID(
packetID)
# tranByID = gate.transfers_out_by_idurl().get(supplier_idurl, [])
# if len(tranByID) > 3:
# log.write(u'transfers by %s: %d\n' % (supplier_idurl, len(tranByID)))
# continue
customerGlobalID, pathID = packetid.SplitPacketID(
packetID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customerGlobalID,
pathID,
)
if not os.path.isfile(filename):
if _Debug:
lg.out(_DebugLevel,
' %s is not a file\n' % filename)
continue
if io_throttle.QueueSendFile(
filename,
packetID,
supplier_idurl,
my_id.getLocalID(),
self._packetAcked,
self._packetFailed,
):
progress += 1
if _Debug:
lg.out(
_DebugLevel,
' io_throttle.QueueSendFile %s' %
packetID)
else:
if _Debug:
lg.out(
_DebugLevel,
' io_throttle.QueueSendFile FAILED %s'
% packetID)
if _Debug:
lg.out(_DebugLevel,
'data_sender.doScanAndQueue progress=%s' % progress)
self.automat('scan-done')
def doScanAndQueue(self, *args, **kwargs):
"""
Action method.
"""
global _ShutdownFlag
if _ShutdownFlag:
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue _ShutdownFlag is True\n')
self.automat('scan-done', 0)
return
from storage import backup_matrix
from storage import backup_fs
backup_matrix.ReadLocalFiles()
progress = 0
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue with %d known customers' % len(contactsdb.known_customers()))
for customer_idurl in contactsdb.known_customers():
if customer_idurl != my_id.getLocalID():
# TODO: check that later
if _Debug:
lg.out(_DebugLevel + 6, 'data_sender.doScanAndQueue skip sending to another customer: %r' % customer_idurl)
continue
known_suppliers = contactsdb.suppliers(customer_idurl)
if not known_suppliers or id_url.is_some_empty(known_suppliers):
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue found empty supplier(s) for customer %r, SKIP' % customer_idurl)
continue
known_backups = misc.sorted_backup_ids(list(backup_matrix.local_files().keys()), True)
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue found %d known suppliers for customer %r with %d backups' % (
len(known_suppliers), customer_idurl, len(known_backups)))
for backupID in known_backups:
this_customer_idurl = packetid.CustomerIDURL(backupID)
if this_customer_idurl != customer_idurl:
continue
customerGlobalID, pathID, _ = packetid.SplitBackupID(backupID, normalize_key_alias=True)
item = backup_fs.GetByID(pathID, iterID=backup_fs.fsID(customer_idurl=customer_idurl))
if not item:
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue skip sending backup %r path not exist in catalog' % backupID)
continue
if item.key_id and customerGlobalID and customerGlobalID != item.key_id:
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue skip sending backup %r key is different in the catalog' % backupID)
continue
packetsBySupplier = backup_matrix.ScanBlocksToSend(backupID, limit_per_supplier=None)
total_for_customer = sum([len(v) for v in packetsBySupplier.values()])
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue to be delivered for customer %r : %d' % (customer_idurl, total_for_customer))
for supplierNum in packetsBySupplier.keys():
# supplier_idurl = contactsdb.supplier(supplierNum, customer_idurl=customer_idurl)
if supplierNum >= 0 and supplierNum < len(known_suppliers):
supplier_idurl = known_suppliers[supplierNum]
else:
supplier_idurl = None
if not supplier_idurl:
lg.warn('skip sending, unknown supplier_idurl supplierNum=%s for %s, customer_idurl=%r' % (
supplierNum, backupID, customer_idurl))
continue
for packetID in packetsBySupplier[supplierNum]:
backupID_, _, supplierNum_, _ = packetid.BidBnSnDp(packetID)
if backupID_ != backupID:
lg.warn('skip sending, unexpected backupID supplierNum=%s for %s, customer_idurl=%r' % (
packetID, backupID, customer_idurl))
continue
if supplierNum_ != supplierNum:
lg.warn('skip sending, unexpected supplierNum %s for %s, customer_idurl=%r' % (
packetID, backupID, customer_idurl))
continue
if io_throttle.HasPacketInSendQueue(supplier_idurl, packetID):
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue %s already in sending queue for %r' % (packetID, supplier_idurl))
continue
if not io_throttle.OkToSend(supplier_idurl):
if _Debug:
lg.out(_DebugLevel + 6, 'data_sender.doScanAndQueue skip sending, queue is busy for %r\n' % supplier_idurl)
continue
# customerGlobalID, pathID = packetid.SplitPacketID(packetID)
# tranByID = gate.transfers_out_by_idurl().get(supplier_idurl, [])
# if len(tranByID) > 3:
# log.write(u'transfers by %s: %d\n' % (supplier_idurl, len(tranByID)))
# continue
customerGlobalID, pathID = packetid.SplitPacketID(packetID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customerGlobalID,
pathID,
)
if not os.path.isfile(filename):
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue %s is not a file\n' % filename)
continue
if io_throttle.QueueSendFile(
filename,
packetID,
supplier_idurl,
my_id.getIDURL(),
self._packetAcked,
self._packetFailed,
):
progress += 1
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue put %s in the queue progress=%d' % (packetID, progress, ))
else:
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue io_throttle.QueueSendFile FAILED %s' % packetID)
if _Debug:
lg.out(_DebugLevel, 'data_sender.doScanAndQueue progress=%s' % progress)
self.automat('scan-done', progress)
def RunRequest(self):
#out(6, 'io_throttle.RunRequest')
packetsToRemove = set()
for i in range(
0, min(self.fileRequestMaxLength, len(self.fileRequestQueue))):
packetID = self.fileRequestQueue[i]
currentTime = time.time()
if self.fileRequestDict[packetID].requestTime is not None:
# the packet were requested
if self.fileRequestDict[packetID].fileReceivedTime is None:
# but no answer yet ...
if currentTime - self.fileRequestDict[
packetID].requestTime > self.fileRequestDict[
packetID].requestTimeout:
# and time is out!!!
self.fileRequestDict[packetID].report = 'timeout'
packetsToRemove.add(packetID)
else:
# the packet were received (why it is not removed from the queue yet ???)
self.fileRequestDict[packetID].result = 'received'
packetsToRemove.add(packetID)
if self.fileRequestDict[packetID].requestTime is None:
customer, pathID = packetid.SplitPacketID(packetID)
if not os.path.exists(
os.path.join(settings.getLocalBackupsDir(), customer,
pathID)):
fileRequest = self.fileRequestDict[packetID]
lg.out(
10, "io_throttle.RunRequest for packetID " +
fileRequest.packetID)
# transport_control.RegisterInterest(self.DataReceived,fileRequest.creatorID,fileRequest.packetID)
# callback.register_interest(self.DataReceived, fileRequest.creatorID, fileRequest.packetID)
p2p_service.SendRetreive(fileRequest.ownerID,
fileRequest.creatorID,
fileRequest.packetID,
fileRequest.remoteID,
callbacks={
commands.Data():
self.DataReceived,
commands.Fail():
self.DataReceived,
})
# newpacket = signed.Packet(
# commands.Retrieve(),
# fileRequest.ownerID,
# fileRequest.creatorID,
# packetid.RemotePath(fileRequest.packetID),
# "",
# fileRequest.remoteID)
# gateway.outbox(newpacket, callbacks={
# commands.Data(): self.DataReceived,
# commands.Fail(): self.DataReceived})
fileRequest.requestTime = time.time()
else:
# we have the data file, no need to request it
self.fileRequestDict[packetID].result = 'exist'
packetsToRemove.add(packetID)
# remember requests results
result = len(packetsToRemove)
# remove finished requests
if len(packetsToRemove) > 0:
for packetID in packetsToRemove:
self.fileRequestQueue.remove(packetID)
del packetsToRemove
return result
def _request_files(self):
from storage import backup_matrix
from customer import io_throttle
from customer import data_sender
self.missingPackets = 0
# here we want to request some packets before we start working to
# rebuild the missed blocks
availableSuppliers = backup_matrix.GetActiveArray(customer_idurl=self.currentCustomerIDURL)
# remember how many requests we did on this iteration
total_requests_count = 0
# at the moment I do download everything I have available and needed
if '' in contactsdb.suppliers(customer_idurl=self.currentCustomerIDURL):
lg.out(8, 'backup_rebuilder._request_files SKIP - empty supplier')
self.automat('no-requests')
return
for supplierNum in range(contactsdb.num_suppliers(customer_idurl=self.currentCustomerIDURL)):
supplierID = contactsdb.supplier(supplierNum, customer_idurl=self.currentCustomerIDURL)
if not supplierID:
continue
requests_count = 0
# we do requests in reverse order because we start rebuilding from
# the last block
for blockIndex in range(len(self.workingBlocksQueue) - 1, -1, -1):
blockNum = self.workingBlocksQueue[blockIndex]
# do not keep too many requests in the queue
if io_throttle.GetRequestQueueLength(supplierID) >= 16:
break
# also don't do too many requests at once
if requests_count > 16:
break
remoteData = backup_matrix.GetRemoteDataArray(
self.currentBackupID, blockNum)
remoteParity = backup_matrix.GetRemoteParityArray(
self.currentBackupID, blockNum)
localData = backup_matrix.GetLocalDataArray(
self.currentBackupID, blockNum)
localParity = backup_matrix.GetLocalParityArray(
self.currentBackupID, blockNum)
if supplierNum >= len(remoteData) or supplierNum >= len(remoteParity):
break
if supplierNum >= len(localData) or supplierNum >= len(localParity):
break
# if remote Data exist and is available because supplier is on-line,
# but we do not have it on hand - do request
if localData[supplierNum] == 0:
PacketID = packetid.MakePacketID(
self.currentBackupID, blockNum, supplierNum, 'Data')
if remoteData[supplierNum] == 1:
if availableSuppliers[supplierNum]:
# if supplier is not alive - we can't request from him
if not io_throttle.HasPacketInRequestQueue(supplierID, PacketID):
customer, remotePath = packetid.SplitPacketID(PacketID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customer,
remotePath,
)
if not os.path.exists(filename):
if io_throttle.QueueRequestFile(
self._file_received,
my_id.getLocalID(),
PacketID,
my_id.getLocalID(),
supplierID):
requests_count += 1
else:
# count this packet as missing
self.missingPackets += 1
# also mark this guy as one who dont have any data - nor local nor remote
else:
# but if local Data already exists, but was not sent - do it now
if remoteData[supplierNum] != 1:
data_sender.A('new-data')
# same for Parity
if localParity[supplierNum] == 0:
PacketID = packetid.MakePacketID(
self.currentBackupID, blockNum, supplierNum, 'Parity')
if remoteParity[supplierNum] == 1:
if availableSuppliers[supplierNum]:
if not io_throttle.HasPacketInRequestQueue(
supplierID, PacketID):
customer, remotePath = packetid.SplitPacketID(PacketID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customer,
remotePath,
)
if not os.path.exists(filename):
if io_throttle.QueueRequestFile(
self._file_received,
my_id.getLocalID(),
PacketID,
my_id.getLocalID(),
supplierID,
):
requests_count += 1
else:
self.missingPackets += 1
else:
# but if local Parity already exists, but was not sent - do it now
if remoteParity[supplierNum] != 1:
data_sender.A('new-data')
total_requests_count += requests_count
if total_requests_count > 0:
lg.out(8, 'backup_rebuilder._request_files : %d chunks requested' % total_requests_count)
self.automat('requests-sent', total_requests_count)
else:
if self.missingPackets:
lg.out(8, 'backup_rebuilder._request_files : found %d missing packets' % self.missingPackets)
self.automat('found-missing')
else:
lg.out(8, 'backup_rebuilder._request_files : nothing was requested')
self.automat('no-requests')
def RunRequest(self):
#out(6, 'io_throttle.RunRequest')
packetsToRemove = {}
for i in range(0, min(self.fileRequestMaxLength, len(self.fileRequestQueue))):
packetID = self.fileRequestQueue[i]
# we got notify that this packet was failed to send
if packetID in self.requestFailedPacketIDs:
self.requestFailedPacketIDs.remove(packetID)
packetsToRemove[packetID] = 'failed'
continue
# request timeouts are disabled for now
# currentTime = time.time()
# if self.fileRequestDict[packetID].requestTime is not None:
# # the packet was requested
# if self.fileRequestDict[packetID].fileReceivedTime is None:
# # but no answer yet ...
# if currentTime - self.fileRequestDict[packetID].requestTime > self.fileRequestDict[packetID].requestTimeout:
# # and time is out!!!
# self.fileRequestDict[packetID].report = 'timeout'
# packetsToRemove[packetID] = 'timeout'
# else:
# # the packet were received (why it is not removed from the queue yet ???)
# self.fileRequestDict[packetID].result = 'received'
# packetsToRemove[packetID] = 'received'
# the packet was not requested yet
if self.fileRequestDict[packetID].requestTime is None:
customer, pathID = packetid.SplitPacketID(packetID)
if not os.path.exists(os.path.join(settings.getLocalBackupsDir(), customer, pathID)):
fileRequest = self.fileRequestDict[packetID]
if _Debug:
lg.out(_DebugLevel, "io_throttle.RunRequest for packetID " + fileRequest.packetID)
# transport_control.RegisterInterest(self.DataReceived,fileRequest.creatorID,fileRequest.packetID)
# callback.register_interest(self.DataReceived, fileRequest.creatorID, fileRequest.packetID)
p2p_service.SendRetreive(
fileRequest.ownerID,
fileRequest.creatorID,
fileRequest.packetID,
fileRequest.remoteID,
callbacks={
commands.Data(): self.OnDataReceived,
commands.Fail(): self.OnDataReceived,
# None: lambda pkt_out: self.OnDataReceived(fileRequest.packetID, 'timeout'), # timeout
},
# response_timeout=10,
)
# newpacket = signed.Packet(
# commands.Retrieve(),
# fileRequest.ownerID,
# fileRequest.creatorID,
# packetid.RemotePath(fileRequest.packetID),
# "",
# fileRequest.remoteID)
# gateway.outbox(newpacket, callbacks={
# commands.Data(): self.DataReceived,
# commands.Fail(): self.DataReceived})
fileRequest.requestTime = time.time()
else:
# we have the data file, no need to request it
self.fileRequestDict[packetID].result = 'exist'
packetsToRemove[packetID] = 'exist'
# if request queue is empty - remove all records about packets failed to request
if len(self.fileRequestQueue) == 0:
del self.requestFailedPacketIDs[:]
# remember requests results
result = len(packetsToRemove)
# remove finished requests
for packetID, why in packetsToRemove.items():
# self.fileRequestQueue.remove(packetID)
if _Debug:
lg.out(_DebugLevel, "io_throttle.RunRequest removed %s from %s receiving queue, %d more items" % (
packetID, self.remoteName, len(self.fileRequestQueue)))
self.OnDataRequestFailed(packetID, why)
del packetsToRemove
return result
def doRemoveUnusedFiles(self, *args, **kwargs):
"""
Action method.
"""
if not list_files_orator.is_synchronized():
# always make sure we have a very fresh info about remote files before take any actions
return
# we want to remove files for this block
# because we only need them during rebuilding
if settings.getBackupsKeepLocalCopies() is True:
# if user set this in settings - he want to keep the local files
return
# ... user do not want to keep local backups
if settings.getGeneralWaitSuppliers() is True:
from customer import fire_hire
# but he want to be sure - all suppliers are green for a long time
if len(online_status.listOfflineSuppliers()) > 0 or (
time.time() - fire_hire.GetLastFireTime() < 24 * 60 * 60):
# some people are not there or we do not have stable team yet
# do not remove the files because we need it to rebuild
return
count = 0
from storage import backup_matrix
from storage import restore_monitor
from storage import backup_rebuilder
if _Debug:
lg.out(_DebugLevel, 'data_sender.doRemoveUnusedFiles')
for backupID in misc.sorted_backup_ids(
list(backup_matrix.local_files().keys())):
if restore_monitor.IsWorking(backupID):
if _Debug:
lg.out(_DebugLevel,
' %s : SKIP, because restoring' % backupID)
continue
if backup_rebuilder.IsBackupNeedsWork(backupID):
if _Debug:
lg.out(
_DebugLevel,
' %s : SKIP, because needs rebuilding' %
backupID)
continue
if not backup_rebuilder.ReadStoppedFlag():
if backup_rebuilder.A().currentBackupID is not None:
if backup_rebuilder.A().currentBackupID == backupID:
if _Debug:
lg.out(
_DebugLevel,
' %s : SKIP, because rebuilding is in process'
% backupID)
continue
if backupID not in backup_matrix.remote_files():
if _Debug:
lg.out(
_DebugLevel,
' going to erase %s because not found in remote files'
% backupID)
customer, pathID, version = packetid.SplitBackupID(backupID)
dirpath = os.path.join(settings.getLocalBackupsDir(), customer,
pathID, version)
if os.path.isdir(dirpath):
try:
count += bpio.rmdir_recursive(dirpath,
ignore_errors=True)
except:
lg.exc()
continue
packets = backup_matrix.ScanBlocksToRemove(
backupID,
check_all_suppliers=settings.getGeneralWaitSuppliers())
for packetID in packets:
customer, pathID = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(),
customer, pathID)
if os.path.isfile(filename):
try:
os.remove(filename)
except:
lg.exc()
continue
count += 1
if _Debug:
lg.out(_DebugLevel, ' %d files were removed' % count)
backup_matrix.ReadLocalFiles()