def doScanExistingPackets(self, *args, **kwargs):
"""
Action method.
"""
for SupplierNumber in range(self.EccMap.datasegments):
PacketID = packetid.MakePacketID(self.backup_id, self.block_number, SupplierNumber, 'Data')
customerID, remotePath = packetid.SplitPacketID(PacketID)
self.OnHandData[SupplierNumber] = bool(os.path.exists(os.path.join(
settings.getLocalBackupsDir(), customerID, remotePath)))
for SupplierNumber in range(self.EccMap.paritysegments):
PacketID = packetid.MakePacketID(self.backup_id, self.block_number, SupplierNumber, 'Parity')
customerID, remotePath = packetid.SplitPacketID(PacketID)
self.OnHandParity[SupplierNumber] = bool(os.path.exists(os.path.join(
settings.getLocalBackupsDir(), customerID, remotePath)))
def doScanExistingPackets(self, arg):
for SupplierNumber in range(self.EccMap.datasegments):
PacketID = packetid.MakePacketID(self.BackupID, self.BlockNumber,
SupplierNumber, 'Data')
customer, remotePath = packetid.SplitPacketID(PacketID)
self.OnHandData[SupplierNumber] = os.path.exists(
os.path.join(settings.getLocalBackupsDir(), customer,
remotePath))
for SupplierNumber in range(self.EccMap.paritysegments):
PacketID = packetid.MakePacketID(self.BackupID, self.BlockNumber,
SupplierNumber, 'Parity')
customer, remotePath = packetid.SplitPacketID(PacketID)
self.OnHandParity[SupplierNumber] = os.path.exists(
os.path.join(settings.getLocalBackupsDir(), customer,
remotePath))
def doBlockPushAndRaid(self, arg):
"""
Action method.
"""
newblock = arg
if self.terminating:
self.automat('block-raid-done', (newblock.BlockNumber, None))
lg.out(_DebugLevel, 'backup.doBlockPushAndRaid SKIP, terminating=True')
return
fileno, filename = tmpfile.make('raid')
serializedblock = newblock.Serialize()
blocklen = len(serializedblock)
os.write(fileno, str(blocklen) + ":" + serializedblock)
os.close(fileno)
self.workBlocks[newblock.BlockNumber] = filename
dt = time.time()
outputpath = os.path.join(settings.getLocalBackupsDir(), self.backupID)
task_params = (filename, self.eccmap.name, self.backupID, newblock.BlockNumber, outputpath)
raid_worker.add_task('make', task_params,
lambda cmd, params, result: self._raidmakeCallback(params, result, dt),)
self.automat('block-raid-started', newblock)
del serializedblock
if _Debug:
lg.out(_DebugLevel, 'backup.doBlockPushAndRaid %s : start process data from %s to %s, %d' % (
newblock.BlockNumber, filename, outputpath, id(self.terminating)))
def QueueRequestFile(self, callOnReceived, creatorID, packetID, ownerID,
remoteID):
# make sure that we don't actually already have the file
# if packetID != settings.BackupInfoFileName():
remoteID = id_url.field(remoteID)
ownerID = id_url.field(ownerID)
creatorID = id_url.field(creatorID)
if packetID not in [
settings.BackupInfoFileName(),
settings.BackupInfoFileNameOld(),
settings.BackupInfoEncryptedFileName(),
]:
customer, pathID = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(), customer,
pathID)
if os.path.exists(filename):
lg.warn("%s already exist " % filename)
if callOnReceived:
reactor.callLater(0, callOnReceived, packetID,
'exist') # @UndefinedVariable
return False
if remoteID not in list(self.supplierQueues.keys()):
# made a new queue for this man
self.supplierQueues[remoteID] = SupplierQueue(
remoteID, self.creatorID)
lg.info("made a new receiving queue for %s" %
nameurl.GetName(remoteID))
# lg.out(10, "io_throttle.QueueRequestFile asking for %s from %s" % (packetID, nameurl.GetName(remoteID)))
return self.supplierQueues[remoteID].SupplierRequestFile(
callOnReceived, creatorID, packetID, ownerID)
def _start_one_block(self):
from storage import backup_matrix
if self.blockIndex < 0:
if _Debug:
lg.out(
_DebugLevel,
'backup_rebuilder._start_one_block finish all blocks blockIndex=%d'
% self.blockIndex)
reactor.callLater(0, self._finish_rebuilding) # @UndefinedVariable
return
BlockNumber = self.workingBlocksQueue[self.blockIndex]
if _Debug:
lg.out(
_DebugLevel,
'backup_rebuilder._start_one_block %d to rebuild, blockIndex=%d, other blocks: %s'
% (
BlockNumber,
self.blockIndex,
str(self.workingBlocksQueue),
))
task_params = (
self.currentBackupID,
BlockNumber,
eccmap.Current().name,
backup_matrix.GetActiveArray(),
backup_matrix.GetRemoteMatrix(self.currentBackupID, BlockNumber),
backup_matrix.GetLocalMatrix(self.currentBackupID, BlockNumber),
settings.getLocalBackupsDir(),
)
raid_worker.add_task(
'rebuild', task_params,
lambda cmd, params, result: self._block_finished(result, params))
def doSavePacket(self, *args, **kwargs):
"""
Action method.
"""
if not args or not args[0]:
raise Exception('no input found')
NewPacket, PacketID = args[0]
glob_path = global_id.ParseGlobalID(PacketID, detect_version=True)
packetID = global_id.CanonicalID(PacketID)
customer_id, _, _, _, SupplierNumber, dataORparity = packetid.SplitFull(packetID)
if dataORparity == 'Data':
self.OnHandData[SupplierNumber] = True
elif dataORparity == 'Parity':
self.OnHandParity[SupplierNumber] = True
if not NewPacket:
lg.warn('packet %r already exists locally' % packetID)
return
filename = os.path.join(settings.getLocalBackupsDir(), customer_id, glob_path['path'])
dirpath = os.path.dirname(filename)
if not os.path.exists(dirpath):
try:
bpio._dirs_make(dirpath)
except:
lg.exc()
# either way the payload of packet is saved
if not bpio.WriteBinaryFile(filename, NewPacket.Payload):
lg.err("unable to write to %s" % filename)
return
if self.packetInCallback is not None:
self.packetInCallback(self.backup_id, NewPacket)
if _Debug:
lg.out(_DebugLevel, "restore_worker.doSavePacket %s saved to %s" % (packetID, filename))
def doSavePacket(self, NewPacket):
glob_path = global_id.ParseGlobalID(NewPacket.PacketID,
detect_version=True)
packetID = global_id.CanonicalID(NewPacket.PacketID)
customer_id, _, _, _, SupplierNumber, dataORparity = packetid.SplitFull(
packetID)
if dataORparity == 'Data':
self.OnHandData[SupplierNumber] = True
elif NewPacket.DataOrParity() == 'Parity':
self.OnHandParity[SupplierNumber] = True
filename = os.path.join(settings.getLocalBackupsDir(), customer_id,
glob_path['path'])
dirpath = os.path.dirname(filename)
if not os.path.exists(dirpath):
try:
bpio._dirs_make(dirpath)
except:
lg.exc()
# either way the payload of packet is saved
if not bpio.WriteFile(filename, NewPacket.Payload):
lg.warn("unable to write to %s" % filename)
return
if self.packetInCallback is not None:
self.packetInCallback(self.BackupID, NewPacket)
lg.out(6, "restore.doSavePacket %s saved to %s" % (packetID, filename))
def report_local_storage():
# TODO
# if customers folder placed outside of BaseDir()
# need to add: total = total + customers
r = {}
r['backups'] = bpio.getDirectorySize(settings.getLocalBackupsDir())
r['backups_str'] = diskspace.MakeStringFromBytes(r['backups'])
r['temp'] = bpio.getDirectorySize(settings.getTempDir())
r['temp_str'] = diskspace.MakeStringFromBytes(r['temp'])
r['customers'] = bpio.getDirectorySize(settings.getCustomersFilesDir())
r['customers_str'] = diskspace.MakeStringFromBytes(r['customers'])
r['total'] = bpio.getDirectorySize(settings.GetBaseDir())
r['total_str'] = diskspace.MakeStringFromBytes(r['total'])
dataDriveFreeSpace, dataDriveTotalSpace = diskusage.GetDriveSpace(settings.getCustomersFilesDir())
if dataDriveFreeSpace is None:
dataDriveFreeSpace = 0
r['disktotal'] = int(dataDriveTotalSpace)
r['disktotal_str'] = diskspace.MakeStringFromBytes(r['disktotal'])
r['diskfree'] = int(dataDriveFreeSpace)
r['diskfree_str'] = diskspace.MakeStringFromBytes(r['diskfree'])
try:
r['total_percent'] = misc.percent2string(float(r['total']) / float(r['disktotal']), 5)
except:
r['total_percent'] = ''
try:
r['diskfree_percent'] = misc.percent2string(float(r['diskfree']) / float(r['disktotal']), 5)
except:
r['diskfree_percent'] = ''
return r
def report_local_storage():
# TODO
# if customers folder placed outside of BaseDir()
# need to add: total = total + customers
r = {}
r['backups'] = bpio.getDirectorySize(settings.getLocalBackupsDir())
# r['backups_str'] = diskspace.MakeStringFromBytes(r['backups'])
r['temp'] = bpio.getDirectorySize(settings.getTempDir())
# r['temp_str'] = diskspace.MakeStringFromBytes(r['temp'])
r['customers'] = bpio.getDirectorySize(settings.getCustomersFilesDir())
# r['customers_str'] = diskspace.MakeStringFromBytes(r['customers'])
r['total'] = bpio.getDirectorySize(settings.BaseDir())
# r['total_str'] = diskspace.MakeStringFromBytes(r['total'])
dataDriveFreeSpace, dataDriveTotalSpace = diskusage.GetDriveSpace(
settings.getCustomersFilesDir())
if dataDriveFreeSpace is None:
dataDriveFreeSpace = 0
r['disktotal'] = int(dataDriveTotalSpace)
# r['disktotal_str'] = diskspace.MakeStringFromBytes(r['disktotal'])
r['diskfree'] = int(dataDriveFreeSpace)
# r['diskfree_str'] = diskspace.MakeStringFromBytes(r['diskfree'])
try:
r['total_percent'] = misc.value2percent(float(r['total']),
float(r['disktotal']), 5)
except:
r['total_percent'] = ''
try:
r['diskfree_percent'] = misc.value2percent(float(r['diskfree']),
float(r['disktotal']), 5)
except:
r['diskfree_percent'] = ''
return r
def _delete(params):
# localPath = params['path'].lstrip('/')
pathID = params['id']
if not packetid.Valid(pathID):
return {
'result': {
"success": False,
"error": "path %s is not valid" % pathID
}
}
if not backup_fs.ExistsID(pathID):
return {
'result': {
"success": False,
"error": "path %s not found" % pathID
}
}
backup_control.DeletePathBackups(pathID, saveDB=False, calculate=False)
backup_fs.DeleteLocalDir(settings.getLocalBackupsDir(), pathID)
backup_fs.DeleteByID(pathID)
backup_fs.Scan()
backup_fs.Calculate()
backup_control.Save()
control.request_update([
('pathID', pathID),
])
backup_monitor.A('restart')
return {'result': {"success": True, "error": None}}
def OnJobDone(backupID, result):
"""
A callback method fired when backup is finished.
Here we need to save the index data base.
"""
from storage import backup_rebuilder
# from customer import io_throttle
lg.info('job done [%s] with result "%s", %d more tasks' %
(backupID, result, len(tasks())))
jobs().pop(backupID)
customerGlobalID, remotePath, version = packetid.SplitBackupID(backupID)
customer_idurl = global_id.GlobalUserToIDURL(customerGlobalID)
if result == 'done':
maxBackupsNum = settings.getBackupsMaxCopies()
if maxBackupsNum:
item = backup_fs.GetByID(remotePath,
iterID=backup_fs.fsID(customer_idurl))
if item:
versions = item.list_versions(sorted=True, reverse=True)
if len(versions) > maxBackupsNum:
for version in versions[maxBackupsNum:]:
item.delete_version(version)
backupID = packetid.MakeBackupID(
customerGlobalID, remotePath, version)
backup_rebuilder.RemoveBackupToWork(backupID)
# io_throttle.DeleteBackupRequests(backupID)
# io_throttle.DeleteBackupSendings(backupID)
# callback.delete_backup_interest(backupID)
backup_fs.DeleteLocalBackup(
settings.getLocalBackupsDir(), backupID)
backup_matrix.EraseBackupLocalInfo(backupID)
backup_matrix.EraseBackupLocalInfo(backupID)
backup_fs.ScanID(remotePath)
backup_fs.Calculate()
Save()
control.request_update([
('pathID', remotePath),
])
# TODO: check used space, if we have over use - stop all tasks immediately
backup_matrix.RepaintBackup(backupID)
elif result == 'abort':
DeleteBackup(backupID)
if len(tasks()) == 0:
# do we really need to restart backup_monitor after each backup?
# if we have a lot tasks started this will produce a lot unneeded actions
# will be smarter to restart it once we finish all tasks
# because user will probably leave BitDust working after starting a long running operations
from storage import backup_monitor
if _Debug:
lg.out(
_DebugLevel,
'backup_control.OnJobDone restarting backup_monitor() machine because no tasks left'
)
backup_monitor.A('restart')
reactor.callLater(0, RunTask) # @UndefinedVariable
reactor.callLater(0, FireTaskFinishedCallbacks, remotePath, version,
result) # @UndefinedVariable
def DeleteBackup(backupID,
removeLocalFilesToo=True,
saveDB=True,
calculate=True):
"""
This removes a single backup ID completely. Perform several operations:
1) abort backup if it just started and is running at the moment
2) if we requested for files for this backup we do not need it anymore - remove 'Data' requests
3) remove interests in transport_control, see ``lib.transport_control.DeleteBackupInterest()``
4) remove that ID from the index data base
5) remove local files for this backup ID
6) remove all remote info for this backup from the memory, see ``p2p.backup_matrix.EraseBackupRemoteInfo()``
7) also remove local info from memory, see ``p2p.backup_matrix.EraseBackupLocalInfo()``
8) stop any rebuilding, we will restart it soon
9) check and calculate used space
10) save the modified index data base, soon it will be synchronized with "index_synchronizer()" state machine
"""
backupID = global_id.CanonicalID(backupID)
# if the user deletes a backup, make sure we remove any work we're doing on it
# abort backup if it just started and is running at the moment
if AbortRunningBackup(backupID):
lg.out(
8, 'backup_control.DeleteBackup %s is in process, stopping' %
backupID)
return True
from customer import io_throttle
from . import backup_rebuilder
lg.out(8, 'backup_control.DeleteBackup ' + backupID)
# if we requested for files for this backup - we do not need it anymore
io_throttle.DeleteBackupRequests(backupID)
io_throttle.DeleteBackupSendings(backupID)
# remove interests in transport_control
# callback.delete_backup_interest(backupID)
# mark it as being deleted in the db, well... just remove it from the index now
if not backup_fs.DeleteBackupID(backupID):
return False
# finally remove local files for this backupID
if removeLocalFilesToo:
backup_fs.DeleteLocalBackup(settings.getLocalBackupsDir(), backupID)
# remove all remote info for this backup from the memory
backup_matrix.EraseBackupRemoteInfo(backupID)
# also remove local info
backup_matrix.EraseBackupLocalInfo(backupID)
# stop any rebuilding, we will restart it soon
backup_rebuilder.RemoveAllBackupsToWork()
backup_rebuilder.SetStoppedFlag()
# check and calculate used space
if calculate:
backup_fs.Scan()
backup_fs.Calculate()
# in some cases we want to save the DB later
if saveDB:
Save()
control.request_update([
('backupID', backupID),
])
return True
def _bk_done(bid, result):
from crypt import signed
try:
os.mkdir(os.path.join(settings.getLocalBackupsDir(), bid + '.out'))
except:
pass
for filename in os.listdir(os.path.join(settings.getLocalBackupsDir(), bid)):
filepath = os.path.join(settings.getLocalBackupsDir(), bid, filename)
payld = str(bpio.ReadBinaryFile(filepath))
newpacket = signed.Packet(
'Data',
my_id.getLocalID(),
my_id.getLocalID(),
filename,
payld,
'http://megafaq.ru/cvps1010.xml')
newfilepath = os.path.join(settings.getLocalBackupsDir(), bid + '.out', filename)
bpio.AtomicWriteFile(newfilepath, newpacket.Serialize())
reactor.stop()
def _bk_done(bid, result):
from crypt import signed
customer, remotePath = packetid.SplitPacketID(bid)
try:
os.mkdir(os.path.join(settings.getLocalBackupsDir(), customer, remotePath + '.out'))
except:
pass
for filename in os.listdir(os.path.join(settings.getLocalBackupsDir(), customer, remotePath)):
filepath = os.path.join(settings.getLocalBackupsDir(), customer, remotePath, filename)
payld = bpio.ReadBinaryFile(filepath)
newpacket = signed.Packet(
'Data',
my_id.getLocalID(),
my_id.getLocalID(),
filename,
payld,
'http://megafaq.ru/cvps1010.xml')
newfilepath = os.path.join(settings.getLocalBackupsDir(), customer, remotePath + '.out', filename)
bpio.WriteBinaryFile(newfilepath, newpacket.Serialize())
reactor.stop()
def ReadLocalFiles():
"""
This method scans local backups and build the whole "local" matrix.
"""
global _LocalFilesNotifyCallback
local_files().clear()
local_max_block_numbers().clear()
local_backup_size().clear()
_counter = [0]
def visit(realpath, subpath, name):
# subpath is something like 0/0/1/0/F20131120053803PM/0-1-Data
if not os.path.isfile(realpath):
return True
if realpath.startswith("newblock-"):
return False
if subpath in [
settings.BackupIndexFileName(),
settings.BackupInfoFileName(),
settings.BackupInfoFileNameOld(),
settings.BackupInfoEncryptedFileName(),
]:
return False
try:
version = subpath.split("/")[-2]
except:
return False
if not packetid.IsCanonicalVersion(version):
return True
LocalFileReport(packetID=subpath)
_counter[0] += 1
return False
bpio.traverse_dir_recursive(visit, settings.getLocalBackupsDir())
lg.out(8, "backup_matrix.ReadLocalFiles %d files indexed" % _counter[0])
if lg.is_debug(8):
try:
if sys.version_info >= (2, 6):
# localSZ = sys.getsizeof(local_files())
# remoteSZ = sys.getsizeof(remote_files())
import lib.getsizeof
localSZ = lib.getsizeof.total_size(local_files())
remoteSZ = lib.getsizeof.total_size(remote_files())
indexByName = lib.getsizeof.total_size(backup_fs.fs())
indexByID = lib.getsizeof.total_size(backup_fs.fsID())
lg.out(10, " all local info uses %d bytes in the memory" % localSZ)
lg.out(10, " all remote info uses %d bytes in the memory" % remoteSZ)
lg.out(10, " index by name takes %d bytes in the memory" % indexByName)
lg.out(10, " index by ID takes %d bytes in the memory" % indexByID)
except:
lg.exc()
if _LocalFilesNotifyCallback is not None:
_LocalFilesNotifyCallback()
def doRemoveTempFile(self, *args, **kwargs):
"""
Action method.
"""
if not args or not len(args) > 1:
return
filename = args[1]
if filename:
tmpfile.throw_out(filename, 'block restored')
if settings.getBackupsKeepLocalCopies():
return
from storage import backup_matrix
from storage import backup_rebuilder
if not backup_rebuilder.ReadStoppedFlag():
if backup_rebuilder.A().currentBackupID is not None:
if backup_rebuilder.A().currentBackupID == self.backup_id:
if _Debug:
lg.out(
_DebugLevel,
'restore_worker.doRemoveTempFile SKIP because rebuilding in process'
)
return
count = 0
for supplierNum in range(
contactsdb.num_suppliers(customer_idurl=self.customer_idurl)):
supplierIDURL = contactsdb.supplier(
supplierNum, customer_idurl=self.customer_idurl)
if not supplierIDURL:
continue
for dataORparity in [
'Data',
'Parity',
]:
packetID = packetid.MakePacketID(self.backup_id,
self.block_number,
supplierNum, dataORparity)
customer, remotePath = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(),
customer, remotePath)
if os.path.isfile(filename):
try:
os.remove(filename)
except:
lg.exc()
continue
count += 1
backup_matrix.LocalBlockReport(self.backup_id, self.block_number,
*args, **kwargs)
if _Debug:
lg.out(
_DebugLevel,
'restore_worker.doRemoveTempFile %d files were removed' %
count)
def DeletePathBackups(pathID,
removeLocalFilesToo=True,
saveDB=True,
calculate=True):
"""
This removes all backups of given path ID
Doing same operations as ``DeleteBackup()``.
"""
from . import backup_rebuilder
from customer import io_throttle
pathID = global_id.CanonicalID(pathID)
# get the working item
customer, remotePath = packetid.SplitPacketID(pathID)
customer_idurl = global_id.GlobalUserToIDURL(customer)
item = backup_fs.GetByID(remotePath, iterID=backup_fs.fsID(customer_idurl))
if item is None:
return False
lg.out(8, 'backup_control.DeletePathBackups ' + pathID)
# this is a list of all known backups of this path
versions = item.list_versions()
for version in versions:
backupID = packetid.MakeBackupID(customer, remotePath, version)
lg.out(8, ' removing %s' % backupID)
# abort backup if it just started and is running at the moment
AbortRunningBackup(backupID)
# if we requested for files for this backup - we do not need it anymore
io_throttle.DeleteBackupRequests(backupID)
io_throttle.DeleteBackupSendings(backupID)
# remove interests in transport_control
# callback.delete_backup_interest(backupID)
# remove local files for this backupID
if removeLocalFilesToo:
backup_fs.DeleteLocalBackup(settings.getLocalBackupsDir(),
backupID)
# remove remote info for this backup from the memory
backup_matrix.EraseBackupRemoteInfo(backupID)
# also remove local info
backup_matrix.EraseBackupLocalInfo(backupID)
# finally remove this backup from the index
item.delete_version(version)
# lg.out(8, 'backup_control.DeletePathBackups ' + backupID)
# stop any rebuilding, we will restart it soon
backup_rebuilder.RemoveAllBackupsToWork()
backup_rebuilder.SetStoppedFlag()
# check and calculate used space
if calculate:
backup_fs.Scan()
backup_fs.Calculate()
# save the index if needed
if saveDB:
Save()
control.request_update()
return True
def doReadRaid(self, *args, **kwargs):
"""
Action method.
"""
_, outfilename = tmpfile.make(
'restore',
extension='.raid',
prefix=self.backup_id.replace(':', '_').replace('@', '_').replace('/', '_') + '_' + str(self.block_number) + '_',
close_fd=True,
)
inputpath = os.path.join(settings.getLocalBackupsDir(), self.customer_id, self.path_id)
task_params = (outfilename, self.EccMap.name, self.version, self.block_number, inputpath)
raid_worker.add_task('read', task_params, lambda cmd, params, result: self._on_block_restored(result, outfilename))
def DeleteBackup(backupID, removeLocalFilesToo=True, saveDB=True, calculate=True):
"""
This removes a single backup ID completely. Perform several operations:
1) abort backup if it just started and is running at the moment
2) if we requested for files for this backup we do not need it anymore - remove 'Data' requests
3) remove interests in transport_control, see ``lib.transport_control.DeleteBackupInterest()``
4) remove that ID from the index data base
5) remove local files for this backup ID
6) remove all remote info for this backup from the memory, see ``p2p.backup_matrix.EraseBackupRemoteInfo()``
7) also remove local info from memory, see ``p2p.backup_matrix.EraseBackupLocalInfo()``
8) stop any rebuilding, we will restart it soon
9) check and calculate used space
10) save the modified index data base, soon it will be synchronized with "index_synchronizer()" state machine
"""
# if the user deletes a backup, make sure we remove any work we're doing on it
# abort backup if it just started and is running at the moment
if AbortRunningBackup(backupID):
lg.out(8, 'backup_control.DeleteBackup %s is in process, stopping' % backupID)
return True
from customer import io_throttle
import backup_rebuilder
lg.out(8, 'backup_control.DeleteBackup ' + backupID)
# if we requested for files for this backup - we do not need it anymore
io_throttle.DeleteBackupRequests(backupID)
io_throttle.DeleteBackupSendings(backupID)
# remove interests in transport_control
callback.delete_backup_interest(backupID)
# mark it as being deleted in the db, well... just remove it from the index now
if not backup_fs.DeleteBackupID(backupID):
return False
# finally remove local files for this backupID
if removeLocalFilesToo:
backup_fs.DeleteLocalBackup(settings.getLocalBackupsDir(), backupID)
# remove all remote info for this backup from the memory
backup_matrix.EraseBackupRemoteInfo(backupID)
# also remove local info
backup_matrix.EraseBackupLocalInfo(backupID)
# stop any rebuilding, we will restart it soon
backup_rebuilder.RemoveAllBackupsToWork()
backup_rebuilder.SetStoppedFlag()
# check and calculate used space
if calculate:
backup_fs.Scan()
backup_fs.Calculate()
# in some cases we want to save the DB later
if saveDB:
Save()
control.request_update([('backupID', backupID), ])
return True
def LocalFileReport(packetID=None, backupID=None, blockNum=None, supplierNum=None, dataORparity=None):
"""
Writes info for a single piece of data into "local" matrix.
You can use two forms:
* pass ``packetID`` parameter only
* pass all other parameters and do not use ``packetID``
This is called when new local file created, for example during rebuilding process.
"""
if packetID is not None:
backupID, blockNum, supplierNum, dataORparity = packetid.Split(packetID)
if backupID is None:
lg.warn("incorrect filename: " + packetID)
return
else:
blockNum = int(blockNum)
supplierNum = int(supplierNum)
dataORparity = dataORparity
packetID = packetid.MakePacketID(backupID, blockNum, supplierNum, dataORparity)
filename = packetID
if dataORparity not in ["Data", "Parity"]:
lg.warn("Data or Parity? " + filename)
return
if supplierNum >= contactsdb.num_suppliers():
# lg.warn('supplier number? %d > %d : %s' % (supplierNum, contactsdb.num_suppliers(), filename))
return
localDest = os.path.join(settings.getLocalBackupsDir(), filename)
if backupID not in local_files():
local_files()[backupID] = {}
if blockNum not in local_files()[backupID]:
local_files()[backupID][blockNum] = {
"D": [0] * contactsdb.num_suppliers(),
"P": [0] * contactsdb.num_suppliers(),
}
if not os.path.isfile(localDest):
local_files()[backupID][blockNum][dataORparity[0]][supplierNum] = 0
return
local_files()[backupID][blockNum][dataORparity[0]][supplierNum] = 1
if backupID not in local_max_block_numbers():
local_max_block_numbers()[backupID] = -1
if local_max_block_numbers()[backupID] < blockNum:
local_max_block_numbers()[backupID] = blockNum
if backupID not in local_backup_size():
local_backup_size()[backupID] = 0
try:
local_backup_size()[backupID] += os.path.getsize(localDest)
except:
lg.exc()
RepaintBackup(backupID)
def LocalBlockReport(backupID, blockNumber, result):
"""
This updates "local" matrix - a several pieces corresponding to given block of data.
"""
# if contactsdb.num_suppliers() != num_suppliers:
# lg.out(6, 'backup_matrix.LocalBlockReport %s skipped, because number of suppliers were changed' % str(newblock))
# return
if result is None:
lg.warn("result is None")
return
try:
blockNum = int(blockNumber)
except:
lg.exc()
return
repaint_flag = False
for supplierNum in xrange(contactsdb.num_suppliers()):
for dataORparity in ("Data", "Parity"):
packetID = packetid.MakePacketID(backupID, blockNum, supplierNum, dataORparity)
local_file = os.path.join(settings.getLocalBackupsDir(), packetID)
if backupID not in local_files():
local_files()[backupID] = {}
repaint_flag = True
# lg.out(14, 'backup_matrix.LocalFileReport new local entry for %s created in the memory' % backupID)
if blockNum not in local_files()[backupID]:
local_files()[backupID][blockNum] = {
"D": [0] * contactsdb.num_suppliers(),
"P": [0] * contactsdb.num_suppliers(),
}
repaint_flag = True
if not os.path.isfile(local_file):
local_files()[backupID][blockNum][dataORparity[0]][supplierNum] = 0
repaint_flag = True
continue
local_files()[backupID][blockNum][dataORparity[0]][supplierNum] = 1
# lg.out(6, 'backup_matrix.LocalFileReport %s max block num is %d' % (backupID, local_max_block_numbers()[backupID]))
if backupID not in local_backup_size():
local_backup_size()[backupID] = 0
repaint_flag = True
try:
local_backup_size()[backupID] += os.path.getsize(local_file)
repaint_flag = True
except:
lg.exc()
if backupID not in local_max_block_numbers():
local_max_block_numbers()[backupID] = -1
if local_max_block_numbers()[backupID] < blockNum:
local_max_block_numbers()[backupID] = blockNum
if repaint_flag:
RepaintBackup(backupID)
def doReadRaid(self, arg):
fd, outfilename = tmpfile.make(
'restore',
prefix=self.BackupID.replace(':', '_').replace('@', '_').replace(
'/', '_') + '_' + str(self.BlockNumber) + '_',
)
os.close(fd)
inputpath = os.path.join(settings.getLocalBackupsDir(),
self.CustomerGlobalID, self.PathID)
task_params = (outfilename, eccmap.CurrentName(), self.Version,
self.BlockNumber, inputpath)
raid_worker.add_task(
'read', task_params,
lambda cmd, params, result: self._on_block_restored(
result, outfilename))
def doBlockPushAndRaid(self, arg):
"""
Action method.
"""
newblock = arg
if newblock is None:
self.abort()
self.automat('fail')
lg.out(
_DebugLevel,
'backup.doBlockPushAndRaid ERROR newblock is empty, terminating=%s'
% self.terminating)
lg.warn('failed to encrypt block, ABORTING')
return
if self.terminating:
self.automat('block-raid-done', (newblock.BlockNumber, None))
lg.out(_DebugLevel,
'backup.doBlockPushAndRaid SKIP, terminating=True')
return
fileno, filename = tmpfile.make('raid')
serializedblock = newblock.Serialize()
blocklen = len(serializedblock)
os.write(fileno, str(blocklen) + ":" + serializedblock)
os.close(fileno)
self.workBlocks[newblock.BlockNumber] = filename
# key_alias = 'master'
# if self.keyID:
# key_alias = packetid.KeyAlias(self.keyID)
dt = time.time()
customer_dir = self.customerGlobalID # global_id.MakeGlobalID(customer=self.customerGlobalID, key_alias=key_alias)
outputpath = os.path.join(settings.getLocalBackupsDir(), customer_dir,
self.pathID, self.version)
task_params = (filename, self.eccmap.name, self.version,
newblock.BlockNumber, outputpath)
raid_worker.add_task(
'make',
task_params,
lambda cmd, params, result: self._raidmakeCallback(
params, result, dt),
)
self.automat('block-raid-started', newblock)
del serializedblock
if _Debug:
lg.out(
_DebugLevel,
'backup.doBlockPushAndRaid %s : start process data from %s to %s, %d'
% (newblock.BlockNumber, filename, outputpath,
id(self.terminating)))
def _delete(params):
# localPath = params['path'].lstrip('/')
pathID = params['id']
if not packetid.Valid(pathID):
return {'result': {"success": False, "error": "path %s is not valid" % pathID}}
if not backup_fs.ExistsID(pathID):
return {'result': {"success": False, "error": "path %s not found" % pathID}}
backup_control.DeletePathBackups(pathID, saveDB=False, calculate=False)
backup_fs.DeleteLocalDir(settings.getLocalBackupsDir(), pathID)
backup_fs.DeleteByID(pathID)
backup_fs.Scan()
backup_fs.Calculate()
backup_control.Save()
control.request_update([('pathID', pathID), ])
backup_monitor.A('restart')
return {'result': {"success": True, "error": None}}
def main():
sourcePath = sys.argv[1]
backupID = sys.argv[2]
lg.set_debug_level(24)
compress_mode = 'none' # 'gz'
raid_worker.A('init')
backupPath = backup_fs.MakeLocalDir(settings.getLocalBackupsDir(), backupID)
if bpio.pathIsDir(sourcePath):
backupPipe = backup_tar.backuptar(sourcePath, compress=compress_mode)
else:
backupPipe = backup_tar.backuptarfile(sourcePath, compress=compress_mode)
backupPipe.make_nonblocking()
job = backup.backup(backupID, backupPipe, backup_done)
reactor.callLater(1, job.automat, 'start')
reactor.run()
def RunRequest(self):
packetsToRemove = {}
for i in range(
0, min(self.fileRequestMaxLength, len(self.fileRequestQueue))):
packetID = self.fileRequestQueue[i]
# must never happen, but just in case
if packetID not in self.fileRequestDict:
packetsToRemove[packetID] = 'broken'
lg.err('file %r not found in downloading queue for %r' %
(packetID, self.remoteID))
continue
f_down = self.fileRequestDict[packetID]
if f_down.state == 'IN_QUEUE':
customer, pathID = packetid.SplitPacketID(packetID)
if os.path.exists(
os.path.join(settings.getLocalBackupsDir(), customer,
pathID)):
# we have the data file, no need to request it
packetsToRemove[packetID] = 'exist'
else:
f_down.event('start')
# remember requests results
result = len(packetsToRemove)
# remove finished requests
for packetID, why in packetsToRemove.items():
if _Debug:
lg.out(
_DebugLevel,
"io_throttle.RunRequest %r to be removed from [%s] downloading queue because %r, %d more items"
%
(packetID, self.remoteID, why, len(self.fileRequestQueue)))
if packetID in self.fileRequestQueue:
f_down = self.fileRequestDict[packetID]
if why == 'exist':
f_down.event('file-already-exists')
else:
lg.warn(
'unexpected result "%r" for %r in downloading queue for %s'
% (why, packetID, self.remoteID))
f_down.event('stop')
else:
lg.warn('packet %r not found in request queue for [%s]' %
(packetID, self.remoteID))
del packetsToRemove
if result:
self.DoRequest()
return result
def OnJobDone(backupID, result):
"""
A callback method fired when backup is finished.
Here we need to save the index data base.
"""
import backup_rebuilder
from customer import io_throttle
lg.out(4, '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
lg.out(4, 'backup_control.OnJobDone [%s] %s, %d more tasks' % (backupID, result, len(tasks())))
jobs().pop(backupID)
pathID, version = packetid.SplitBackupID(backupID)
if result == 'done':
maxBackupsNum = settings.getBackupsMaxCopies()
if maxBackupsNum:
item = backup_fs.GetByID(pathID)
if item:
versions = item.list_versions(sorted=True, reverse=True)
if len(versions) > maxBackupsNum:
for version in versions[maxBackupsNum:]:
item.delete_version(version)
backupID = pathID + '/' + version
backup_rebuilder.RemoveBackupToWork(backupID)
io_throttle.DeleteBackupRequests(backupID)
io_throttle.DeleteBackupSendings(backupID)
callback.delete_backup_interest(backupID)
backup_fs.DeleteLocalBackup(settings.getLocalBackupsDir(), backupID)
backup_matrix.EraseBackupLocalInfo(backupID)
backup_matrix.EraseBackupLocalInfo(backupID)
backup_fs.ScanID(pathID)
backup_fs.Calculate()
Save()
control.request_update([('pathID', pathID), ])
# TODO: check used space, if we have over use - stop all tasks immediately
backup_matrix.RepaintBackup(backupID)
elif result == 'abort':
DeleteBackup(backupID)
if len(tasks()) == 0:
# do we really need to restart backup_monitor after each backup?
# if we have a lot tasks started this will produce a lot unneeded actions
# will be smarter to restart it once we finish all tasks
# because user will probably leave BitDust working after starting a long running operations
from storage import backup_monitor
backup_monitor.A('restart')
RunTasks()
reactor.callLater(0, FireTaskFinishedCallbacks, pathID, version, result)
def backup_done(bid, result):
from crypt import signed
try:
os.mkdir(os.path.join(settings.getLocalBackupsDir(), bid + '.out'))
except:
pass
for filename in os.listdir(os.path.join(settings.getLocalBackupsDir(),
bid)):
filepath = os.path.join(settings.getLocalBackupsDir(), bid, filename)
payld = str(bpio.ReadBinaryFile(filepath))
outpacket = signed.Packet('Data', my_id.getLocalID(),
my_id.getLocalID(), filename, payld,
'http://megafaq.ru/cvps1010.xml')
newfilepath = os.path.join(settings.getLocalBackupsDir(), bid + '.out',
filename)
bpio.WriteBinaryFile(newfilepath, outpacket.Serialize())
# Assume we delivered all pieces from ".out" to suppliers and lost original data
# Then we requested the data back and got it into ".inp"
try:
os.mkdir(os.path.join(settings.getLocalBackupsDir(), bid + '.inp'))
except:
pass
for filename in os.listdir(
os.path.join(settings.getLocalBackupsDir(), bid + '.out')):
filepath = os.path.join(settings.getLocalBackupsDir(), bid + '.out',
filename)
data = bpio.ReadBinaryFile(filepath)
inppacket = signed.Unserialize(data)
assert inppacket
assert inppacket.Valid()
newfilepath = os.path.join(settings.getLocalBackupsDir(), bid + '.inp',
filename)
bpio.WriteBinaryFile(newfilepath, inppacket.Payload)
# Now do restore from input data
backupID = bid + '.inp'
outfd, tarfilename = tmpfile.make(
'restore',
extension='.tar.gz',
prefix=backupID.replace('/', '_') + '_',
)
r = restore_worker.RestoreWorker(backupID, outfd)
r.MyDeferred.addBoth(restore_done, tarfilename)
reactor.callLater(1, r.automat, 'init')
def main():
from system import bpio
from . import backup_tar
from . import backup_fs
lg.set_debug_level(24)
sourcePath = sys.argv[1]
compress_mode = 'none' # 'gz'
backupID = sys.argv[2]
raid_worker.A('init')
backupPath = backup_fs.MakeLocalDir(settings.getLocalBackupsDir(),
backupID)
if bpio.pathIsDir(sourcePath):
backupPipe = backup_tar.backuptardir(sourcePath,
compress=compress_mode)
else:
backupPipe = backup_tar.backuptarfile(sourcePath,
compress=compress_mode)
backupPipe.make_nonblocking()
def _bk_done(bid, result):
from crypt import signed
customer, remotePath = packetid.SplitPacketID(bid)
try:
os.mkdir(
os.path.join(settings.getLocalBackupsDir(), customer,
remotePath + '.out'))
except:
pass
for filename in os.listdir(
os.path.join(settings.getLocalBackupsDir(), customer,
remotePath)):
filepath = os.path.join(settings.getLocalBackupsDir(), customer,
remotePath, filename)
payld = bpio.ReadBinaryFile(filepath)
newpacket = signed.Packet('Data', my_id.getLocalID(),
my_id.getLocalID(), filename, payld,
'http://megafaq.ru/cvps1010.xml')
newfilepath = os.path.join(settings.getLocalBackupsDir(), customer,
remotePath + '.out', filename)
bpio.WriteBinaryFile(newfilepath, newpacket.Serialize())
reactor.stop() # @UndefinedVariable
job = backup(backupID, backupPipe, _bk_done)
reactor.callLater(1, job.automat, 'start') # @UndefinedVariable
reactor.run() # @UndefinedVariable
def doBlockPushAndRaid(self, *args, **kwargs):
"""
Action method.
"""
newblock = args[0]
if newblock is None:
self.abort()
self.automat('fail')
if _Debug:
lg.out(
_DebugLevel,
'backup.doBlockPushAndRaid ERROR newblock is empty, terminating=%s'
% self.terminating)
lg.warn('failed to encrypt block, ABORTING')
return
if self.terminating:
self.automat('block-raid-done', (newblock.BlockNumber, None))
if _Debug:
lg.out(_DebugLevel,
'backup.doBlockPushAndRaid SKIP, terminating=True')
return
fileno, filename = tmpfile.make('raid', extension='.raid')
serializedblock = newblock.Serialize()
blocklen = len(serializedblock)
os.write(fileno, strng.to_bin(blocklen) + b":" + serializedblock)
os.close(fileno)
self.workBlocks[newblock.BlockNumber] = filename
dt = time.time()
outputpath = os.path.join(settings.getLocalBackupsDir(),
self.customerGlobalID, self.pathID,
self.version)
task_params = (filename, self.eccmap.name, self.version,
newblock.BlockNumber, outputpath)
raid_worker.add_task(
'make', task_params,
lambda cmd, params, result: self._raidmakeCallback(
params, result, dt))
self.automat('block-raid-started', newblock)
del serializedblock
if _Debug:
lg.out(
_DebugLevel,
'backup.doBlockPushAndRaid %s : start process data from %s to %s, %d'
% (newblock.BlockNumber, filename, outputpath,
id(self.terminating)))
def DeletePathBackups(pathID, removeLocalFilesToo=True, saveDB=True, calculate=True):
"""
This removes all backups of given path ID.
Doing same operations as ``DeleteBackup()``.
"""
import backup_rebuilder
from customer import io_throttle
# get the working item
item = backup_fs.GetByID(pathID)
if item is None:
return False
# this is a list of all known backups of this path
versions = item.list_versions()
for version in versions:
backupID = pathID + '/' + version
# abort backup if it just started and is running at the moment
AbortRunningBackup(backupID)
# if we requested for files for this backup - we do not need it anymore
io_throttle.DeleteBackupRequests(backupID)
io_throttle.DeleteBackupSendings(backupID)
# remove interests in transport_control
callback.delete_backup_interest(backupID)
# remove local files for this backupID
if removeLocalFilesToo:
backup_fs.DeleteLocalBackup(settings.getLocalBackupsDir(), backupID)
# remove remote info for this backup from the memory
backup_matrix.EraseBackupLocalInfo(backupID)
# also remove local info
backup_matrix.EraseBackupLocalInfo(backupID)
# finally remove this backup from the index
item.delete_version(version)
# lg.out(8, 'backup_control.DeletePathBackups ' + backupID)
# stop any rebuilding, we will restart it soon
backup_rebuilder.RemoveAllBackupsToWork()
backup_rebuilder.SetStoppedFlag()
# check and calculate used space
if calculate:
backup_fs.Scan()
backup_fs.Calculate()
# save the index if needed
if saveDB:
Save()
control.request_update()
return True
def _file_received(self, newpacket, state):
if state in ['in queue', 'shutdown', 'exist', 'failed']:
return
if state != 'received':
lg.warn("incorrect state [%s] for packet %s" %
(str(state), str(newpacket)))
return
if not newpacket.Valid():
# TODO: if we didn't get a valid packet ... re-request it or delete
# it?
lg.warn("%s is not a valid packet: %r" %
(newpacket.PacketID, newpacket))
return
# packetID = newpacket.PacketID
packetID = global_id.CanonicalID(newpacket.PacketID)
customer, remotePath = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(), customer,
remotePath)
if os.path.isfile(filename):
lg.warn("found existed file" + filename)
self.automat('inbox-data-packet', packetID)
return
# try:
# os.remove(filename)
# except:
# lg.exc()
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
bpio._dirs_make(dirname)
except:
lg.out(
2,
"backup_rebuilder._file_received ERROR can not create sub dir "
+ dirname)
return
if not bpio.WriteFile(filename, newpacket.Payload):
lg.out(2,
"backup_rebuilder._file_received ERROR writing " + filename)
return
from storage import backup_matrix
backup_matrix.LocalFileReport(packetID)
lg.out(10, "backup_rebuilder._file_received and wrote to " + filename)
self.automat('inbox-data-packet', packetID)
def main():
sourcePath = sys.argv[1]
backupID = sys.argv[2]
lg.set_debug_level(24)
compress_mode = 'none' # 'gz'
raid_worker.A('init')
backupPath = backup_fs.MakeLocalDir(settings.getLocalBackupsDir(),
backupID)
if bpio.pathIsDir(sourcePath):
backupPipe = backup_tar.backuptardir(sourcePath,
compress=compress_mode)
else:
backupPipe = backup_tar.backuptarfile(sourcePath,
compress=compress_mode)
backupPipe.make_nonblocking()
job = backup.backup(backupID, backupPipe, backup_done)
reactor.callLater(1, job.automat, 'start')
reactor.run()
def backup_done(bid, result):
from crypt import signed
try:
os.mkdir(os.path.join(settings.getLocalBackupsDir(), bid + '.out'))
except:
pass
for filename in os.listdir(os.path.join(settings.getLocalBackupsDir(), bid)):
filepath = os.path.join(settings.getLocalBackupsDir(), bid, filename)
payld = str(bpio.ReadBinaryFile(filepath))
outpacket = signed.Packet(
'Data',
my_id.getLocalID(),
my_id.getLocalID(),
filename,
payld,
'http://megafaq.ru/cvps1010.xml')
newfilepath = os.path.join(settings.getLocalBackupsDir(),
bid + '.out', filename)
bpio.AtomicWriteFile(newfilepath, outpacket.Serialize())
# Assume we delivered all pieces from ".out" to suppliers and lost original data
# Then we requested the data back and got it into ".inp"
try:
os.mkdir(os.path.join(settings.getLocalBackupsDir(), bid + '.inp'))
except:
pass
for filename in os.listdir(os.path.join(settings.getLocalBackupsDir(), bid + '.out')):
filepath = os.path.join(settings.getLocalBackupsDir(), bid + '.out', filename)
data = bpio.ReadBinaryFile(filepath)
inppacket = signed.Unserialize(data)
assert inppacket.Valid()
newfilepath = os.path.join(settings.getLocalBackupsDir(),
bid + '.inp', filename)
bpio.AtomicWriteFile(newfilepath, inppacket.Payload)
# Now do restore from input data
backupID = bid + '.inp'
outfd, tarfilename = tmpfile.make('restore', '.tar.gz', backupID.replace('/', '_') + '_')
r = restore.restore(backupID, outfd)
r.MyDeferred.addBoth(restore_done, tarfilename)
reactor.callLater(1, r.automat, 'init')
def doRemoveTempFile(self, arg):
try:
filename = arg[1]
except:
return
tmpfile.throw_out(filename, 'block restored')
if settings.getBackupsKeepLocalCopies():
return
import backup_rebuilder
import backup_matrix
if not backup_rebuilder.ReadStoppedFlag():
if backup_rebuilder.A().currentBackupID is not None:
if backup_rebuilder.A().currentBackupID == self.BackupID:
lg.out(
6,
'restore.doRemoveTempFile SKIP because rebuilding in process'
)
return
count = 0
for supplierNum in xrange(
contactsdb.num_suppliers(customer_idurl=self.CustomerIDURL)):
supplierIDURL = contactsdb.supplier(
supplierNum, customer_idurl=self.CustomerIDURL)
if not supplierIDURL:
continue
for dataORparity in ['Data', 'Parity']:
packetID = packetid.MakePacketID(self.BackupID,
self.BlockNumber, supplierNum,
dataORparity)
customer, remotePath = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(),
customer, remotePath)
if os.path.isfile(filename):
try:
os.remove(filename)
except:
lg.exc()
continue
count += 1
backup_matrix.LocalBlockReport(self.BackupID, self.BlockNumber, arg)
lg.out(6, 'restore.doRemoveTempFile %d files were removed' % count)
def _do_start_archive_backup(self):
local_path = self.local_data_callback(self.queue_id,
self.latest_sequence_id)
supplier_path_id = os.path.join(self.archive_folder_path,
strng.to_text(self.latest_sequence_id))
dataID = misc.NewBackupID()
backup_id = packetid.MakeBackupID(
customer=self.queue_owner_id,
path_id=supplier_path_id,
key_alias=self.queue_alias,
version=dataID,
)
backup_fs.MakeLocalDir(settings.getLocalBackupsDir(), backup_id)
if bpio.Android():
compress_mode = 'none'
else:
compress_mode = 'bz2'
arcname = os.path.basename(local_path)
backupPipe = backup_tar.backuptarfile_thread(local_path,
arcname=arcname,
compress=compress_mode)
self.backup_job = backup.backup(
backupID=backup_id,
pipe=backupPipe,
blockResultCallback=self._on_archive_backup_block_result,
finishCallback=self._on_archive_backup_done,
blockSize=1024 * 1024 * 10,
sourcePath=local_path,
keyID=self.group_key_id,
ecc_map=eccmap.eccmap(self.ecc_map),
creatorIDURL=self.queue_owner_idurl,
)
self.backup_job.automat('start')
if _Debug:
lg.args(_DebugLevel,
job=self.backup_job,
backup_id=backup_id,
local_path=local_path,
group_key_id=self.group_key_id)
def main():
from system import bpio
import backup_tar
import backup_fs
lg.set_debug_level(24)
sourcePath = sys.argv[1]
compress_mode = 'none' # 'gz'
backupID = sys.argv[2]
raid_worker.A('init')
backupPath = backup_fs.MakeLocalDir(settings.getLocalBackupsDir(), backupID)
if bpio.pathIsDir(sourcePath):
backupPipe = backup_tar.backuptar(sourcePath, compress=compress_mode)
else:
backupPipe = backup_tar.backuptarfile(sourcePath, compress=compress_mode)
backupPipe.make_nonblocking()
def _bk_done(bid, result):
from crypt import signed
try:
os.mkdir(os.path.join(settings.getLocalBackupsDir(), bid + '.out'))
except:
pass
for filename in os.listdir(os.path.join(settings.getLocalBackupsDir(), bid)):
filepath = os.path.join(settings.getLocalBackupsDir(), bid, filename)
payld = str(bpio.ReadBinaryFile(filepath))
newpacket = signed.Packet(
'Data',
my_id.getLocalID(),
my_id.getLocalID(),
filename,
payld,
'http://megafaq.ru/cvps1010.xml')
newfilepath = os.path.join(settings.getLocalBackupsDir(), bid + '.out', filename)
bpio.AtomicWriteFile(newfilepath, newpacket.Serialize())
reactor.stop()
job = backup(backupID, backupPipe, _bk_done)
reactor.callLater(1, job.automat, 'start')
reactor.run()
def __init__(self):
self.installer_state_to_page = {
'AT_STARTUP': self.renderSelectPage,
'WHAT_TO_DO?': self.renderSelectPage,
'INPUT_NAME': self.renderInputNamePage,
'REGISTER': self.renderRegisterNewUserPage,
'AUTHORIZED': self.renderRegisterNewUserPage,
'LOAD_KEY': self.renderLoadKeyPage,
'RECOVER': self.renderRestorePage,
'RESTORED': self.renderRestorePage,
'WIZARD': self.renderWizardPage,
'DONE': self.renderLastPage,
}
self.install_wizard_state_to_page = {
'READY': self.renderWizardStartPage,
'STORAGE': self.renderWizardStoragePage,
'CONTACTS': self.renderWizardContactsPage,
'LAST_PAGE': self.renderLastPage,
'DONE': self.renderLastPage,
}
self.data = {
'username': bpio.ReadTextFile(settings.UserNameFilename()).strip(),
'pksize': settings.DefaultPrivateKeySize(),
'needed': str(int(settings.DefaultNeededBytes() / (1024 * 1024))),
'donated': str(int(settings.DefaultDonatedBytes() / (1024 * 1024))),
'suppliers': str(settings.DefaultDesiredSuppliers()),
'customersdir': unicode(settings.getCustomersFilesDir()),
'localbackupsdir': unicode(settings.getLocalBackupsDir()),
'restoredir': unicode(settings.getRestoreDir()),
'idurl': '',
'keysrc': '',
'name': '',
'surname': '',
'nickname': '',
}
installer.A('init')
def doRemoveUnusedFiles(self, arg):
# we want to remove files for this block
# because we only need them during rebuilding
if settings.getBackupsKeepLocalCopies() is True:
# if user set this in settings - he want to keep the local files
return
# ... user do not want to keep local backups
if settings.getGeneralWaitSuppliers() is True:
from customer import fire_hire
# but he want to be sure - all suppliers are green for a long time
if len(contact_status.listOfflineSuppliers(
)) > 0 or time.time() - fire_hire.GetLastFireTime() < 24 * 60 * 60:
# some people are not there or we do not have stable team yet
# do not remove the files because we need it to rebuild
return
count = 0
from storage import backup_matrix
from storage import restore_monitor
from storage import backup_rebuilder
if _Debug:
lg.out(_DebugLevel, 'data_sender.doRemoveUnusedFiles')
for backupID in misc.sorted_backup_ids(
backup_matrix.local_files().keys()):
if restore_monitor.IsWorking(backupID):
if _Debug:
lg.out(_DebugLevel,
' %s : SKIP, because restoring' % backupID)
continue
if backup_rebuilder.IsBackupNeedsWork(backupID):
if _Debug:
lg.out(
_DebugLevel,
' %s : SKIP, because needs rebuilding' %
backupID)
continue
if not backup_rebuilder.ReadStoppedFlag():
if backup_rebuilder.A().currentBackupID is not None:
if backup_rebuilder.A().currentBackupID == backupID:
if _Debug:
lg.out(
_DebugLevel,
' %s : SKIP, because rebuilding is in process'
% backupID)
continue
packets = backup_matrix.ScanBlocksToRemove(
backupID, settings.getGeneralWaitSuppliers())
for packetID in packets:
customer, pathID = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(),
customer, pathID)
if os.path.isfile(filename):
try:
os.remove(filename)
# lg.out(6, ' ' + os.path.basename(filename))
except:
lg.exc()
continue
count += 1
if _Debug:
lg.out(_DebugLevel, ' %d files were removed' % count)
backup_matrix.ReadLocalFiles()
def run(self):
"""
Runs a new ``Job`` from that ``Task``.
Called from ``RunTasks()`` method if it is possible to start a
new task - the maximum number of simultaneously running ``Jobs``
is limited.
"""
import backup_tar
import backup
iter_and_path = backup_fs.WalkByID(self.pathID)
if iter_and_path is None:
lg.out(4, 'backup_control.Task.run ERROR %s not found in the index' % self.pathID)
# self.defer.callback('error', self.pathID)
return
itemInfo, sourcePath = iter_and_path
if isinstance(itemInfo, dict):
try:
itemInfo = itemInfo[backup_fs.INFO_KEY]
except:
lg.exc()
return
if self.localPath and self.localPath != sourcePath:
lg.warn('local path were changed: %s -> %s' % (self.localPath, sourcePath))
self.localPath = sourcePath
if not bpio.pathExist(sourcePath):
lg.warn('path not exist: %s' % sourcePath)
reactor.callLater(0, OnTaskFailed, self.pathID, 'not exist')
return
dataID = misc.NewBackupID()
if itemInfo.has_version(dataID):
# ups - we already have same version
# let's add 1,2,3... to the end to make absolutely unique version ID
i = 1
while itemInfo.has_version(dataID + str(i)):
i += 1
dataID += str(i)
backupID = self.pathID + '/' + dataID
try:
backup_fs.MakeLocalDir(settings.getLocalBackupsDir(), backupID)
except:
lg.exc()
lg.out(4, 'backup_control.Task.run ERROR creating destination folder for %s' % self.pathID)
# self.defer.callback('error', self.pathID)
return
compress_mode = 'bz2' # 'none' # 'gz'
if bpio.pathIsDir(sourcePath):
backupPipe = backup_tar.backuptar(sourcePath, compress=compress_mode)
else:
backupPipe = backup_tar.backuptarfile(sourcePath, compress=compress_mode)
backupPipe.make_nonblocking()
job = backup.backup(
backupID, backupPipe,
OnJobDone, OnBackupBlockReport,
settings.getBackupBlockSize(),
sourcePath)
jobs()[backupID] = job
itemInfo.add_version(dataID)
if itemInfo.type in [backup_fs.PARENT, backup_fs.DIR]:
dirsize.ask(sourcePath, OnFoundFolderSize, (self.pathID, dataID))
else:
jobs()[backupID].totalSize = os.path.getsize(sourcePath)
jobs()[backupID].automat('start')
reactor.callLater(0, FireTaskStartedCallbacks, self.pathID, dataID)
lg.out(4, 'backup_control.Task-%d.run [%s/%s], size=%d, %s' % (
self.number, self.pathID, dataID, itemInfo.size, sourcePath))
def RunRequest(self):
#out(6, 'io_throttle.RunRequest')
packetsToRemove = {}
for i in range(0, min(self.fileRequestMaxLength, len(self.fileRequestQueue))):
packetID = self.fileRequestQueue[i]
# we got notify that this packet was failed to send
if packetID in self.requestFailedPacketIDs:
self.requestFailedPacketIDs.remove(packetID)
packetsToRemove[packetID] = 'failed'
continue
# request timeouts are disabled for now
# currentTime = time.time()
# if self.fileRequestDict[packetID].requestTime is not None:
# # the packet was requested
# if self.fileRequestDict[packetID].fileReceivedTime is None:
# # but no answer yet ...
# if currentTime - self.fileRequestDict[packetID].requestTime > self.fileRequestDict[packetID].requestTimeout:
# # and time is out!!!
# self.fileRequestDict[packetID].report = 'timeout'
# packetsToRemove[packetID] = 'timeout'
# else:
# # the packet were received (why it is not removed from the queue yet ???)
# self.fileRequestDict[packetID].result = 'received'
# packetsToRemove[packetID] = 'received'
# the packet was not requested yet
if self.fileRequestDict[packetID].requestTime is None:
customer, pathID = packetid.SplitPacketID(packetID)
if not os.path.exists(os.path.join(settings.getLocalBackupsDir(), customer, pathID)):
fileRequest = self.fileRequestDict[packetID]
if _Debug:
lg.out(_DebugLevel, "io_throttle.RunRequest for packetID " + fileRequest.packetID)
# transport_control.RegisterInterest(self.DataReceived,fileRequest.creatorID,fileRequest.packetID)
# callback.register_interest(self.DataReceived, fileRequest.creatorID, fileRequest.packetID)
p2p_service.SendRetreive(
fileRequest.ownerID,
fileRequest.creatorID,
fileRequest.packetID,
fileRequest.remoteID,
callbacks={
commands.Data(): self.OnDataReceived,
commands.Fail(): self.OnDataReceived,
# None: lambda pkt_out: self.OnDataReceived(fileRequest.packetID, 'timeout'), # timeout
},
# response_timeout=10,
)
# newpacket = signed.Packet(
# commands.Retrieve(),
# fileRequest.ownerID,
# fileRequest.creatorID,
# packetid.RemotePath(fileRequest.packetID),
# "",
# fileRequest.remoteID)
# gateway.outbox(newpacket, callbacks={
# commands.Data(): self.DataReceived,
# commands.Fail(): self.DataReceived})
fileRequest.requestTime = time.time()
else:
# we have the data file, no need to request it
self.fileRequestDict[packetID].result = 'exist'
packetsToRemove[packetID] = 'exist'
# if request queue is empty - remove all records about packets failed to request
if len(self.fileRequestQueue) == 0:
del self.requestFailedPacketIDs[:]
# remember requests results
result = len(packetsToRemove)
# remove finished requests
for packetID, why in packetsToRemove.items():
# self.fileRequestQueue.remove(packetID)
if _Debug:
lg.out(_DebugLevel, "io_throttle.RunRequest removed %s from %s receiving queue, %d more items" % (
packetID, self.remoteName, len(self.fileRequestQueue)))
self.OnDataRequestFailed(packetID, why)
del packetsToRemove
return result
def _request_files(self):
from storage import backup_matrix
from customer import io_throttle
from customer import data_sender
self.missingPackets = 0
# here we want to request some packets before we start working to
# rebuild the missed blocks
availableSuppliers = backup_matrix.GetActiveArray(customer_idurl=self.currentCustomerIDURL)
# remember how many requests we did on this iteration
total_requests_count = 0
# at the moment I do download everything I have available and needed
if '' in contactsdb.suppliers(customer_idurl=self.currentCustomerIDURL):
lg.out(8, 'backup_rebuilder._request_files SKIP - empty supplier')
self.automat('no-requests')
return
for supplierNum in range(contactsdb.num_suppliers(customer_idurl=self.currentCustomerIDURL)):
supplierID = contactsdb.supplier(supplierNum, customer_idurl=self.currentCustomerIDURL)
if not supplierID:
continue
requests_count = 0
# we do requests in reverse order because we start rebuilding from
# the last block
for blockIndex in range(len(self.workingBlocksQueue) - 1, -1, -1):
blockNum = self.workingBlocksQueue[blockIndex]
# do not keep too many requests in the queue
if io_throttle.GetRequestQueueLength(supplierID) >= 16:
break
# also don't do too many requests at once
if requests_count > 16:
break
remoteData = backup_matrix.GetRemoteDataArray(
self.currentBackupID, blockNum)
remoteParity = backup_matrix.GetRemoteParityArray(
self.currentBackupID, blockNum)
localData = backup_matrix.GetLocalDataArray(
self.currentBackupID, blockNum)
localParity = backup_matrix.GetLocalParityArray(
self.currentBackupID, blockNum)
if supplierNum >= len(remoteData) or supplierNum >= len(remoteParity):
break
if supplierNum >= len(localData) or supplierNum >= len(localParity):
break
# if remote Data exist and is available because supplier is on-line,
# but we do not have it on hand - do request
if localData[supplierNum] == 0:
PacketID = packetid.MakePacketID(
self.currentBackupID, blockNum, supplierNum, 'Data')
if remoteData[supplierNum] == 1:
if availableSuppliers[supplierNum]:
# if supplier is not alive - we can't request from him
if not io_throttle.HasPacketInRequestQueue(supplierID, PacketID):
customer, remotePath = packetid.SplitPacketID(PacketID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customer,
remotePath,
)
if not os.path.exists(filename):
if io_throttle.QueueRequestFile(
self._file_received,
my_id.getLocalID(),
PacketID,
my_id.getLocalID(),
supplierID):
requests_count += 1
else:
# count this packet as missing
self.missingPackets += 1
# also mark this guy as one who dont have any data - nor local nor remote
else:
# but if local Data already exists, but was not sent - do it now
if remoteData[supplierNum] != 1:
data_sender.A('new-data')
# same for Parity
if localParity[supplierNum] == 0:
PacketID = packetid.MakePacketID(
self.currentBackupID, blockNum, supplierNum, 'Parity')
if remoteParity[supplierNum] == 1:
if availableSuppliers[supplierNum]:
if not io_throttle.HasPacketInRequestQueue(
supplierID, PacketID):
customer, remotePath = packetid.SplitPacketID(PacketID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customer,
remotePath,
)
if not os.path.exists(filename):
if io_throttle.QueueRequestFile(
self._file_received,
my_id.getLocalID(),
PacketID,
my_id.getLocalID(),
supplierID,
):
requests_count += 1
else:
self.missingPackets += 1
else:
# but if local Parity already exists, but was not sent - do it now
if remoteParity[supplierNum] != 1:
data_sender.A('new-data')
total_requests_count += requests_count
if total_requests_count > 0:
lg.out(8, 'backup_rebuilder._request_files : %d chunks requested' % total_requests_count)
self.automat('requests-sent', total_requests_count)
else:
if self.missingPackets:
lg.out(8, 'backup_rebuilder._request_files : found %d missing packets' % self.missingPackets)
self.automat('found-missing')
else:
lg.out(8, 'backup_rebuilder._request_files : nothing was requested')
self.automat('no-requests')
def _do_send_packets(self, backup_id, block_num):
customer_id, path_id, version_name = packetid.SplitBackupID(backup_id)
archive_snapshot_dir = os.path.join(settings.getLocalBackupsDir(),
customer_id, path_id, version_name)
if _Debug:
lg.args(_DebugLevel,
backup_id=backup_id,
block_num=block_num,
archive_snapshot_dir=archive_snapshot_dir)
if not os.path.isdir(archive_snapshot_dir):
self.block_failed = True
lg.err('archive snapshot folder was not found in %r' %
archive_snapshot_dir)
return None
failed_supliers = 0
for supplier_num in range(len(self.suppliers_list)):
supplier_idurl = self.suppliers_list[supplier_num]
if not supplier_idurl:
failed_supliers += 1
lg.warn('unknown supplier supplier_num=%d' % supplier_num)
continue
for dataORparity in (
'Data',
'Parity',
):
packet_id = packetid.MakePacketID(backup_id, block_num,
supplier_num, dataORparity)
packet_filename = os.path.join(
archive_snapshot_dir, '%d-%d-%s' % (
block_num,
supplier_num,
dataORparity,
))
if not os.path.isfile(packet_filename):
lg.err('%s is not a file' % packet_filename)
continue
packet_payload = bpio.ReadBinaryFile(packet_filename)
if not packet_payload:
lg.err('file %r reading error' % packet_filename)
continue
if block_num not in self.packets_out:
self.packets_out[block_num] = {}
self.packets_out[block_num][packet_id] = None
p2p_service.SendData(
raw_data=packet_payload,
ownerID=self.queue_owner_idurl,
creatorID=my_id.getIDURL(),
remoteID=supplier_idurl,
packetID=packet_id,
callbacks={
commands.Ack():
lambda newpacket, _: self.automat('ack',
newpacket=newpacket),
commands.Fail():
lambda newpacket, _: self.automat('fail',
newpacket=newpacket),
},
)
if failed_supliers > self.correctable_errors:
self.block_failed = True
lg.err('too many failed suppliers %d in block %d' % (
failed_supliers,
block_num,
))
def doScanAndQueue(self, arg):
global _ShutdownFlag
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue _ShutdownFlag=%r' % _ShutdownFlag)
if _Debug:
log = open(os.path.join(settings.LogsDir(), 'data_sender.log'),
'w')
log.write('doScanAndQueue %s\n' % time.asctime())
if _ShutdownFlag:
if _Debug:
log.write('doScanAndQueue _ShutdownFlag is True\n')
self.automat('scan-done')
if _Debug:
log.flush()
log.close()
return
for customer_idurl in contactsdb.known_customers():
if '' not in contactsdb.suppliers(customer_idurl):
from storage import backup_matrix
for backupID in misc.sorted_backup_ids(
backup_matrix.local_files().keys(), True):
packetsBySupplier = backup_matrix.ScanBlocksToSend(
backupID)
if _Debug:
log.write('%s\n' % packetsBySupplier)
for supplierNum in packetsBySupplier.keys():
supplier_idurl = contactsdb.supplier(
supplierNum, customer_idurl=customer_idurl)
if not supplier_idurl:
lg.warn('?supplierNum? %s for %s' %
(supplierNum, backupID))
continue
for packetID in packetsBySupplier[supplierNum]:
backupID_, _, supplierNum_, _ = packetid.BidBnSnDp(
packetID)
if backupID_ != backupID:
lg.warn('?backupID? %s for %s' %
(packetID, backupID))
continue
if supplierNum_ != supplierNum:
lg.warn('?supplierNum? %s for %s' %
(packetID, backupID))
continue
if io_throttle.HasPacketInSendQueue(
supplier_idurl, packetID):
if _Debug:
log.write(
'%s already in sending queue for %s\n'
% (packetID, supplier_idurl))
continue
if not io_throttle.OkToSend(supplier_idurl):
if _Debug:
log.write('ok to send %s ? - NO!\n' %
supplier_idurl)
continue
customerGlobalID, pathID = packetid.SplitPacketID(
packetID)
# tranByID = gate.transfers_out_by_idurl().get(supplier_idurl, [])
# if len(tranByID) > 3:
# log.write('transfers by %s: %d\n' % (supplier_idurl, len(tranByID)))
# continue
customerGlobalID, pathID = packetid.SplitPacketID(
packetID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customerGlobalID,
pathID,
)
if not os.path.isfile(filename):
if _Debug:
log.write('%s is not a file\n' % filename)
continue
if io_throttle.QueueSendFile(
filename,
packetID,
supplier_idurl,
my_id.getLocalID(),
self._packetAcked,
self._packetFailed,
):
if _Debug:
log.write(
'io_throttle.QueueSendFile %s\n' %
packetID)
else:
if _Debug:
log.write(
'io_throttle.QueueSendFile FAILED %s\n'
% packetID)
# lg.out(6, ' %s for %s' % (packetID, backupID))
# DEBUG
# break
self.automat('scan-done')
if _Debug:
log.flush()
log.close()
