def cmd_backups(opts, args, overDict):
if len(args) < 2 or args[1] == 'list':
return call_xmlrpc_method_and_stop('backups_list')
elif len(args) < 2 or args[1] == 'idlist':
return call_xmlrpc_method_and_stop('backups_id_list')
elif args[1] == 'start' and len(args) >= 3:
from lib import packetid
if packetid.Valid(args[2]):
return call_xmlrpc_method_and_stop('backup_start_id', args[2])
if not os.path.exists(os.path.abspath(args[2])):
print_text('path %s not exist\n' % args[2])
return 1
return call_xmlrpc_method_and_stop('backup_start_path', args[2])
elif args[1] == 'add' and len(args) >= 3:
localpath = os.path.abspath(args[2])
if not os.path.exists(localpath):
print_text('path %s not exist\n' % args[2])
return 1
from system import bpio
if bpio.pathIsDir(localpath):
m = 'backup_dir_add'
else:
m = 'backup_file_add'
return call_xmlrpc_method_and_stop(m, localpath)
elif args[1] == 'addtree' and len(args) >= 3:
localpath = os.path.abspath(args[2])
from system import bpio
if not bpio.pathIsDir(localpath):
print_text('folder %s not exist\n' % args[2])
return 1
return call_xmlrpc_method_and_stop('backup_tree_add', localpath)
elif args[1] == 'delete' and len(args) >= 3:
if args[2] == 'local':
if len(args) < 4:
return 2
return call_xmlrpc_method_and_stop('backup_delete_local', args[3].replace('/','_'))
if packetid.Valid(args[2]):
return call_xmlrpc_method_and_stop('backup_delete_id', args[2].replace('/','_'))
return call_xmlrpc_method_and_stop('backup_delete_path', os.path.abspath(args[2]))
elif args[1] == 'update':
return call_xmlrpc_method_and_stop('backups_update')
return 2
def _upload(params):
path = params['path']
if bpio.Linux() or bpio.Mac():
path = '/' + (path.lstrip('/'))
localPath = unicode(path)
if not bpio.pathExist(localPath):
return {
'result': {
"success": False,
"error": 'local path %s was not found' % path
}
}
result = []
pathID = backup_fs.ToID(localPath)
if pathID is None:
if bpio.pathIsDir(localPath):
pathID, iter, iterID = backup_fs.AddDir(localPath, read_stats=True)
result.append('new folder was added: %s' % localPath)
else:
pathID, iter, iterID = backup_fs.AddFile(localPath,
read_stats=True)
result.append('new file was added: %s' % localPath)
pathID = global_id.CanonicalID(pathID)
backup_control.StartSingle(pathID=pathID, localPath=localPath)
backup_fs.Calculate()
backup_control.Save()
control.request_update([
('pathID', pathID),
])
result.append('backup started: %s' % pathID)
return {
'result': result,
}
def backuptardir_thread(directorypath, arcname=None, recursive_subfolders=True, compress=None):
"""
Makes tar archive of a single file inside a thread.
Returns `BytesLoop` object instance which can be used to read produced data in parallel.
"""
if not bpio.pathIsDir(directorypath):
lg.err('folder %s not found' % directorypath)
return None
if arcname is None:
arcname = os.path.basename(directorypath)
p = BytesLoop()
def _run():
from storage import tar_file
ret = tar_file.writetar(
sourcepath=directorypath,
arcname=arcname,
subdirs=recursive_subfolders,
compression=compress or 'none',
encoding='utf-8',
fileobj=p,
)
p.mark_finished()
if _Debug:
lg.out(_DebugLevel, 'backup_tar.backuptardir_thread writetar() finished')
return ret
reactor.callInThread(_run) # @UndefinedVariable
return p
def backuptardir(directorypath, arcname=None, recursive_subfolders=True, compress=None):
"""
Returns file descriptor for process that makes tar archive.
In other words executes a child process and create a Pipe to
communicate with it.
"""
if not bpio.pathIsDir(directorypath):
lg.out(1, 'backup_tar.backuptar ERROR %s not found' % directorypath)
return None
subdirs = 'subdirs'
if not recursive_subfolders:
subdirs = 'nosubdirs'
if compress is None:
compress = 'none'
if arcname is None:
arcname = strng.to_text(os.path.basename(directorypath))
# lg.out(14, "backup_tar.backuptar %s %s compress=%s" % (directorypath, subdirs, compress))
if bpio.Windows():
if bpio.isFrozen():
commandpath = "bppipe.exe"
cmdargs = [commandpath, subdirs, compress, directorypath, arcname]
else:
commandpath = "bppipe.py"
cmdargs = [sys.executable, commandpath, subdirs, compress, directorypath, arcname]
else:
commandpath = "bppipe.py"
cmdargs = [sys.executable, commandpath, subdirs, compress, directorypath, arcname]
if not os.path.isfile(commandpath):
lg.out(1, 'backup_tar.backuptar ERROR %s not found' % commandpath)
return None
# lg.out(14, "backup_tar.backuptar going to execute %s" % str(cmdargs))
# p = child_process.run('bppipe', cmdargs[2:])
p = child_process.pipe(cmdargs)
return p
def backuptar(directorypath, recursive_subfolders=True, compress=None):
"""
Returns file descriptor for process that makes tar archive.
In other words executes a child process and create a Pipe to
communicate with it.
"""
if not bpio.pathIsDir(directorypath):
lg.out(1, 'backup_tar.backuptar ERROR %s not found' % directorypath)
return None
subdirs = 'subdirs'
if not recursive_subfolders:
subdirs = 'nosubdirs'
if compress is None:
compress = 'none'
# lg.out(14, "backup_tar.backuptar %s %s compress=%s" % (directorypath, subdirs, compress))
if bpio.Windows():
if bpio.isFrozen():
commandpath = "bppipe.exe"
cmdargs = [commandpath, subdirs, compress, directorypath]
else:
commandpath = "bppipe.py"
cmdargs = [sys.executable, commandpath, subdirs, compress, directorypath]
else:
commandpath = "bppipe.py"
cmdargs = [sys.executable, commandpath, subdirs, compress, directorypath]
if not os.path.isfile(commandpath):
lg.out(1, 'backup_tar.backuptar ERROR %s not found' % commandpath)
return None
# lg.out(14, "backup_tar.backuptar going to execute %s" % str(cmdargs))
# p = child_process.run('bppipe', cmdargs[2:])
p = child_process.pipe(cmdargs)
return p
def main():
sourcePath = sys.argv[1]
backupID = sys.argv[2]
lg.set_debug_level(24)
compress_mode = 'none' # 'gz'
raid_worker.A('init')
backupPath = backup_fs.MakeLocalDir(settings.getLocalBackupsDir(), backupID)
if bpio.pathIsDir(sourcePath):
backupPipe = backup_tar.backuptar(sourcePath, compress=compress_mode)
else:
backupPipe = backup_tar.backuptarfile(sourcePath, compress=compress_mode)
backupPipe.make_nonblocking()
job = backup.backup(backupID, backupPipe, backup_done)
reactor.callLater(1, job.automat, 'start')
reactor.run()
def cmd_schedule(opts, args, overDict):
if len(args) < 2:
return 2
from system import bpio
if not bpio.pathIsDir(os.path.abspath(args[1])):
print_text('folder %s not exist\n' % args[1])
return 1
backupDir = os.path.abspath(args[1])
if len(args) < 3:
return call_xmlrpc_method_and_stop('getschedule', backupDir)
from lib import schedule
shed = schedule.from_compact_string(args[2])
if shed is None:
print_text(schedule.format()+'\n')
return 0
return call_xmlrpc_method_and_stop(
'setschedule', backupDir, shed.type, shed.interval, shed.daytime, shed.details,)
def main():
from system import bpio
from . import backup_tar
from . import backup_fs
lg.set_debug_level(24)
sourcePath = sys.argv[1]
compress_mode = 'none' # 'gz'
backupID = sys.argv[2]
raid_worker.A('init')
backupPath = backup_fs.MakeLocalDir(settings.getLocalBackupsDir(),
backupID)
if bpio.pathIsDir(sourcePath):
backupPipe = backup_tar.backuptardir(sourcePath,
compress=compress_mode)
else:
backupPipe = backup_tar.backuptarfile(sourcePath,
compress=compress_mode)
backupPipe.make_nonblocking()
def _bk_done(bid, result):
from crypt import signed
customer, remotePath = packetid.SplitPacketID(bid)
try:
os.mkdir(
os.path.join(settings.getLocalBackupsDir(), customer,
remotePath + '.out'))
except:
pass
for filename in os.listdir(
os.path.join(settings.getLocalBackupsDir(), customer,
remotePath)):
filepath = os.path.join(settings.getLocalBackupsDir(), customer,
remotePath, filename)
payld = bpio.ReadBinaryFile(filepath)
newpacket = signed.Packet('Data', my_id.getLocalID(),
my_id.getLocalID(), filename, payld,
'http://megafaq.ru/cvps1010.xml')
newfilepath = os.path.join(settings.getLocalBackupsDir(), customer,
remotePath + '.out', filename)
bpio.WriteBinaryFile(newfilepath, newpacket.Serialize())
reactor.stop() # @UndefinedVariable
job = backup(backupID, backupPipe, _bk_done)
reactor.callLater(1, job.automat, 'start') # @UndefinedVariable
reactor.run() # @UndefinedVariable
def main():
sourcePath = sys.argv[1]
backupID = sys.argv[2]
lg.set_debug_level(24)
compress_mode = 'none' # 'gz'
raid_worker.A('init')
backupPath = backup_fs.MakeLocalDir(settings.getLocalBackupsDir(),
backupID)
if bpio.pathIsDir(sourcePath):
backupPipe = backup_tar.backuptardir(sourcePath,
compress=compress_mode)
else:
backupPipe = backup_tar.backuptarfile(sourcePath,
compress=compress_mode)
backupPipe.make_nonblocking()
job = backup.backup(backupID, backupPipe, backup_done)
reactor.callLater(1, job.automat, 'start')
reactor.run()
def _upload(params):
path = params['path']
if bpio.Linux() or bpio.Mac():
path = '/' + (path.lstrip('/'))
localPath = unicode(path)
if not bpio.pathExist(localPath):
return {'result': {"success": False, "error": 'local path %s was not found' % path}}
result = []
pathID = backup_fs.ToID(localPath)
if pathID is None:
if bpio.pathIsDir(localPath):
pathID, iter, iterID = backup_fs.AddDir(localPath, True)
result.append('new folder was added: %s' % localPath)
else:
pathID, iter, iterID = backup_fs.AddFile(localPath, True)
result.append('new file was added: %s' % localPath)
backup_control.StartSingle(pathID, localPath)
backup_fs.Calculate()
backup_control.Save()
control.request_update([('pathID', pathID), ])
result.append('backup started: %s' % pathID)
return {'result': result, }
def main():
from system import bpio
import backup_tar
import backup_fs
lg.set_debug_level(24)
sourcePath = sys.argv[1]
compress_mode = 'none' # 'gz'
backupID = sys.argv[2]
raid_worker.A('init')
backupPath = backup_fs.MakeLocalDir(settings.getLocalBackupsDir(), backupID)
if bpio.pathIsDir(sourcePath):
backupPipe = backup_tar.backuptar(sourcePath, compress=compress_mode)
else:
backupPipe = backup_tar.backuptarfile(sourcePath, compress=compress_mode)
backupPipe.make_nonblocking()
def _bk_done(bid, result):
from crypt import signed
try:
os.mkdir(os.path.join(settings.getLocalBackupsDir(), bid + '.out'))
except:
pass
for filename in os.listdir(os.path.join(settings.getLocalBackupsDir(), bid)):
filepath = os.path.join(settings.getLocalBackupsDir(), bid, filename)
payld = str(bpio.ReadBinaryFile(filepath))
newpacket = signed.Packet(
'Data',
my_id.getLocalID(),
my_id.getLocalID(),
filename,
payld,
'http://megafaq.ru/cvps1010.xml')
newfilepath = os.path.join(settings.getLocalBackupsDir(), bid + '.out', filename)
bpio.AtomicWriteFile(newfilepath, newpacket.Serialize())
reactor.stop()
job = backup(backupID, backupPipe, _bk_done)
reactor.callLater(1, job.automat, 'start')
reactor.run()
def main():
from system import bpio
from storage import backup_tar
bpio.init()
settings.init()
lg.set_debug_level(24)
lg.life_begins()
automat.LifeBegins(lg.when_life_begins())
automat.OpenLogFile(settings.AutomatsLog())
key.InitMyKey()
my_id.init()
sourcePath = sys.argv[1]
compress_mode = 'none' # 'gz'
backupID = sys.argv[2]
raid_worker.A('init')
if bpio.pathIsDir(sourcePath):
backupPipe = backup_tar.backuptardir_thread(sourcePath,
compress=compress_mode)
else:
backupPipe = backup_tar.backuptarfile_thread(sourcePath,
compress=compress_mode)
def _bk_done(bid, result):
from crypt import signed
customer, remotePath = packetid.SplitPacketID(bid)
try:
os.mkdir(
os.path.join(settings.getLocalBackupsDir(), customer,
remotePath + '.out'))
except:
pass
for filename in os.listdir(
os.path.join(settings.getLocalBackupsDir(), customer,
remotePath)):
filepath = os.path.join(settings.getLocalBackupsDir(), customer,
remotePath, filename)
payld = bpio.ReadBinaryFile(filepath)
newpacket = signed.Packet('Data', my_id.getLocalID(),
my_id.getLocalID(), filename, payld,
'http://megafaq.ru/cvps1010.xml')
newfilepath = os.path.join(settings.getLocalBackupsDir(), customer,
remotePath + '.out', filename)
bpio.WriteBinaryFile(newfilepath, newpacket.Serialize())
def _bk_closed(*args, **kwargs):
# job.automat('fail')
# del job
reactor.stop() # @UndefinedVariable
def _bk_start():
job = backup(backupID, backupPipe, blockSize=16 * 1024 * 1024)
job.finishCallback = _bk_done # lambda bid, result: _bk_done(bid, result, job)
job.addStateChangedCallback(_bk_closed, oldstate=None, newstate='DONE')
reactor.callLater(1, job.automat, 'start') # @UndefinedVariable
reactor.callLater(0, _bk_start) # @UndefinedVariable
reactor.run() # @UndefinedVariable
settings.shutdown()
def run(self):
"""
Runs a new ``Job`` from that ``Task``.
"""
iter_and_path = backup_fs.WalkByID(self.remotePath, iterID=backup_fs.fsID(self.customerIDURL))
if iter_and_path is None:
lg.out(4, 'backup_control.Task.run ERROR %s not found in the index' % self.remotePath)
# self.defer.callback('error', self.pathID)
# self._on_job_failed(self.pathID)
err = 'remote path "%s" not found in the catalog' % self.remotePath
OnTaskFailed(self.pathID, err)
return err
itemInfo, sourcePath = iter_and_path
if isinstance(itemInfo, dict):
try:
itemInfo = itemInfo[backup_fs.INFO_KEY]
except:
lg.exc()
# self._on_job_failed(self.pathID)
err = 'catalog item related to "%s" is broken' % self.remotePath
OnTaskFailed(self.pathID, err)
return err
if not self.localPath:
self.localPath = sourcePath
lg.out('backup_control.Task.run local path was populated from catalog: %s' % self.localPath)
if self.localPath != sourcePath:
lg.warn('local path is differ from catalog: %s != %s' % (self.localPath, sourcePath))
if not bpio.pathExist(self.localPath):
lg.warn('path not exist: %s' % self.localPath)
# self._on_job_failed(self.pathID)
err = 'local path "%s" not exist' % self.localPath
OnTaskFailed(self.pathID, err)
return err
# if os.path.isfile(self.localPath) and self.localPath != sourcePath:
# tmpfile.make(name, extension, prefix)
dataID = misc.NewBackupID()
if itemInfo.has_version(dataID):
# ups - we already have same version
# let's add 1,2,3... to the end to make absolutely unique version ID
i = 1
while itemInfo.has_version(dataID + str(i)):
i += 1
dataID += str(i)
self.backupID = packetid.MakeBackupID(
customer=self.fullCustomerID,
path_id=self.remotePath,
version=dataID,
)
if self.backupID in jobs():
lg.warn('backup job %s already started' % self.backupID)
return 'backup job %s already started' % self.backupID
try:
backup_fs.MakeLocalDir(settings.getLocalBackupsDir(), self.backupID)
except:
lg.exc()
lg.out(4, 'backup_control.Task.run ERROR creating destination folder for %s' % self.pathID)
# self.defer.callback('error', self.pathID)
# self._on_job_failed(self.backupID)
err = 'failed creating destination folder for "%s"' % self.backupID
return OnTaskFailed(self.backupID, err)
compress_mode = 'bz2' # 'none' # 'gz'
arcname = os.path.basename(sourcePath)
if bpio.pathIsDir(self.localPath):
backupPipe = backup_tar.backuptardir(self.localPath, arcname=arcname, compress=compress_mode)
else:
backupPipe = backup_tar.backuptarfile(self.localPath, arcname=arcname, compress=compress_mode)
backupPipe.make_nonblocking()
job = backup.backup(
self.backupID,
backupPipe,
finishCallback=OnJobDone,
blockResultCallback=OnBackupBlockReport,
blockSize=settings.getBackupBlockSize(),
sourcePath=self.localPath,
keyID=self.keyID or itemInfo.key_id,
)
jobs()[self.backupID] = job
itemInfo.add_version(dataID)
if itemInfo.type == backup_fs.DIR:
dirsize.ask(self.localPath, OnFoundFolderSize, (self.pathID, dataID))
else:
sz = os.path.getsize(self.localPath)
jobs()[self.backupID].totalSize = sz
itemInfo.set_size(sz)
backup_fs.Calculate()
Save()
jobs()[self.backupID].automat('start')
reactor.callLater(0, FireTaskStartedCallbacks, self.pathID, dataID)
lg.out(4, 'backup_control.Task-%d.run [%s/%s], size=%d, %s' % (
self.number, self.pathID, dataID, itemInfo.size, self.localPath))
return None
def run(self):
"""
Runs a new ``Job`` from that ``Task``.
Called from ``RunTasks()`` method if it is possible to start a
new task - the maximum number of simultaneously running ``Jobs``
is limited.
"""
import backup_tar
import backup
iter_and_path = backup_fs.WalkByID(self.pathID)
if iter_and_path is None:
lg.out(4, 'backup_control.Task.run ERROR %s not found in the index' % self.pathID)
# self.defer.callback('error', self.pathID)
return
itemInfo, sourcePath = iter_and_path
if isinstance(itemInfo, dict):
try:
itemInfo = itemInfo[backup_fs.INFO_KEY]
except:
lg.exc()
return
if self.localPath and self.localPath != sourcePath:
lg.warn('local path were changed: %s -> %s' % (self.localPath, sourcePath))
self.localPath = sourcePath
if not bpio.pathExist(sourcePath):
lg.warn('path not exist: %s' % sourcePath)
reactor.callLater(0, OnTaskFailed, self.pathID, 'not exist')
return
dataID = misc.NewBackupID()
if itemInfo.has_version(dataID):
# ups - we already have same version
# let's add 1,2,3... to the end to make absolutely unique version ID
i = 1
while itemInfo.has_version(dataID + str(i)):
i += 1
dataID += str(i)
backupID = self.pathID + '/' + dataID
try:
backup_fs.MakeLocalDir(settings.getLocalBackupsDir(), backupID)
except:
lg.exc()
lg.out(4, 'backup_control.Task.run ERROR creating destination folder for %s' % self.pathID)
# self.defer.callback('error', self.pathID)
return
compress_mode = 'bz2' # 'none' # 'gz'
if bpio.pathIsDir(sourcePath):
backupPipe = backup_tar.backuptar(sourcePath, compress=compress_mode)
else:
backupPipe = backup_tar.backuptarfile(sourcePath, compress=compress_mode)
backupPipe.make_nonblocking()
job = backup.backup(
backupID, backupPipe,
OnJobDone, OnBackupBlockReport,
settings.getBackupBlockSize(),
sourcePath)
jobs()[backupID] = job
itemInfo.add_version(dataID)
if itemInfo.type in [backup_fs.PARENT, backup_fs.DIR]:
dirsize.ask(sourcePath, OnFoundFolderSize, (self.pathID, dataID))
else:
jobs()[backupID].totalSize = os.path.getsize(sourcePath)
jobs()[backupID].automat('start')
reactor.callLater(0, FireTaskStartedCallbacks, self.pathID, dataID)
lg.out(4, 'backup_control.Task-%d.run [%s/%s], size=%d, %s' % (
self.number, self.pathID, dataID, itemInfo.size, sourcePath))
