def run(self, args, config, storage, remotes):
table_lines = [('', '<b>NAME</b>', '<b>START DATE</b>',
'<b>DURATION</b>')]
pattern = '%s/*' % args.remote if args.remote else None
for label, backup in sorted(storage.list_backups(pattern=pattern,
since=args.since,
until=args.until),
key=ORDERS[args.order]):
flags = []
if backup.parent:
flags.append('<b>P</b>')
if backup.errors:
flags.append('<color fg=red><b>E</b></color>')
# Extract remote from backup name:
name = label
split_name = label.split('/', 1)
if len(split_name) == 2 and remotes.get(split_name[0]):
name = '<b>%s</b>/%s' % tuple(split_name)
else:
flags.append('<b>O</b>')
table_lines.append(
(''.join(flags), name, backup.start_date.format(DATE_FORMAT),
str(backup.duration)))
printer.table(table_lines)
printer.p('\nFlags: <b>P</b> have parent, '
'<color fg=red><b>E</b></color> - have errors, '
'<b>O</b> orphan backup')
def run(self, args, config, storage, remotes):
count, size = gc(storage, delete=not args.dry_run)
if count:
printer.p('Done. Deleted {n} objects, total size: {s}',
n=count,
s=humanize.naturalsize(size, binary=True))
else:
printer.p('Done. Nothing to delete.')
def export_directory(tree, storage, output):
""" Export a tree in a directory.
"""
os.mkdir(output)
for fullname, item in walk_tree(storage, tree):
outfullname = os.path.join(output.encode('utf-8'),
fullname.lstrip(b'/'))
if item.type == 'blob':
blob = storage.get_blob(item.ref).blob
with open(outfullname, 'wb') as fout:
shutil.copyfileobj(blob, fout)
printer.verbose('Exporting to {out}: <b>{fn}</b> ({size})',
out=output,
fn=fullname.decode('utf-8', errors='replace'),
size=humanize.naturalsize(item['size'],
binary=True))
elif item.type == 'tree':
os.mkdir(outfullname)
printer.verbose('Exporting to {out}: <b>{fn}</b> (directory)',
out=output,
fn=fullname.decode('utf-8', errors='replace'))
else:
if item['filetype'] == 'link':
os.symlink(item['link'], outfullname)
printer.verbose(
'Exporting to {out}: <b>{fn}</b> (link to {link})',
out=output,
fn=fullname.decode('utf-8', errors='replace'),
link=item['link'].decode('utf-8', errors='replace'))
elif item['filetype'] == 'fifo':
os.mkfifo(outfullname)
printer.verbose('Exporting to {out}: <b>{fn}</b> (fifo)',
out=output,
fn=fullname.decode('utf-8', errors='replace'))
else:
continue # Ignore unknown file types
try:
if 'mode' in item:
try:
os.chmod(outfullname, item['mode'], follow_symlinks=False)
except SystemError:
pass # Workaround follow_symlinks not implemented in Python 3.5 (bug?)
if 'uid' in item or 'gid' in item:
os.chown(outfullname,
item.get('uid', -1),
item.get('gid', -1),
follow_symlinks=False)
except PermissionError:
printer.p(
'<color fg=yellow><b>Warning:</b> unable to set attributes on {fn}</color>',
fn=fullname.decode('utf-8', errors='replace'))
def check(storage, read_size=4096):
""" Check hash of all objects in the pool.
"""
for ref in storage.list():
printer.verbose('Checking {ref}', ref=ref, err=True)
hasher = hashlib.sha1()
fobject = storage.open(ref)
buf = fobject.read(read_size)
while buf:
hasher.update(buf)
buf = fobject.read(read_size)
if hasher.hexdigest() != ref:
printer.p(ref)
def check(storage, read_size=4096):
""" Check hash of all objects in the pool.
"""
for ref in storage.list():
printer.verbose('Checking {ref}', ref=ref, err=True)
hasher = hashlib.sha1()
fobject = storage.open(ref)
buf = fobject.read(read_size)
while buf:
hasher.update(buf)
buf = fobject.read(read_size)
if hasher.hexdigest() != ref:
printer.p(ref)
def export_directory(tree, storage, output):
""" Export a tree in a directory.
"""
os.mkdir(output)
for fullname, item in walk_tree(storage, tree):
outfullname = os.path.join(output.encode('utf-8'), fullname.lstrip(b'/'))
if item.type == 'blob':
blob = storage.get_blob(item.ref).blob
with open(outfullname, 'wb') as fout:
shutil.copyfileobj(blob, fout)
printer.verbose('Exporting to {out}: <b>{fn}</b> ({size})',
out=output,
fn=fullname.decode('utf-8', errors='replace'),
size=humanize.naturalsize(item['size'], binary=True))
elif item.type == 'tree':
os.mkdir(outfullname)
printer.verbose('Exporting to {out}: <b>{fn}</b> (directory)',
out=output,
fn=fullname.decode('utf-8', errors='replace'))
else:
if item['filetype'] == 'link':
os.symlink(item['link'], outfullname)
printer.verbose('Exporting to {out}: <b>{fn}</b> (link to {link})',
out=output,
fn=fullname.decode('utf-8', errors='replace'),
link=item['link'])
elif item['filetype'] == 'fifo':
os.mkfifo(outfullname)
printer.verbose('Exporting to {out}: <b>{fn}</b> (fifo)',
out=output,
fn=fullname.decode('utf-8', errors='replace'))
else:
continue # Ignore unknown file types
try:
if 'mode' in item:
try:
os.chmod(outfullname, item['mode'], follow_symlinks=False)
except SystemError:
pass # Workaround follow_symlinks not implemented in Python 3.5 (bug?)
if 'uid' in item or 'gid' in item:
os.chown(outfullname, item.get('uid', -1), item.get('gid', -1), follow_symlinks=False)
except PermissionError:
printer.p('<color fg=yellow><b>Warning:</b> unable to set attributes on {fn}</color>',
fn=fullname.decode('utf-8', errors='replace'))
def _print_tree(self, storage, tree, level=()):
last = False
next_level = level + (True, )
for i, (name, item) in enumerate(list(tree.items())):
if len(tree) == i + 1:
last = True
next_level = level + (False, )
header = ''.join(['│ ' if x else ' ' for x in level])
if last:
header += '└── '
else:
header += '├── '
filename = name.decode('utf-8', 'replace')
if item.type == 'tree':
printer.p('{h}<b><color fg=blue>{f}</color></b>',
h=header,
f=filename)
self._print_tree(storage,
storage.get_tree(item.ref),
level=next_level)
elif item.get('filetype') == 'link':
printer.p('{h}<color fg=cyan><b>{f}</b> -> {l}</color>',
h=header,
f=filename,
l=item.get('link', '?'))
else:
printer.p('{h}{f}', h=header, f=filename)
def scheduler(storage, remotes, workers=1, loop_interval=10):
""" Execute the scheduler for the specified remotes.
"""
printer.p('Scheduler started for {n} remotes', n=len(remotes))
running = {} # remote -> future backup task result
with concurrent.futures.ThreadPoolExecutor(
max_workers=workers) as executor:
while True:
for remote in remotes:
if remote in running:
continue # Ignore still running remotes
# Check if the remote has been backuped since configured interval:
interval = datetime.timedelta(
seconds=remote.scheduler['interval'] * 60)
parent = '%s/latest' % remote.name
backup = storage.get_backup(parent)
if backup is None:
parent = None
if backup is None or backup.start_date + interval < arrow.now(
):
running[remote] = executor.submit(scheduler_task, storage,
remote, parent)
printer.p('Queued a new backup for {n}', n=remote.name)
# Handle completed backups:
for remote, result in list(running.items()):
if result.done():
if result.exception() is not None:
printer.p('Backup for {n} failed: {e}',
n=remote.name,
e=result.exception())
else:
backup = result.result()
printer.p(
'Backup for {n} has been completed in {d} seconds',
n=remote.name,
d=backup.duration.seconds)
del running[remote]
time.sleep(loop_interval)
def scheduler(storage, remotes, workers=1, loop_interval=10):
""" Execute the scheduler for the specified remotes.
"""
printer.p('Scheduler started for {n} remotes', n=len(remotes))
running = {} # remote -> future backup task result
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
while True:
for remote in remotes:
if remote in running:
continue # Ignore still running remotes
# Check if the remote has been backuped since configured interval:
interval = datetime.timedelta(seconds=remote.scheduler['interval'] * 60)
parent = '%s/latest' % remote.name
backup = storage.get_backup(parent)
if backup is None:
parent = None
if backup is None or backup.start_date + interval < arrow.now():
running[remote] = executor.submit(scheduler_task, storage, remote, parent)
printer.p('Queued a new backup for {n}', n=remote.name)
# Handle completed backups:
for remote, result in list(running.items()):
if result.done():
if result.exception() is not None:
printer.p('Backup for {n} failed: {e}', n=remote.name, e=result.exception())
else:
backup = result.result()
printer.p('Backup for {n} has been completed in {d} seconds',
n=remote.name,
d=backup.duration.seconds)
del running[remote]
time.sleep(loop_interval)
def run(self, args, config, storage, remotes):
remote = remotes.get(args.remote)
if remote is None:
raise RuntimeError('Given remote (%s) does not exist' %
args.remote)
backup_label = '%s/%s' % (remote.name, args.name)
if not args.overwrite and storage.resolve(backup_label):
raise RuntimeError(
'A backup with this name already exists for this remote')
if args.parent:
parent = '%s/%s' % (remote.name, args.parent)
else:
parent = None
ref, backup = create_backup(storage, remote, parent=parent)
# Create labels for the new backup:
storage.set_label(backup_label, ref)
storage.set_label('%s/latest' % remote.name, ref)
printer.p('<b>Duration:</b> {d}', d=backup.duration)
printer.p('<b>Root:</b> {r}', r=backup.root)
if backup.errors:
printer.hr()
printer.p('<b>{n} errors:</b>', n=len(backup.errors))
printer.p()
for filename, error in backup.errors.items():
printer.p(' - <b>{fn}</b>: {error}',
fn=filename.decode('utf-8', 'replace'),
error=error)
if args.stats:
printer.hr()
printer.table(backup.stats_table(), fixed_width=80, center=True)
printer.p()
def run(self, args, config, storage, remotes):
count, size = gc(storage, delete=not args.dry_run)
if count:
printer.p('Done. Deleted {n} objects, total size: {s}', n=count, s=humanize.naturalsize(size, binary=True))
else:
printer.p('Done. Nothing to delete.')
def run(self, args, config, storage, remotes):
name = '%s/%s' % (args.remote, args.name) if args.remote else args.name
tree = storage.get_tree(name)
printer.p('<b><color fg=blue>.</color></b>')
self._print_tree(storage, tree)
def run(self, args, config, storage, remotes):
name = '%s/%s' % (args.remote, args.name) if args.remote else args.name
backup = storage.get_backup(name)
printer.p('<b>Date:</b> {s} -> {e} ({d})',
s=backup.start_date.format('DD/MM/YYYY HH:mm:ss'),
e=backup.end_date.format('DD/MM/YYYY HH:mm:ss'),
d=backup.duration)
printer.p('<b>Root:</b> {r}', r=backup.root)
if backup.parent:
printer.p('<b>Parent:</b> {b}', b=backup.parent)
if backup.errors:
printer.hr()
printer.p('<b>{n} errors:</b>', n=len(backup.errors))
printer.p()
for filename, error in list(backup.errors.items()):
printer.p(' - <b>{fn}</b>: {error}',
fn=filename.decode('utf-8', 'replace'),
error=error)
printer.p()
printer.p('-' * 80)
printer.p()
printer.table(backup.stats_table(), fixed_width=80, center=True)
printer.p()
def run(self, args, config, storage, remotes):
remote = remotes.get(args.remote)
backup_label = '%s/%s' % (remote.name, args.name)
if not args.overwrite and storage.resolve(backup_label):
raise RuntimeError('A backup with this name already exists for this remote')
if args.parent:
parent = '%s/%s' % (remote.name, args.parent)
else:
parent = None
ref, backup = create_backup(storage, remote, parent=parent)
# Create labels for the new backup:
storage.set_label(backup_label, ref)
storage.set_label('%s/latest' % remote.name, ref)
printer.p('<b>Duration:</b> {d}', d=backup.duration)
printer.p('<b>Root:</b> {r}', r=backup.root)
if backup.errors:
printer.hr()
printer.p('<b>{n} errors:</b>', n=len(backup.errors))
printer.p()
for filename, error in backup.errors.items():
printer.p(' - <b>{fn}</b>: {error}', fn=filename.decode('utf-8', 'replace'), error=error)
if args.stats:
printer.hr()
printer.table(backup.stats_table(), fixed_width=80, center=True)
printer.p()