def main(argv=None):
parser = argparse.ArgumentParser(description=_DESCRIPTION)
parser.add_argument('root', help='root directory of filetracker storage')
parser.add_argument('-s', '--silent', action='store_true',
help='if set, progress bar is not printed')
parser.add_argument('-f', '--full', action='store_true',
help='if set, logical size of all blobs is recalculated '
'(this may take a lot of time)')
args = parser.parse_args(argv)
root = args.root
silent = args.silent
full = args.full
ensure_storage_format(root)
db_init(os.path.join(root, 'db'))
# Create a FileStorage object to use the same db settings as usual
file_storage = FileStorage(root)
db = file_storage.db
links_widgets = [
' [', progress_bar.Timer(format='Time: %(elapsed)s'), '] ',
' Checking links '.ljust(_ACTION_LENGTH),
' ', progress_bar.Counter(), ' ',
progress_bar.BouncingBar()
]
processed_links = 0
broken_links = 0
blob_links = {}
with progress_bar.conditional(show=not silent,
widgets=links_widgets) as bar:
for cur_dir, _, files in os.walk(file_storage.links_dir):
for file_name in files:
link_path = os.path.join(cur_dir, file_name)
# In an unlikely case when links/ contains files
# that are not links, they are removed.
if not os.path.islink(link_path):
os.unlink(link_path)
broken_links += 1
else:
blob_path = os.path.join(
os.path.dirname(link_path), os.readlink(link_path))
if (os.path.islink(blob_path)
or not os.path.exists(blob_path)
or 'blobs/' not in blob_path):
os.unlink(link_path)
broken_links += 1
else:
digest = os.path.basename(blob_path)
blob_links[digest] = blob_links.get(digest, 0) + 1
processed_links += 1
bar.update(processed_links)
for digest, link_count in six.iteritems(blob_links):
db.put(digest.encode(), str(link_count).encode())
blobs_widgets = [
' [', progress_bar.Timer(format='Time: %(elapsed)s'), '] ',
' Checking blobs '.ljust(_ACTION_LENGTH),
' ', progress_bar.Counter(), ' ',
progress_bar.BouncingBar()
]
processed_blobs = 0
broken_blobs = 0
with progress_bar.conditional(show=not silent,
widgets=blobs_widgets) as bar:
for cur_dir, _, files in os.walk(file_storage.blobs_dir):
for blob_name in files:
if blob_name not in blob_links:
os.unlink(os.path.join(cur_dir, blob_name))
broken_blobs += 1
continue
size_key = '{}:logical_size'.format(blob_name).encode()
if not db.has_key(size_key) or full:
blob_path = os.path.join(cur_dir, blob_name)
with gzip.open(blob_path, 'rb') as zf:
logical_size = _read_stream_for_size(zf)
db.put(size_key, str(logical_size).encode())
processed_blobs += 1
bar.update(processed_blobs)
if not silent:
print('Completed, {} broken links and {} stray blobs found.'.format(
broken_links, broken_blobs))
def main(args=None):
parser = argparse.ArgumentParser(description=_DESCRIPTION)
parser.add_argument('files', help='file tree to be uploaded')
parser.add_argument('url', help='URL of the filetracker server')
parser.add_argument(
'--root', help='the directory that corresponds to the storage root')
parser.add_argument('-s',
'--silent',
action='store_true',
help='if set, progress bar is not printed')
args = parser.parse_args(args)
upload_root = args.files
url = args.url
storage_root = args.root
silent = args.silent
if storage_root is None:
storage_root = upload_root
# Create a client without local cache.
client = Client(local_store=None, remote_url=url)
# Calculate total size
total_size = 0
size_widgets = [
' [',
progress_bar.ShortTimer(),
'] ',
' Calculating file size '.ljust(_ACTION_LENGTH),
' ',
progress_bar.DataSize(),
' ',
progress_bar.BouncingBar(),
]
with progress_bar.conditional(show=not silent,
widgets=size_widgets) as bar:
for cur_dir, _, files in os.walk(upload_root):
for file_name in files:
total_size += os.path.getsize(os.path.join(cur_dir, file_name))
bar.update(total_size)
upload_widgets = [
' [',
progress_bar.ShortTimer(),
'] ',
' Uploading files '.ljust(_ACTION_LENGTH),
' ',
progress_bar.DataSize(),
' ',
progress_bar.Bar(),
' ',
progress_bar.Percentage(),
' ',
' (',
progress_bar.AdaptiveETA(),
') ',
]
processed_size = 0
with progress_bar.conditional(show=not silent,
max_value=total_size,
widgets=upload_widgets) as bar:
for cur_dir, _, files in os.walk(upload_root):
for file_name in files:
file_path = os.path.join(cur_dir, file_name)
remote_path = '/' + os.path.relpath(file_path, storage_root)
file_stat = os.stat(file_path)
file_size = file_stat.st_size
file_version = int(file_stat.st_mtime)
remote_name = '{}@{}'.format(remote_path, file_version)
try:
client.put_file(remote_name,
file_path,
to_local_store=False)
except FiletrackerError as e:
print('ERROR when uploading {}:\n{}'.format(file_path, e),
file=sys.stderr)
processed_size += file_size
bar.update(processed_size)