def main():
models = {
model: make_query(model).count()
for model in StoredFileMixin.__subclasses__()
}
models = {model: total for model, total in models.iteritems() if total}
labels = {
model: cformat(
'Processing %{blue!}{}%{reset} (%{cyan}{}%{reset} rows)').format(
model.__name__, total)
for model, total in models.iteritems()
}
max_length = max(len(x) for x in labels.itervalues())
labels = {
model: label.ljust(max_length)
for model, label in labels.iteritems()
}
for model, total in sorted(models.items(), key=itemgetter(1)):
with click.progressbar(query_chunked(model, 100),
length=total,
label=labels[model],
show_percent=True,
show_pos=True) as objects:
for obj in objects:
try:
with obj.open() as f:
checksum = get_file_checksum(f)
except Exception as exc:
click.echo(
cformat(
'\n%{red!}Could not open %{reset}%{yellow}{}%{red!}: %{reset}%{yellow!}{}'
).format(obj, exc))
else:
obj.md5 = checksum
def _save(self, bucket, name, content_type, fileobj):
fileobj = self._ensure_fileobj(fileobj)
checksum = get_file_checksum(fileobj)
fileobj.seek(0)
content_md5 = checksum.decode('hex').encode('base64').strip()
self.client.put_object(Body=fileobj, Bucket=bucket, Key=name,
ContentType=content_type, ContentMD5=content_md5)
return checksum
def main():
models = {model: make_query(model).count() for model in StoredFileMixin.__subclasses__()}
models = {model: total for model, total in models.iteritems() if total}
labels = {model: cformat('Processing %{blue!}{}%{reset} (%{cyan}{}%{reset} rows)').format(model.__name__, total)
for model, total in models.iteritems()}
max_length = max(len(x) for x in labels.itervalues())
labels = {model: label.ljust(max_length) for model, label in labels.iteritems()}
for model, total in sorted(models.items(), key=itemgetter(1)):
with click.progressbar(query_chunked(model, 100), length=total, label=labels[model],
show_percent=True, show_pos=True) as objects:
for obj in objects:
try:
with obj.open() as f:
checksum = get_file_checksum(f)
except Exception as exc:
click.echo(cformat('\n%{red!}Could not open %{reset}%{yellow}{}%{red!}: %{reset}%{yellow!}{}')
.format(obj, exc))
else:
obj.md5 = checksum
def _save(self, bucket, name, content_type, fileobj):
fileobj = self._ensure_fileobj(fileobj)
checksum = get_file_checksum(fileobj)
fileobj.seek(0)
content_md5 = b64encode(bytes.fromhex(checksum)).decode().strip()
metadata = {
'ContentType': content_type,
# the md5chksum header is used by tools like rclone in case the etag
# is not a md5 (in case of chunked uploads)
'Metadata': {
'md5chksum': content_md5
},
}
# only switch to chunked upload for large files
transfer_config = TransferConfig(
multipart_threshold=(512 * 1024 * 1024),
multipart_chunksize=(256 * 1024 * 1024))
self.client.upload_fileobj(Fileobj=fileobj,
Bucket=bucket,
Key=name,
ExtraArgs=metadata,
Config=transfer_config)
return checksum