def run(bucket,
public,
files,
endpoint=None,
indexes=True,
parent_links=True,
topdir_parent_link=False,
partition=False,
footer='index_footer.html',
prefix=None,
aws_access_key=None,
aws_secret_key=None):
if prefix:
prefix = prefix.lstrip('/')
if partition and prefix:
parts = prefix.split('/')
if len(parts) > 1:
bucket += '_' + parts[0]
prefix = '/'.join(parts[1:])
# Create the objects to make sure the arguments are sound.
with FileList() as file_list:
# Scan the files.
for file_path in files:
file_list.add(file_path)
indexer = Indexer(file_list)
# (Possibly) make indexes.
if indexes:
indexer.make_indexes(create_parent_links=parent_links,
create_topdir_parent_link=topdir_parent_link,
append_footer=footer)
logging.debug("List of files prepared to upload:")
for x in file_list:
logging.debug(x)
# Upload.
uploader = Uploader(bucket,
public,
endpoint,
prefix,
aws_access_key=aws_access_key,
aws_secret_key=aws_secret_key)
upload_failures = uploader.upload(file_list)
return uploader.url, upload_failures
def run(container,
files,
indexes=True,
parent_links=True,
topdir_parent_link=False,
partition=False,
footer='index_footer.html',
prefix=None,
dry_run=False,
credentials_file=None,
project=None):
if credentials_file:
cred = Credentials()
cred._set_path(credentials_file)
client = storage.Client(credentials=cred, project=project)
else:
client = storage.Client()
if prefix:
prefix = prefix.lstrip('/')
if partition and prefix:
parts = prefix.split('/')
if len(parts) > 1:
container += '_' + parts[0]
prefix = '/'.join(parts[1:])
# Create the objects to make sure the arguments are sound.
with FileList() as file_list:
# Scan the files.
for file_path in files:
file_list.add(file_path)
indexer = Indexer(file_list)
# (Possibly) make indexes.
if indexes:
indexer.make_indexes(create_parent_links=parent_links,
create_topdir_parent_link=topdir_parent_link,
append_footer=footer)
logging.debug("List of files prepared to upload:")
for x in file_list:
logging.debug(x)
# Upload.
uploader = Uploader(client, container, prefix, dry_run)
uploader.upload(file_list)
return uploader.url
def run(cloud,
container,
files,
indexes=True,
parent_links=True,
topdir_parent_link=False,
partition=False,
footer='index_footer.html',
delete_after=15552000,
prefix=None,
public=True,
archive_mode=False,
dry_run=False):
if prefix:
prefix = prefix.lstrip('/')
if partition and prefix:
parts = prefix.split('/')
if len(parts) > 1:
container += '_' + parts[0]
prefix = '/'.join(parts[1:])
# Create the objects to make sure the arguments are sound.
with FileList() as file_list:
# Scan the files.
for file_path in files:
file_list.add(file_path)
indexer = Indexer(file_list)
# (Possibly) make indexes.
if indexes:
indexer.make_indexes(create_parent_links=parent_links,
create_topdir_parent_link=topdir_parent_link,
append_footer=footer)
logging.debug("List of files prepared to upload:")
for x in file_list:
logging.debug(x)
# Upload.
uploader = Uploader(cloud, container, prefix, delete_after, public,
archive_mode, dry_run)
upload_failures = uploader.upload(file_list)
return uploader.url, uploader.endpoint, uploader.path, upload_failures
def run(container,
files,
indexes=True,
parent_links=True,
topdir_parent_link=False,
partition=False,
footer='index_footer.html',
prefix=None,
public=True,
dry_run=False,
connection_string=None):
client = BlobServiceClient.from_connection_string(connection_string)
if prefix:
prefix = prefix.lstrip('/')
if partition and prefix:
parts = prefix.split('/')
if len(parts) > 1:
container += '_' + parts[0]
prefix = '/'.join(parts[1:])
# Create the objects to make sure the arguments are sound.
with FileList() as file_list:
# Scan the files.
for file_path in files:
file_list.add(file_path)
indexer = Indexer(file_list)
# (Possibly) make indexes.
if indexes:
indexer.make_indexes(create_parent_links=parent_links,
create_topdir_parent_link=topdir_parent_link,
append_footer=footer)
logging.debug("List of files prepared to upload:")
for x in file_list:
logging.debug(x)
# Upload.
uploader = Uploader(client, container, prefix, public, dry_run)
uploader.upload(file_list)
return uploader.url