def download(self, container_name, dest, blob_names=None):
"""
Get blobs from the container to the `dest` directory.
"""
blobs = []
if not self.block_blob_service.exists(container_name):
logger.error("Container [%s] does not exist, aborted." %
container_name)
return blobs
# Get the list of blobs and then do comparision would be much more efficient
blobs_in_container = self.list_blobs(container_name)
# Get all blobs if blob_names was not specified
if not blob_names:
blob_names = blobs_in_container
for blob_name in progressbar.progressbar(\
blob_names, widgets=self.widgets):
if ppath(blob_name) in blobs_in_container:
dest_filepath = normpath(safe_join(dest, blob_name))
# TODO: not sure posix-style path works for files on container
# are windows-style
self.get_blob_to_path(container_name, ppath(blob_name),
dest_filepath)
logger.debug("Got blob '{}' to '{}'.".format(
blob_name, dest_filepath))
blobs.append(blob_name)
else:
logger.warning(
"Blob name '{}' specified does not exist.".format(
blob_name))
return blobs
def execute(self, context):
files = self.get_all_files(context)
self.stats.set_value("files/all", len(files))
passed, removed = self.partition(files, context)
self.stats.set_value("files/passed", len(passed))
self.stats.set_value("files/removed", len(removed))
blob_pairs = self.get_blob_pairs(passed, context['relpath'])
container_name = context['task_id']
if self.upload_files:
self.stats.set_value("upload/total", len(blob_pairs))
blobs = self.azure.upload(container_name,
blob_pairs,
overwrite=self.overwrite)
self.stats.set_value("upload/upload", len(blobs))
self.output.append("%s files were uploaded to [%s]." %
(len(blobs), container_name))
if self.generate_index:
index_file = safe_join(self.app.data_dirname,
container_name + '.json')
catalog = self.index(blob_pairs, context)
with open(index_file, 'w') as f:
f.write(self.to_string(catalog))
self.output.append("Index was written to '%s'." % index_file)
self.terminate(context)
return self.get_stats_id(context)
def download(self, container_name, dest, blob_names=None):
"""
Get blobs from the container to the `dest` directory.
"""
blobs = []
if not self.block_blob_service.exists(container_name):
logger.error("Container [%s] does not exist, aborted." % container_name)
return blobs
# Get the list of blobs and then do comparision would be much more efficient
blobs_in_container = self.list_blobs(container_name)
# Get all blobs if blob_names was not specified
if not blob_names:
blob_names = blobs_in_container
for blob_name in blob_names:
if ppath(blob_name) in blobs_in_container:
dest_filepath = safe_join(dest, blob_name)
# TODO: not sure posix-style path works for files on container
# are windows-style
blob = self.get_blob_to_path(container_name, ppath(blob_name), dest_filepath)
blobs.append(blob)
return blobs
def dest_filepath(self):
return safe_join(self.dest_root, self.filepath)
def dest_root(self):
return safe_join(self.app.data_dirname, self.title)