def start(args):
download_config = json.loads(base64.urlsafe_b64decode(args.download))
for download_arg in download_config["download_args"]:
from_path = download_arg["from"]
to_path = download_arg["to"]
item_name = download_arg.get("item_name", "")
bucket_name, prefix = S3.deconstruct_s3_path(from_path)
s3_client = S3(bucket_name, client_config={})
if item_name != "":
if download_arg.get("hide_from_log", False):
logger().info("downloading {}".format(item_name))
else:
logger().info("downloading {} from {}".format(
item_name, from_path))
if download_arg.get("to_file", False):
s3_client.download_file(prefix, to_path)
else:
s3_client.download(prefix, to_path)
if download_arg.get("unzip", False):
if item_name != "" and not download_arg.get(
"hide_unzipping_log", False):
logger().info("unzipping {}".format(item_name))
if download_arg.get("to_file", False):
util.extract_zip(to_path, delete_zip_file=True)
else:
util.extract_zip(os.path.join(to_path,
os.path.basename(from_path)),
delete_zip_file=True)
if download_config.get("last_log", "") != "":
logger().info(download_config["last_log"])
def start(args):
download = json.loads(base64.urlsafe_b64decode(args.download))
for download_arg in download:
from_path = download_arg["from"]
to_path = download_arg["to"]
item_name = download_arg.get("item_name", "")
bucket_name, prefix = S3.deconstruct_s3_path(from_path)
s3_client = S3(bucket_name, client_config={})
if item_name != "":
cx_logger().info("downloading {} from {}".format(item_name, from_path))
s3_client.download(prefix, to_path)
if download_arg.get("unzip", False):
if item_name != "":
cx_logger().info("unzipping {}".format(item_name))
util.extract_zip(
os.path.join(to_path, os.path.basename(from_path)), delete_zip_file=True
)
if download_arg.get("tf_model_version_rename", "") != "":
dest = util.trim_suffix(download_arg["tf_model_version_rename"], "/")
dir_path = os.path.dirname(dest)
entries = os.listdir(dir_path)
if len(entries) == 1:
src = os.path.join(dir_path, entries[0])
os.rename(src, dest)
def start(args):
download_config = json.loads(base64.urlsafe_b64decode(args.download))
for download_arg in download_config["download_args"]:
from_path = download_arg["from"]
to_path = download_arg["to"]
item_name = download_arg.get("item_name", "")
if from_path.startswith("s3://"):
bucket_name, prefix = S3.deconstruct_s3_path(from_path)
client = S3(bucket_name, client_config={})
elif from_path.startswith("gs://"):
bucket_name, prefix = GCS.deconstruct_gcs_path(from_path)
client = GCS(bucket_name)
else:
raise ValueError(
'"from" download arg can either have the "s3://" or "gs://" prefixes'
)
if item_name != "":
if download_arg.get("hide_from_log", False):
logger().info("downloading {}".format(item_name))
else:
logger().info("downloading {} from {}".format(
item_name, from_path))
if download_arg.get("to_file", False):
client.download_file(prefix, to_path)
else:
client.download(prefix, to_path)
if download_arg.get("unzip", False):
if item_name != "" and not download_arg.get(
"hide_unzipping_log", False):
logger().info("unzipping {}".format(item_name))
if download_arg.get("to_file", False):
util.extract_zip(to_path, delete_zip_file=True)
else:
util.extract_zip(os.path.join(to_path,
os.path.basename(from_path)),
delete_zip_file=True)
if download_config.get("last_log", "") != "":
logger().info(download_config["last_log"])
def download_and_unzip_external(self, s3_path, local_dir):
util.mkdir_p(local_dir)
local_zip = os.path.join(local_dir, "zip.zip")
self.download_file_external(s3_path, local_zip)
util.extract_zip(local_zip, delete_zip_file=True)