def get_spec(
provider: str,
spec_path: str,
cache_dir: str,
region: Optional[str] = None,
spec_name: str = "api_spec.json",
) -> Tuple[Union[S3, GCS], dict]:
"""
Args:
provider: "aws" or "gcp".
spec_path: Path to API spec (i.e. "s3://cortex-dev-0/apis/iris-classifier/api/69b93378fa5c0218-jy1fjtyihu-9fcc10739e7fc8050cefa8ca27ece1ee/master-spec.json").
cache_dir: Local directory where the API spec gets saved to.
region: Region of the bucket. Only required for "S3" provider.
spec_name: File name of the spec as it is saved on disk.
"""
if provider == "aws":
bucket, key = S3.deconstruct_s3_path(spec_path)
storage = S3(bucket=bucket, region=region)
elif provider == "gcp":
bucket, key = GCS.deconstruct_gcs_path(spec_path)
storage = GCS(bucket=bucket)
else:
raise ValueError('invalid "provider" argument')
local_spec_path = os.path.join(cache_dir, spec_name)
if not os.path.isfile(local_spec_path):
storage.download_file(key, local_spec_path)
return storage, read_json(local_spec_path)
def start(args):
download_config = json.loads(base64.urlsafe_b64decode(args.download))
for download_arg in download_config["download_args"]:
from_path = download_arg["from"]
to_path = download_arg["to"]
item_name = download_arg.get("item_name", "")
if from_path.startswith("s3://"):
bucket_name, prefix = S3.deconstruct_s3_path(from_path)
client = S3(bucket_name, client_config={})
elif from_path.startswith("gs://"):
bucket_name, prefix = GCS.deconstruct_gcs_path(from_path)
client = GCS(bucket_name)
else:
raise ValueError(
'"from" download arg can either have the "s3://" or "gs://" prefixes'
)
if item_name != "":
if download_arg.get("hide_from_log", False):
logger().info("downloading {}".format(item_name))
else:
logger().info("downloading {} from {}".format(
item_name, from_path))
if download_arg.get("to_file", False):
client.download_file(prefix, to_path)
else:
client.download(prefix, to_path)
if download_arg.get("unzip", False):
if item_name != "" and not download_arg.get(
"hide_unzipping_log", False):
logger().info("unzipping {}".format(item_name))
if download_arg.get("to_file", False):
util.extract_zip(to_path, delete_zip_file=True)
else:
util.extract_zip(os.path.join(to_path,
os.path.basename(from_path)),
delete_zip_file=True)
if download_config.get("last_log", "") != "":
logger().info(download_config["last_log"])