def download_assets(iterable):
client = get_s3_client()
for item in iterable():
if item is not None and not os.path.exists(item.filename):
dirname = os.path.dirname(item.filename)
if not os.path.exists(dirname):
print("Making directories to {}".format(dirname))
os.makedirs(dirname)
def conditionally_download_item(item):
if item is not None and not os.path.exists(item.filename):
print("Downloading asset to {}".format(item.filename))
client.client.download_file(item.bucket, item.path, item.filename)
else:
print("Asset {} already exists, skipping download".format(
item.filename))
thread_pool = concurrent.futures.ThreadPoolExecutor(
max_workers=min(os.cpu_count() * 2, 24))
futures = []
# Download scans in parallel
for item in iterable():
future = thread_pool.submit(conditionally_download_item, item)
futures.append(future)
_ = concurrent.futures.wait(futures)
thread_pool.shutdown()
def load_svg_from_s3(bucket, path):
client = get_s3_client()
s3_bucket = client.s3.Bucket(bucket)
s3_object = s3_bucket.Object(path)
with io.BytesIO(s3_object.get()["Body"].read()) as stream:
return ET.parse(stream)
def load_xml_from_s3(bucket, path):
client = get_s3_client()
s3_bucket = client.s3.Bucket(bucket)
s3_object = s3_bucket.Object(path)
with io.BytesIO(s3_object.get()["Body"].read()) as stream:
with io.TextIOWrapper(stream, encoding="utf-8") as xml_file:
return xmltodict.parse(xml_file.read())
def upload_file(filename, bucket, path, **kwargs):
_, ext = os.path.splitext(path)
if ext in _EXT_METADATA_MAP:
if "extra_args" not in kwargs:
kwargs["extra_args"] = {}
kwargs["extra_args"]["ContentType"] = _EXT_METADATA_MAP[ext]
client = get_s3_client()
client.transfer.upload_file(filename, bucket, path, **kwargs)
def object_exists_s3(bucket, path):
client = get_s3_client()
try:
response = client.client.head_object(Bucket=bucket, Key=path)
return response["ContentLength"] > 0
except botocore.exceptions.ClientError as error:
if error.response["Error"]["Code"] == "404":
# The object does not exist.
print("Warning: s3://{}/{} not found".format(bucket, path))
return False
else:
# Something else has gone wrong.
raise error
def download_file(bucket, path, filename):
client = get_s3_client()
client.client.download_file(bucket, path, filename)