def push_s3(target, args):
"""
Push files from a remote task on an AWS S3 bucket
"""
assert args.task_id is not None, "Missing task id"
assert not args.artifact_folder.endswith(
"/"), "Artifact folder {} must not end in /".format(
args.artifact_folder)
# Load config from file/secret
config = Configuration(args)
assert config.has_aws_auth(), "Missing AWS authentication"
# Configure boto3 client
s3 = boto3.client(
"s3",
aws_access_key_id=config.aws["access_key_id"],
aws_secret_access_key=config.aws["secret_access_key"],
)
# Check the bucket is available
try:
s3.head_bucket(Bucket=args.bucket)
logger.info("S3 Bucket {} is available".format(args.bucket))
except botocore.exceptions.ClientError as e:
logger.error("Bucket {} unavailable: {}".format(args.bucket, e))
return
# Load queue service
queue = taskcluster.Queue(config.get_taskcluster_options())
# Download all files from the specified artifact folder
# These files are then uploaded on the bucket, stripping the artifact folder
# from their final path
artifacts = load_artifacts(args.task_id, queue,
"{}/*".format(args.artifact_folder))
for task_id, artifact_name in artifacts:
# Download each artifact
assert artifact_name.startswith(args.artifact_folder)
local_path = download_artifact(queue, task_id, artifact_name)
# Detect mime/type to set valid content-type for web requests
content_type, _ = mimetypes.guess_type(local_path)
# Push that artifact on the S3 bucket, without the artifact folder
s3_path = artifact_name[len(args.artifact_folder) + 1:]
s3.put_object(
Bucket=args.bucket,
Key=s3_path,
Body=open(local_path, "rb"),
ContentType=content_type,
)
logger.info("Uploaded {} as {} on S3".format(s3_path, content_type))
def push_s3(target: Target, args: argparse.Namespace) -> None:
"""
Push files from a remote task on an AWS S3 bucket
"""
assert args.task_id is not None, "Missing task id"
assert not args.artifact_folder.endswith(
"/"), "Artifact folder {} must not end in /".format(
args.artifact_folder)
# Load config from file/secret
config = Configuration(args)
assert config.has_aws_auth(), "Missing AWS authentication"
# Configure boto3 client
s3 = boto3.client(
"s3",
aws_access_key_id=config.aws["access_key_id"],
aws_secret_access_key=config.aws["secret_access_key"],
)
# Check the bucket is available
try:
s3.head_bucket(Bucket=args.bucket)
logger.info("S3 Bucket {} is available".format(args.bucket))
except botocore.exceptions.ClientError as e:
logger.error("Bucket {} unavailable: {}".format(args.bucket, e))
return
# Load queue service
queue = taskcluster.Queue(config.get_taskcluster_options())
# Download all files from the specified artifact folder
# These files are then uploaded on the bucket, stripping the artifact folder
# from their final path
artifacts = load_artifacts(args.task_id, queue,
"{}/*".format(args.artifact_folder))
for task_id, artifact_name in artifacts:
# Download each artifact
assert artifact_name.startswith(args.artifact_folder)
local_path = download_artifact(queue, task_id, artifact_name)
# Detect mime/type to set valid content-type for web requests
content_type, _ = mimetypes.guess_type(local_path)
if content_type is None:
# Use a default content type to avoid crashes on upload
# when a file's MIME type is not detected
content_type = "text/plain"
# Push that artifact on the S3 bucket, without the artifact folder
s3_path = artifact_name[len(args.artifact_folder) + 1:]
s3.put_object(
Bucket=args.bucket,
Key=s3_path,
Body=open(local_path, "rb"),
ContentType=content_type,
)
logger.info("Uploaded {} as {} on S3".format(s3_path, content_type))
cloudfront_distribution_id = config.aws.get("cloudfront_distribution_id")
if cloudfront_distribution_id is not None:
cloudfront_client = boto3.client(
"cloudfront",
aws_access_key_id=config.aws["access_key_id"],
aws_secret_access_key=config.aws["secret_access_key"],
)
cloudfront_client.create_invalidation(
DistributionId=cloudfront_distribution_id,
InvalidationBatch={
"Paths": {
"Quantity": 1,
"Items": [
"/*",
],
},
"CallerReference": str(int(datetime.utcnow().timestamp())),
},
)
logger.info("Cloudfront invalidation created")
