async def upload_to_s3(context, s3_key, path): product = get_product_name(context.release_props["appName"].lower(), context.release_props["stage_platform"]) mime_type = mimetypes.guess_type(path)[0] if not mime_type: raise ScriptWorkerTaskException( "Unable to discover valid mime-type for path ({}), " "mimetypes.guess_type() returned {}".format(path, mime_type)) api_kwargs = { "Bucket": get_bucket_name(context, product), "Key": s3_key, "ContentType": mime_type } headers = { "Content-Type": mime_type, "Cache-Control": "public, max-age=%d" % CACHE_CONTROL_MAXAGE } creds = context.config["bucket_config"][context.bucket]["credentials"] s3 = boto3.client("s3", aws_access_key_id=creds["id"], aws_secret_access_key=creds["key"]) url = s3.generate_presigned_url("put_object", api_kwargs, ExpiresIn=1800, HttpMethod="PUT") log.info("upload_to_s3: %s -> s3://%s/%s", path, api_kwargs.get("Bucket"), s3_key) await retry_async(put, args=(context, url, headers, path), retry_exceptions=(Exception, ), kwargs={"session": context.session})
async def upload_to_s3(context, s3_key, path): product = get_product_name(context.release_props['appName'].lower(), context.release_props['stage_platform']) mime_type = mimetypes.guess_type(path)[0] if not mime_type: raise ScriptWorkerTaskException("Unable to discover valid mime-type for path ({}), " "mimetypes.guess_type() returned {}".format( path, mime_type )) api_kwargs = { 'Bucket': get_bucket_name(context, product), 'Key': s3_key, 'ContentType': mime_type, } headers = { 'Content-Type': mime_type, 'Cache-Control': 'public, max-age=%d' % CACHE_CONTROL_MAXAGE, } creds = context.config['bucket_config'][context.bucket]['credentials'] s3 = boto3.client('s3', aws_access_key_id=creds['id'], aws_secret_access_key=creds['key'],) url = s3.generate_presigned_url('put_object', api_kwargs, ExpiresIn=1800, HttpMethod='PUT') log.info("upload_to_s3: %s -> s3://%s/%s", path, api_kwargs.get('Bucket'), s3_key) await retry_async(put, args=(context, url, headers, path), retry_exceptions=(Exception, ), kwargs={'session': context.session})
async def push_to_releases(context): """Copy artifacts from one S3 location to another. Determine the list of artifacts to be copied and transfer them. These copies happen in S3 without downloading/reuploading.""" context.artifacts_to_beetmove = {} product = context.task["payload"]["product"] build_number = context.task["payload"]["build_number"] version = context.task["payload"]["version"] context.bucket_name = get_bucket_name(context, product) candidates_prefix = get_candidates_prefix(product, version, build_number) releases_prefix = get_releases_prefix(product, version) creds = get_creds(context) s3_resource = boto3.resource("s3", aws_access_key_id=creds["id"], aws_secret_access_key=creds["key"]) candidates_keys_checksums = list_bucket_objects(context, s3_resource, candidates_prefix) releases_keys_checksums = list_bucket_objects(context, s3_resource, releases_prefix) if not candidates_keys_checksums: raise ScriptWorkerTaskException( "No artifacts to copy from {} so there is no reason to continue.". format(candidates_prefix)) if releases_keys_checksums: log.warning("Destination {} already exists with {} keys".format( releases_prefix, len(releases_keys_checksums))) # Weed out RELEASE_EXCLUDE matches, but allow partners specified in the payload push_partners = context.task["payload"].get("partners", []) for k in candidates_keys_checksums.keys(): if "/partner-repacks/" in k: partner_match = get_partner_match(k, candidates_prefix, push_partners) if partner_match: context.artifacts_to_beetmove[k] = k.replace( get_partner_candidates_prefix(candidates_prefix, partner_match), get_partner_releases_prefix(product, version, partner_match)) else: log.debug("Excluding partner repack {}".format(k)) elif not matches_exclude(k, RELEASE_EXCLUDE): context.artifacts_to_beetmove[k] = k.replace( candidates_prefix, releases_prefix) else: log.debug("Excluding {}".format(k)) copy_beets(context, candidates_keys_checksums, releases_keys_checksums)
async def push_to_releases(context): """Copy artifacts from one S3 location to another. Determine the list of artifacts to be copied and transfer them. These copies happen in S3 without downloading/reuploading.""" context.artifacts_to_beetmove = {} product = context.task['payload']['product'] build_number = context.task['payload']['build_number'] version = context.task['payload']['version'] context.bucket_name = get_bucket_name(context, product) candidates_prefix = get_candidates_prefix(product, version, build_number) releases_prefix = get_releases_prefix(product, version) creds = get_creds(context) s3_resource = boto3.resource('s3', aws_access_key_id=creds['id'], aws_secret_access_key=creds['key']) candidates_keys_checksums = list_bucket_objects(context, s3_resource, candidates_prefix) releases_keys_checksums = list_bucket_objects(context, s3_resource, releases_prefix) if not candidates_keys_checksums: raise ScriptWorkerTaskException( "No artifacts to copy from {} so there is no reason to continue.". format(candidates_prefix)) if releases_keys_checksums: log.warning("Destination {} already exists with {} keys".format( releases_prefix, len(releases_keys_checksums))) # Weed out RELEASE_EXCLUDE matches for k in candidates_keys_checksums.keys(): if not matches_exclude(k, RELEASE_EXCLUDE): context.artifacts_to_beetmove[k] = k.replace( candidates_prefix, releases_prefix) else: log.debug("Excluding {}".format(k)) copy_beets(context, candidates_keys_checksums, releases_keys_checksums)