def main(event, context): """ Gets layer arns for each region and publish to S3 """ regions = get_config.get_aws_regions() dynamodb = boto3.resource('dynamodb') table = dynamodb.Table(os.environ['LAYERS_DB']) bucket = os.environ['BUCKET_NAME'] for region in regions: items = query_versions_table(table=table, region=region) arns = json.dumps(items, cls=DecimalEncoder, indent=4) logger.info(f"Uploading to S3 Bucket") client = boto3.client('s3') client.put_object(Body=arns.encode('utf-8'), Bucket=bucket, Key=f'arns/{region}.json') return {"status": "Done"}
def main(event, context): """ Gets layer arns for each region and publish to S3 """ regions = get_config.get_aws_regions() dynamodb = boto3.resource("dynamodb") table = dynamodb.Table(os.environ["DB_NAME"]) bucket = os.environ["BUCKET_NAME"] region_deploy = dict() for region in regions: items = query_table(table=table, region=region) arns = convert_to_csv(items) region_deploy[region] = len(items) logger.info(f"Uploading to S3 Bucket") client = boto3.client("s3") client.put_object( Body=arns.encode("utf-8"), Bucket=bucket, Key=f"arns/{region}.csv" ) return {"arn_count": region_deploy}
def main(event, context): regions = get_config.get_aws_regions() package = event['package'] version = event['version'] build_flag = event['build_flag'] package_artifact = event['zip_file'] requirements_hash = event['requirements_hash'] license_info = event['license_info'] deployed_flag = False layer_name = f"{os.environ['LAMBDA_PREFIX']}{package}" logger.info( f"Downloading {package_artifact} from {os.environ['BUCKET_NAME']}") s3.meta.client.download_file(os.environ['BUCKET_NAME'], package_artifact, f"/tmp/{package_artifact}") with open(f"/tmp/{package_artifact}", 'rb') as zip_file: zip_binary = zip_file.read() logger.info(f"Package {package_artifact} downloaded") logger.info( f"Deploying new version of {layer_name}, {package}=={version}, requirements hash: {requirements_hash}" ) for region in regions: if check_latest_deploy(package=package, region=region, requirements_hash=requirements_hash): # Publish Layer Version logger.info(f"Deploying {layer_name} to {region}") lambda_client = boto3.client('lambda', region_name=region) response = lambda_client.publish_layer_version( LayerName=layer_name, Description=f"{package}=={version} | {requirements_hash}", Content={'ZipFile': zip_binary}, CompatibleRuntimes=['python3.6', 'python3.7'], LicenseInfo=license_info) layer_version_arn = response['LayerVersionArn'] layer_version_created_date = response['CreatedDate'] logger.info( f"Uploaded new version with arn: {layer_version_arn} at {layer_version_created_date}" ) layer_version = int(layer_version_arn.split(":")[-1]) logger.info( f"Layer version for {layer_version_arn} is {layer_version}") try: dynamodb_client.put_item(TableName=os.environ['LAYERS_DB'], Item={ 'deployed_region': { 'S': region }, 'deployed_region-package': { 'S': f"{region}.{package}" }, 'package': { 'S': package }, 'package_version': { 'S': f"{str(version)}" }, 'requirements_hash': { 'S': requirements_hash }, 'created_date': { 'S': layer_version_created_date }, 'layer_version': { 'N': str(layer_version) }, 'layer_version_arn': { 'S': str(layer_version_arn) } }) logger.info( f"Successfully written {package}:{layer_version} status to DB with hash: {requirements_hash}" ) if layer_version > 1: ttl_value = int(time.time() + 24 * 3600 * 30) dynamodb_client.update_item( TableName=os.environ['LAYERS_DB'], Key={ 'deployed_region-package': { 'S': f"{region}.{package}" }, 'layer_version': { 'N': str(layer_version - 1) } }, UpdateExpression='SET time_to_live = :val1', ExpressionAttributeValues={ ':val1': { 'N': str(ttl_value) } }) logger.info( f"Successfully added TTL for 30 days to {region}.{package} version {layer_version-1}" ) else: logger.info( f"No previous version for {region}.{package} found, bypassing delete" ) except ClientError as e: logger.error("Unexpected error Writing to DB: {}".format( e.response['Error']['Code'])) response = lambda_client.add_layer_version_permission( LayerName=layer_name, VersionNumber=layer_version, StatementId='make_public', Action='lambda:GetLayerVersion', Principal='*') logger.info(response['Statement']) deployed_flag = True else: logger.info(f"{region} already has latest version installed...") return { "deployed_flag": deployed_flag, "build_flag": build_flag, "package": package, "version": version, "requirements_hash": requirements_hash }
def main(event, context): regions = get_config.get_aws_regions() package = event["package"] version = event["version"] build_flag = event["build_flag"] package_artifact = event["zip_file"] requirements_hash = event["requirements_hash"] license_info = event["license_info"] table_name = os.environ["DB_NAME"] expiry_days = int(os.environ["EXPIRY_DAYS"]) dynamo_client = boto3.client("dynamodb") deployed_flag = False # Check if need to deploy regions_to_deploy = check_regions_to_deploy(package, requirements_hash, regions) if len(regions_to_deploy) == 0: logger.info({"message": "No new regions to deploy to, terminating!"}) return { "deployed_flag": deployed_flag, "build_flag": build_flag, "package": package, "version": version, "requirements_hash": requirements_hash, } logger.info({ "message": "Regions to deploy", "regions_to_deploy": regions_to_deploy }) # Download Lambda Artifact layer_name = f"{os.environ['LAMBDA_PREFIX']}{package}" zip_binary = download_artifact(package_artifact) # Get requirements txt requirements_txt = get_requirements_txt(package) for region in regions_to_deploy: # Publish Layer Version logger.info({ "message": "Deploying", "region": region, "package": package }) lambda_client = boto3.client("lambda", region_name=region) response = lambda_client.publish_layer_version( LayerName=layer_name, Description=f"{package}=={version} | {requirements_hash}", Content={"ZipFile": zip_binary}, CompatibleRuntimes=["python3.6", "python3.7", "python3.8"], LicenseInfo=license_info, ) layer_version_arn = response["LayerVersionArn"] layer_version_created_date = datetime.utcnow().isoformat() layer_version = int(layer_version_arn.split(":")[-1]) # Make Layer Publicly accessible logger.info({ "message": "Making Public", "region": region, "package": package, "arn": layer_version_arn, "created_date": layer_version_created_date, }) lambda_client.add_layer_version_permission( LayerName=layer_name, VersionNumber=layer_version, StatementId="make_public", Action="lambda:GetLayerVersion", Principal="*", ) # Insert new entry into DynamoDB logger.info({ "message": "Inserting to table", "region": region, "package": package, "arn": layer_version_arn, }) pk = f"lyr#{region}.{package}" sk_v0 = "lyrVrsn0#" sk = f"lyrVrsn#v{layer_version}" sk_previous = f"lyrVrsn#v{layer_version-1}" dynamo_client.transact_write_items(TransactItems=[ { "Update": { "TableName": table_name, "Key": { "pk": { "S": pk }, "sk": { "S": sk_v0 }, }, "UpdateExpression": "set " "rqrmntsTxt = :rqrmntsTxt, " "pckgVrsn = :pckgVrsn, " "rqrmntsHsh = :rqrmntsHsh," "arn = :arn," "crtdDt = :crtdDt," "lyrVrsn = :lyrVrsn", "ExpressionAttributeValues": { ":rqrmntsTxt": { "S": requirements_txt }, ":crtdDt": { "S": layer_version_created_date }, ":pckgVrsn": { "S": version }, ":rqrmntsHsh": { "S": requirements_hash }, ":arn": { "S": layer_version_arn }, ":lyrVrsn": { "N": str(layer_version) }, }, # Allow update only if # Current lyrVrsn is less than updated value # or lyrVrsn doesn't exists "ConditionExpression": "lyrVrsn <= :lyrVrsn OR attribute_not_exists(lyrVrsn)", } }, { "Put": { "TableName": table_name, "Item": { "pk": { "S": pk }, "sk": { "S": sk }, "pckgVrsn": { "S": version }, "crtdDt": { "S": layer_version_created_date }, "rqrmntsTxt": { "S": requirements_txt }, "rqrmntsHsh": { "S": requirements_hash }, "arn": { "S": layer_version_arn }, "pckg": { "S": package }, "rgn": { "S": region }, "dplySts": { "S": "latest" }, "lyrVrsn": { "N": str(layer_version) }, }, } }, ]) if layer_version > 1: logger.info({ "message": "Updating Previous Version", "region": region, "package": package, "arn": layer_version_arn, }) try: dynamo_client.update_item( TableName=table_name, Key={ "pk": { "S": pk }, "sk": { "S": sk_previous } }, UpdateExpression="set " "dplySts = :dplySts, " "exDt = :exDt", ExpressionAttributeValues={ ":dplySts": { "S": "deprecated" }, ":exDt": { "N": str(int(time.time() + 24 * 3600 * expiry_days)) }, }, ConditionExpression="attribute_exists(sk)", ) except ClientError as e: if e.response["Error"][ "Code"] == "ConditionalCheckFailedException": logger.warning({ "message": "Conditional Check failed", "layer_version": layer_version, "sk": sk_previous, }) deployed_flag = True return { "deployed_to": regions_to_deploy, "deployed_flag": deployed_flag, "build_flag": build_flag, "package": package, "version": version, "requirements_hash": requirements_hash, }