def main(event, context):
"""
Gets layer arns for each region and publish to S3
"""
regions = get_config.get_aws_regions()
dynamodb = boto3.resource('dynamodb')
table = dynamodb.Table(os.environ['LAYERS_DB'])
bucket = os.environ['BUCKET_NAME']
for region in regions:
items = query_versions_table(table=table,
region=region)
arns = json.dumps(items, cls=DecimalEncoder, indent=4)
logger.info(f"Uploading to S3 Bucket")
client = boto3.client('s3')
client.put_object(Body=arns.encode('utf-8'),
Bucket=bucket,
Key=f'arns/{region}.json')
return {"status": "Done"}
def main(event, context):
"""
Gets layer arns for each region and publish to S3
"""
regions = get_config.get_aws_regions()
dynamodb = boto3.resource("dynamodb")
table = dynamodb.Table(os.environ["DB_NAME"])
bucket = os.environ["BUCKET_NAME"]
region_deploy = dict()
for region in regions:
items = query_table(table=table, region=region)
arns = convert_to_csv(items)
region_deploy[region] = len(items)
logger.info(f"Uploading to S3 Bucket")
client = boto3.client("s3")
client.put_object(
Body=arns.encode("utf-8"), Bucket=bucket, Key=f"arns/{region}.csv"
)
return {"arn_count": region_deploy}
def main(event, context):
regions = get_config.get_aws_regions()
package = event['package']
version = event['version']
build_flag = event['build_flag']
package_artifact = event['zip_file']
requirements_hash = event['requirements_hash']
license_info = event['license_info']
deployed_flag = False
layer_name = f"{os.environ['LAMBDA_PREFIX']}{package}"
logger.info(
f"Downloading {package_artifact} from {os.environ['BUCKET_NAME']}")
s3.meta.client.download_file(os.environ['BUCKET_NAME'], package_artifact,
f"/tmp/{package_artifact}")
with open(f"/tmp/{package_artifact}", 'rb') as zip_file:
zip_binary = zip_file.read()
logger.info(f"Package {package_artifact} downloaded")
logger.info(
f"Deploying new version of {layer_name}, {package}=={version}, requirements hash: {requirements_hash}"
)
for region in regions:
if check_latest_deploy(package=package,
region=region,
requirements_hash=requirements_hash):
# Publish Layer Version
logger.info(f"Deploying {layer_name} to {region}")
lambda_client = boto3.client('lambda', region_name=region)
response = lambda_client.publish_layer_version(
LayerName=layer_name,
Description=f"{package}=={version} | {requirements_hash}",
Content={'ZipFile': zip_binary},
CompatibleRuntimes=['python3.6', 'python3.7'],
LicenseInfo=license_info)
layer_version_arn = response['LayerVersionArn']
layer_version_created_date = response['CreatedDate']
logger.info(
f"Uploaded new version with arn: {layer_version_arn} at {layer_version_created_date}"
)
layer_version = int(layer_version_arn.split(":")[-1])
logger.info(
f"Layer version for {layer_version_arn} is {layer_version}")
try:
dynamodb_client.put_item(TableName=os.environ['LAYERS_DB'],
Item={
'deployed_region': {
'S': region
},
'deployed_region-package': {
'S': f"{region}.{package}"
},
'package': {
'S': package
},
'package_version': {
'S': f"{str(version)}"
},
'requirements_hash': {
'S': requirements_hash
},
'created_date': {
'S':
layer_version_created_date
},
'layer_version': {
'N': str(layer_version)
},
'layer_version_arn': {
'S': str(layer_version_arn)
}
})
logger.info(
f"Successfully written {package}:{layer_version} status to DB with hash: {requirements_hash}"
)
if layer_version > 1:
ttl_value = int(time.time() + 24 * 3600 * 30)
dynamodb_client.update_item(
TableName=os.environ['LAYERS_DB'],
Key={
'deployed_region-package': {
'S': f"{region}.{package}"
},
'layer_version': {
'N': str(layer_version - 1)
}
},
UpdateExpression='SET time_to_live = :val1',
ExpressionAttributeValues={
':val1': {
'N': str(ttl_value)
}
})
logger.info(
f"Successfully added TTL for 30 days to {region}.{package} version {layer_version-1}"
)
else:
logger.info(
f"No previous version for {region}.{package} found, bypassing delete"
)
except ClientError as e:
logger.error("Unexpected error Writing to DB: {}".format(
e.response['Error']['Code']))
response = lambda_client.add_layer_version_permission(
LayerName=layer_name,
VersionNumber=layer_version,
StatementId='make_public',
Action='lambda:GetLayerVersion',
Principal='*')
logger.info(response['Statement'])
deployed_flag = True
else:
logger.info(f"{region} already has latest version installed...")
return {
"deployed_flag": deployed_flag,
"build_flag": build_flag,
"package": package,
"version": version,
"requirements_hash": requirements_hash
}
def main(event, context):
regions = get_config.get_aws_regions()
package = event["package"]
version = event["version"]
build_flag = event["build_flag"]
package_artifact = event["zip_file"]
requirements_hash = event["requirements_hash"]
license_info = event["license_info"]
table_name = os.environ["DB_NAME"]
expiry_days = int(os.environ["EXPIRY_DAYS"])
dynamo_client = boto3.client("dynamodb")
deployed_flag = False
# Check if need to deploy
regions_to_deploy = check_regions_to_deploy(package, requirements_hash,
regions)
if len(regions_to_deploy) == 0:
logger.info({"message": "No new regions to deploy to, terminating!"})
return {
"deployed_flag": deployed_flag,
"build_flag": build_flag,
"package": package,
"version": version,
"requirements_hash": requirements_hash,
}
logger.info({
"message": "Regions to deploy",
"regions_to_deploy": regions_to_deploy
})
# Download Lambda Artifact
layer_name = f"{os.environ['LAMBDA_PREFIX']}{package}"
zip_binary = download_artifact(package_artifact)
# Get requirements txt
requirements_txt = get_requirements_txt(package)
for region in regions_to_deploy:
# Publish Layer Version
logger.info({
"message": "Deploying",
"region": region,
"package": package
})
lambda_client = boto3.client("lambda", region_name=region)
response = lambda_client.publish_layer_version(
LayerName=layer_name,
Description=f"{package}=={version} | {requirements_hash}",
Content={"ZipFile": zip_binary},
CompatibleRuntimes=["python3.6", "python3.7", "python3.8"],
LicenseInfo=license_info,
)
layer_version_arn = response["LayerVersionArn"]
layer_version_created_date = datetime.utcnow().isoformat()
layer_version = int(layer_version_arn.split(":")[-1])
# Make Layer Publicly accessible
logger.info({
"message": "Making Public",
"region": region,
"package": package,
"arn": layer_version_arn,
"created_date": layer_version_created_date,
})
lambda_client.add_layer_version_permission(
LayerName=layer_name,
VersionNumber=layer_version,
StatementId="make_public",
Action="lambda:GetLayerVersion",
Principal="*",
)
# Insert new entry into DynamoDB
logger.info({
"message": "Inserting to table",
"region": region,
"package": package,
"arn": layer_version_arn,
})
pk = f"lyr#{region}.{package}"
sk_v0 = "lyrVrsn0#"
sk = f"lyrVrsn#v{layer_version}"
sk_previous = f"lyrVrsn#v{layer_version-1}"
dynamo_client.transact_write_items(TransactItems=[
{
"Update": {
"TableName":
table_name,
"Key": {
"pk": {
"S": pk
},
"sk": {
"S": sk_v0
},
},
"UpdateExpression":
"set "
"rqrmntsTxt = :rqrmntsTxt, "
"pckgVrsn = :pckgVrsn, "
"rqrmntsHsh = :rqrmntsHsh,"
"arn = :arn,"
"crtdDt = :crtdDt,"
"lyrVrsn = :lyrVrsn",
"ExpressionAttributeValues": {
":rqrmntsTxt": {
"S": requirements_txt
},
":crtdDt": {
"S": layer_version_created_date
},
":pckgVrsn": {
"S": version
},
":rqrmntsHsh": {
"S": requirements_hash
},
":arn": {
"S": layer_version_arn
},
":lyrVrsn": {
"N": str(layer_version)
},
},
# Allow update only if
# Current lyrVrsn is less than updated value
# or lyrVrsn doesn't exists
"ConditionExpression":
"lyrVrsn <= :lyrVrsn OR attribute_not_exists(lyrVrsn)",
}
},
{
"Put": {
"TableName": table_name,
"Item": {
"pk": {
"S": pk
},
"sk": {
"S": sk
},
"pckgVrsn": {
"S": version
},
"crtdDt": {
"S": layer_version_created_date
},
"rqrmntsTxt": {
"S": requirements_txt
},
"rqrmntsHsh": {
"S": requirements_hash
},
"arn": {
"S": layer_version_arn
},
"pckg": {
"S": package
},
"rgn": {
"S": region
},
"dplySts": {
"S": "latest"
},
"lyrVrsn": {
"N": str(layer_version)
},
},
}
},
])
if layer_version > 1:
logger.info({
"message": "Updating Previous Version",
"region": region,
"package": package,
"arn": layer_version_arn,
})
try:
dynamo_client.update_item(
TableName=table_name,
Key={
"pk": {
"S": pk
},
"sk": {
"S": sk_previous
}
},
UpdateExpression="set "
"dplySts = :dplySts, "
"exDt = :exDt",
ExpressionAttributeValues={
":dplySts": {
"S": "deprecated"
},
":exDt": {
"N":
str(int(time.time() + 24 * 3600 * expiry_days))
},
},
ConditionExpression="attribute_exists(sk)",
)
except ClientError as e:
if e.response["Error"][
"Code"] == "ConditionalCheckFailedException":
logger.warning({
"message": "Conditional Check failed",
"layer_version": layer_version,
"sk": sk_previous,
})
deployed_flag = True
return {
"deployed_to": regions_to_deploy,
"deployed_flag": deployed_flag,
"build_flag": build_flag,
"package": package,
"version": version,
"requirements_hash": requirements_hash,
}