def start_janis_builder_factory(build_id):
queue_table = get_dynamodb_table()
build_pk, build_sk = parse_build_id(build_id)
queue_table.update_item(
Key={
'pk': build_pk,
'sk': build_sk,
},
UpdateExpression="SET stage = :stage, build_type = :build_type",
ExpressionAttributeValues={
":stage": stages.janis_builder_factory,
# ensure that build_type=rebuild.
# If an "incremental" or "all_pages" BLD did find a janis_builder_task to run, then a "rebuild" is necessary
':build_type': "rebuild",
},
)
# Start CodeBuild Project
codebuild = boto3.client('codebuild')
build_item = get_build_item(build_id)
codebuild.start_build(
projectName=
f'coa-publisher-janis-builder-factory-{os.getenv("DEPLOY_ENV")}',
environmentVariablesOverride=get_janis_builder_factory_env_vars(
build_item),
)
print(f"##### Starting janis_builder_factory for [{build_id}]")
def handler(event, context):
queue_table = get_dynamodb_table()
sns_detail = json.loads(event["Records"][0]["Sns"]['Message'])['detail']
for env_var in sns_detail['additional-information']['environment'][
'environment-variables']:
if (env_var['name'] == 'BUILD_ID'):
build_id = env_var['value']
break
build_pk, build_sk = parse_build_id(build_id)
janis_branch = get_janis_branch(build_id)
print(f"##### janis_builder_factory stage finished for [{build_id}].")
try:
# Update the logs for that BLD
lambda_cloudwatch_url = get_lambda_cloudwatch_url(context)
project_name = sns_detail['project-name']
stream_name = sns_detail['additional-information']['logs'][
'stream-name']
codebuild_url = f'https://console.aws.amazon.com/codesuite/codebuild/projects/{project_name}/build/{project_name}%3A{stream_name}/log?region={os.getenv("AWS_REGION")}'
queue_table.update_item(
Key={
'pk': build_pk,
'sk': build_sk,
},
UpdateExpression=
"SET stage = :stage, logs = list_append(if_not_exists(logs, :empty_list), :logs)",
ExpressionAttributeValues={
":stage":
stages.final,
":logs": [{
'stage': stages.janis_builder_factory,
'url': codebuild_url,
}, {
'stage': stages.final,
'url': lambda_cloudwatch_url,
}],
":empty_list": [],
},
)
build_status = sns_detail['build-status']
if ((build_status == "STOPPED") or (build_status == "FAILED")):
print(
f"##### Failure: janis_builder_factory did not succeed for [{build_id}]."
)
process_build_failure(build_id, context)
elif (build_status == "SUCCEEDED"):
register_janis_builder_task(janis_branch)
process_build_success(build_id, context)
except Exception as error:
import traceback
print(traceback.format_exc())
process_build_failure(build_id, context)
def handler(event, context):
queue_table = get_dynamodb_table()
sns_detail = json.loads(event["Records"][0]["Sns"]['Message'])['detail']
# Someday we might be listening for other tasks besides "janis-builder".
# But for now, "family:janis-builder-{janis_branch}" is the only group of tasks we're running
if sns_detail["group"].startswith("family:janis-builder"):
for env_var in sns_detail["overrides"]["containerOverrides"][0][
"environment"]:
if (env_var['name'] == 'BUILD_ID'):
build_id = env_var['value']
break
print(f"##### janis_builder fargate task finished for [{build_id}].")
build_pk, build_sk = parse_build_id(build_id)
lambda_cloudwatch_url = get_lambda_cloudwatch_url(context)
try:
queue_table.update_item(
Key={
'pk': build_pk,
'sk': build_sk,
},
UpdateExpression=
"SET stage = :stage, logs = list_append(if_not_exists(logs, :empty_list), :logs)",
ExpressionAttributeValues={
":stage": stages.final,
":logs": [{
'stage': stages.final,
'url': lambda_cloudwatch_url,
}],
":empty_list": [],
},
)
exit_code = sns_detail["containers"][0]["exitCode"]
if exit_code == 0:
process_build_success(build_id, context)
else:
print(
f"##### Failure: janis_builder exited with nonzero exit code for [{build_id}]."
)
process_build_failure(build_id, context)
except Exception as error:
import traceback
print(traceback.format_exc())
process_build_failure(build_id, context)
def handler(event, context):
queue_table = get_dynamodb_table()
timestamp = get_datetime()
# Validate body
body = event.get("body")
if not body:
return failure_res("No 'body' passed with request.")
data = json.loads(body)
# Validate joplin_appname
joplin = data.get("joplin_appname")
if not joplin:
return failure_res("joplin_appname is required")
# Validate janis_branch
janis_branch = data.get("janis_branch")
if not janis_branch:
return failure_res("janis_branch is required")
if janis_branch == staging_janis_branch:
if not is_staging():
return failure_res(
"Can only deploy to staging Janis from 'staging' Publisher.")
if joplin != staging_joplin_appname:
return failure_res(
f"Can only deploy to staging Janis from {staging_joplin_appname}."
)
if janis_branch == production_janis_branch:
if not is_production():
return failure_res(
"Can only deploy to production Janis from 'production' Publisher."
)
if joplin != production_joplin_appname:
return failure_res(
f"Can only deploy to production Janis from {production_joplin_appname}."
)
# The DEPLOY_ENV determines custom deployment logic.
# Make sure that the right Janis is being deployed to Production or Staging.
if is_staging() and (janis_branch != staging_janis_branch):
return failure_res(
"'staging' Publisher can only deploy to staging Janis.")
if is_production() and (janis_branch != production_janis_branch):
return failure_res(
"'production' Publisher can only deploy to production Janis.")
pk = f'REQ#{janis_branch}'
sk = timestamp
status = f'waiting#{timestamp}'
# Validate build_type
build_type = data.get("build_type")
valid_build_types = [
"rebuild",
"incremental",
"all_pages",
]
if (not build_type or build_type not in valid_build_types):
return failure_res(f'[{build_type}] is not a valid build_type.')
# Validate pages
api_key = data.get("api_key")
req_pages = data.get("pages")
if not req_pages:
pages = []
else:
if not isinstance(req_pages, list):
return failure_res(f'pages must be a list.')
if has_empty_strings(req_pages):
return failure_res(f'Empty strings are not allowed in pages.')
for page in req_pages:
page["timestamp"] = timestamp
# The api_key is used to send a response back to Joplin on publish success.
# Not required for all requests, only ones that have specific pages that must be updated.
if not api_key:
return failure_res(
"api_key is required when updating specific pages")
pages = req_pages
# Validate env_vars
req_env_vars = data.get("env_vars")
env_vars = {}
if req_env_vars:
if not isinstance(req_env_vars, dict):
return failure_res(f'env_vars must be a dict.')
if has_empty_strings(req_env_vars):
return failure_res(f'Empty strings are not allowed in env_vars.')
for name, value in req_env_vars.items():
if name not in valid_optional_env_vars:
return failure_res(
f'env_var {name} is not a valid_optional_env_var.')
env_vars = req_env_vars
queue_table.put_item(
Item={
'pk': pk,
'sk': sk,
'status': status,
'pages': pages,
'joplin': joplin,
'env_vars': env_vars,
'build_type': build_type,
'api_key': api_key,
})
print(f"##### Submitted Publish Request pk={pk}, sk={sk}")
return {
"statusCode": 200,
'headers': {
'Content-Type': 'application/json'
},
"body": json.dumps({
"pk": pk,
"sk": sk
})
}
def run_janis_builder_task(build_item, latest_task_definition):
ecs_client = boto3.client('ecs')
queue_table = get_dynamodb_table()
build_id = build_item['build_id']
build_pk, build_sk = parse_build_id(build_id)
janis_branch = get_janis_branch(build_id)
api_password, api_username = get_api_credentials()
# Start the Task
print(f'##### Running janis_builder task for [{build_id}]')
task_env_vars = [{
'name': 'BUILD_TYPE',
'value': build_item['build_type'],
}, {
'name': 'CMS_API',
'value': get_cms_api_url(build_item['joplin']),
}, {
'name':
'PAGES',
'value':
json.dumps([str(page) for page in build_item["pages"]]),
}, {
'name': 'BUILD_ID',
'value': build_id,
}, {
'name': 'API_PASSWORD',
'value': api_password,
}, {
'name': 'API_USERNAME',
'value': api_username,
}]
for name, value in build_item["env_vars"].items():
if name in valid_optional_env_vars:
task_env_vars.append({'name': name, 'value': value})
task = ecs_client.run_task(
cluster=os.getenv("ECS_CLUSTER"),
taskDefinition=latest_task_definition,
launchType='FARGATE',
count=1,
platformVersion='LATEST',
networkConfiguration={
"awsvpcConfiguration": {
"subnets": [
os.getenv("PUBLIC_SUBNET_ONE"),
os.getenv("PUBLIC_SUBNET_TWO"),
],
"securityGroups": [
os.getenv("CLUSTER_SECURITY_GROUP"),
],
'assignPublicIp':
'ENABLED'
}
},
overrides={
'containerOverrides': [{
'name': f'janis-builder-{janis_branch}',
'environment': task_env_vars,
}]
},
)
# Update the logs for your BLD
task_id = (task['tasks'][0]['taskArn']).split('task/')[1]
queue_table.update_item(
Key={
'pk': build_pk,
'sk': build_sk,
},
UpdateExpression=
"SET stage = :stage, logs = list_append(if_not_exists(logs, :empty_list), :logs)",
ExpressionAttributeValues={
":stage":
stages.run_janis_builder_task,
":logs": [{
'stage':
stages.run_janis_builder_task,
'url':
f'https://console.aws.amazon.com/ecs/home?region={os.getenv("AWS_REGION")}#/clusters/{os.getenv("ECS_CLUSTER")}/tasks/{task_id}/details',
}],
":empty_list": [],
},
)
def process_build_success(build_id, context):
queue_table = get_dynamodb_table()
client = get_dynamodb_client()
timestamp = get_datetime()
build_item = get_build_item(build_id)
janis_branch = get_janis_branch(build_id)
if build_item["status"] != "building":
print(
f'##### Successful build for [{build_id}] has already been handled'
)
return None
build_pk = build_item["pk"]
build_sk = build_item["sk"]
start_build_time = dateutil.parser.parse(build_item["sk"])
end_build_time = dateutil.parser.parse(timestamp)
total_build_time = str(end_build_time - start_build_time)
send_publish_succeeded_message(build_item)
req_pk = f'REQ#{janis_branch}'
assinged_reqs = queue_table.query(
IndexName="build_id.janis",
Select='ALL_ATTRIBUTES',
ScanIndexForward=True,
KeyConditionExpression=Key('build_id').eq(build_id)
& Key('pk').eq(req_pk))['Items']
write_item_batches = []
write_item_batch = []
updated_current_build_item = {
"Update": {
"TableName": table_name,
"Key": {
"pk": "CURRENT_BLD",
"sk": janis_branch,
},
"UpdateExpression": "REMOVE build_id",
"ConditionExpression": "build_id = :build_id",
"ExpressionAttributeValues": {
":build_id": build_id,
},
"ReturnValuesOnConditionCheckFailure": "ALL_OLD",
}
}
updated_build_item = {
"Update": {
"TableName": table_name,
"Key": {
"pk": build_pk,
"sk": build_sk,
},
"UpdateExpression":
"SET #s = :status, total_build_time = :total_build_time",
"ExpressionAttributeValues": {
":status": f'succeeded#{timestamp}',
":total_build_time": total_build_time,
},
"ExpressionAttributeNames": {
"#s": "status"
},
"ReturnValuesOnConditionCheckFailure": "ALL_OLD",
}
}
write_item_batch.append(updated_current_build_item)
write_item_batch.append(updated_build_item)
for req in assinged_reqs:
# transact_write_items() allows a maximum of 25 TransactItems.
# If there are more than 25 items, then start a new batch.
if len(write_item_batch) >= 25:
write_item_batches.append(write_item_batch)
write_item_batch = []
# reqs must be updated to remove build_id, and reset to status="waiting#{original timestamp}"
updated_req_item = {
"Update": {
"TableName": table_name,
"Key": {
"pk": req["pk"],
"sk": req["sk"],
},
"UpdateExpression": "SET #s = :status",
"ExpressionAttributeNames": {
"#s": "status"
},
"ExpressionAttributeValues": {
":status": f'succeeded#{timestamp}',
},
}
}
write_item_batch.append(updated_req_item)
write_item_batches.append(write_item_batch)
for write_item_batch in write_item_batches:
client.transact_write_items(TransactItems=write_item_batch)
print(f'##### Successful build for [{build_id}] complete.')
start_new_build(janis_branch, context)
def start_new_build(janis_branch, context):
queue_table = get_dynamodb_table()
client = get_dynamodb_client()
timestamp = get_datetime()
build_item = get_current_build_item(janis_branch)
if build_item:
print(
f"##### No new build started. There is already a current build running for [{janis_branch}]."
)
return None
# Construct a build out of the waiting requests for a janis.
# Get all requests that have a "waiting" attribute in reverse chronological order.
# More recent request config values take precedence over old requests
# (in terms of values for "joplin" and "env_vars").
# There would only be conflicts for "joplin" values on PR builds,
# where multiple joplins could potentially update the same janis instance.
req_pk = f'REQ#{janis_branch}'
waiting_reqs = queue_table.query(
IndexName="janis.status",
Select='ALL_ATTRIBUTES',
ScanIndexForward=
False, # Return reqests in reverse chronological order (most recent first)
KeyConditionExpression=Key('pk').eq(req_pk)
& Key('status').begins_with('waiting'))['Items']
if not len(waiting_reqs):
print(
f"##### No new build started. No requests to process for {janis_branch}."
)
return None
build_id = f'BLD#{janis_branch}#{timestamp}'
build_pk = f'BLD#{janis_branch}'
build_config = {
"pk":
build_pk,
"sk":
timestamp,
"build_id":
build_id,
"status":
"building",
"stage":
stages.preparing_to_build,
"build_type":
None,
"joplin":
None,
"pages": [],
"env_vars": {},
"api_keys": [],
"logs": [{
'stage': stages.preparing_to_build,
'url': get_lambda_cloudwatch_url(context),
}],
}
updated_req_configs = []
# Construct build_config based on values from requests
# Modify reqs with updated data
for req in waiting_reqs:
updated_req = {
"pk": req["pk"],
"sk": req["sk"],
}
updated_req_configs.append(updated_req)
# Handle "joplin" attribute
if not build_config["joplin"]:
# The most recent request will set the value of "joplin" for the build
build_config["joplin"] = req["joplin"]
else:
# If req is for a different joplin, then cancel it.
if req["joplin"] != build_config["joplin"]:
updated_req["canceled_by"] = build_id
# A "cancelled" req will not have a "build_id" attribute assigned to it
# And the data from a "cancelled" req should not be added to the build_config
continue
updated_req["build_id"] = build_id
if req["api_key"]:
build_config["api_keys"].append(req["api_key"])
# Handle "env_vars" attribute
for env_var in req["env_vars"]:
# Only add env_var value if it hasn't already been added.
# Otherwise more recent request env_vars would be overwritten by older requests.
if env_var not in build_config["env_vars"]:
build_config["env_vars"][env_var] = req["env_vars"][env_var]
# Handle "build_type" attribute
if req["build_type"] == "rebuild":
build_config["build_type"] = "rebuild"
elif ((req["build_type"] == "all_pages")
and (build_config["build_type"] != "rebuild")):
build_config["build_type"] = "all_pages"
elif ((req["build_type"] == "incremental")
and (build_config["build_type"] != "rebuild")
and (build_config["build_type"] != "all_pages")):
build_config["build_type"] = "incremental"
# Add all "pages" from request
for page in req["pages"]:
page["req_pk"] = req["pk"]
page["req_sk"] = req["sk"]
build_config["pages"].append(page)
write_item_batches = []
write_item_batch = []
new_current_build_item = {
"Put": {
"TableName": table_name,
"Item": {
"pk": "CURRENT_BLD",
"sk": janis_branch,
"build_id": build_id,
},
# ConditionExpression makes sure that there isn't another build process already running for the same janis_branch
"ConditionExpression": "attribute_not_exists(build_id)",
"ReturnValuesOnConditionCheckFailure": "ALL_OLD",
}
}
new_build_item = {
"Put": {
"TableName": table_name,
"Item": build_config,
"ReturnValuesOnConditionCheckFailure": "ALL_OLD",
}
}
write_item_batch.append(new_current_build_item)
write_item_batch.append(new_build_item)
for updated_req in updated_req_configs:
# transact_write_items() allows a maximum of 25 TransactItems.
# If there are more than 25 items, then start a new batch.
if len(write_item_batch) >= 25:
write_item_batches.append(write_item_batch)
write_item_batch = []
UpdateExpression = "SET #s = :status"
ExpressionAttributeNames = {
"#s": "status"
} # because "status" is a reserved word, you can't explicitly use it in an UpdateExpression
ExpressionAttributeValues = {}
if "canceled_by" in updated_req:
UpdateExpression = UpdateExpression + ", canceled_by = :canceled_by"
ExpressionAttributeValues[":status"] = f'cancelled#{timestamp}'
ExpressionAttributeValues[":canceled_by"] = updated_req[
"canceled_by"]
if "build_id" in updated_req:
UpdateExpression = UpdateExpression + ", build_id = :build_id"
ExpressionAttributeValues[":status"] = f'assigned#{timestamp}'
ExpressionAttributeValues[":build_id"] = updated_req["build_id"]
updated_req_item = {
"Update": {
"TableName": table_name,
"Key": {
"pk": updated_req["pk"],
"sk": updated_req["sk"],
},
"UpdateExpression": UpdateExpression,
"ExpressionAttributeNames": ExpressionAttributeNames,
"ExpressionAttributeValues": ExpressionAttributeValues,
"ReturnValuesOnConditionCheckFailure": "ALL_OLD",
}
}
write_item_batch.append(updated_req_item)
write_item_batches.append(write_item_batch)
for write_item_batch in write_item_batches:
client.transact_write_items(TransactItems=write_item_batch)
print(f"##### Started build for {janis_branch}: build_id={build_id}")
from os import path
import sys
import json
from decimal import Decimal
sys.path.append(path.join(path.dirname(__file__), '../handlers'))
from helpers.utils import get_dynamodb_table, get_build_item
queue_table = get_dynamodb_table()
def stringify_decimal(obj):
if isinstance(obj, Decimal):
return str(obj)
else:
return obj
build_item = get_build_item("BLD#master#2020-07-12T15:28:04.697734-05:00")
pages = json.loads(
json.dumps({
"pages": build_item["pages"],
}, default=stringify_decimal))
print(pages)
def process_build_failure(build_id, context):
queue_table = get_dynamodb_table()
client = get_dynamodb_client()
timestamp = get_datetime()
build_item = get_build_item(build_id)
if build_item["status"] != "building":
print(f"##### Failed build for [{build_id}] has already been handled")
return None
janis_branch = get_janis_branch(build_id)
build_pk = build_item["pk"]
build_sk = build_item["sk"]
start_build_time = dateutil.parser.parse(build_item["sk"])
end_build_time = dateutil.parser.parse(timestamp)
total_build_time = str(end_build_time - start_build_time)
req_pk = f'REQ#{janis_branch}'
print(f"##### Build for [{build_id}] failed.")
assinged_reqs = queue_table.query(
IndexName="build_id.janis",
Select='ALL_ATTRIBUTES',
ScanIndexForward=True,
KeyConditionExpression=Key('build_id').eq(build_id) & Key('pk').eq(req_pk)
)['Items']
write_item_batches = []
write_item_batch = []
updated_current_build_item = {
"Update": {
"TableName": table_name,
"Key": {
"pk": "CURRENT_BLD",
"sk": janis_branch,
},
"UpdateExpression": "REMOVE build_id",
"ConditionExpression": "build_id = :build_id",
"ExpressionAttributeValues": {
":build_id": build_id,
},
"ReturnValuesOnConditionCheckFailure": "ALL_OLD",
}
}
updated_build_item = {
"Update": {
"TableName": table_name,
"Key": {
"pk": build_pk,
"sk": build_sk,
},
"UpdateExpression": "SET #s = :status, total_build_time = :total_build_time",
"ExpressionAttributeValues": {
":status": f'failed#{timestamp}',
":total_build_time": total_build_time,
},
"ExpressionAttributeNames": {
"#s": "status"
},
"ReturnValuesOnConditionCheckFailure": "ALL_OLD",
}
}
write_item_batch.append(updated_current_build_item)
write_item_batch.append(updated_build_item)
for req in assinged_reqs:
# transact_write_items() allows a maximum of 25 TransactItems.
# If there are more than 25 items, then start a new batch.
if len(write_item_batch) >= 25:
write_item_batches.append(write_item_batch)
write_item_batch = []
# reqs must be updated to remove build_id, and reset to status="waiting#{original timestamp}"
updated_req_item = {
"Update": {
"TableName": table_name,
"Key": {
"pk": req["pk"],
"sk": req["sk"],
},
"UpdateExpression": "REMOVE build_id SET failed_build_ids = list_append(if_not_exists(failed_build_ids, :empty_list), :failed_build_id), #s = :status",
"ExpressionAttributeNames": {
"#s": "status"
},
"ExpressionAttributeValues": {
# Reset status to original waiting status with original timestamp ("sk") to preserve request queuing order
":status": f'waiting#{req["sk"]}',
":empty_list": [],
":failed_build_id": [build_id],
},
}
}
write_item_batch.append(updated_req_item)
write_item_batches.append(write_item_batch)
for write_item_batch in write_item_batches:
client.transact_write_items(TransactItems=write_item_batch)
print(f"##### Build failure processing for [{build_id}] is complete.")
# Notify slack channel if there are errors in Production
if is_production():
ssm_client = boto3.client('ssm')
webhook_url = ssm_client.get_parameter(Name="/coa-publisher/production/slack_webhook_publisher_errors", WithDecryption=True)['Parameter']['Value']
# refreshed_build_item will have most up to date "logs" urls and "status"
refreshed_build_item = get_build_item(build_id)
slack_message = "Janis build failed\n"
slack_message += f"```{json.dumps(refreshed_build_item, default=stringify_decimal, indent=4)}```"
slack = Slack(url=webhook_url)
slack.post(text=slack_message)