def handle_fanout_status(package_message):
""" Handle a single message by updating the package state in the fanout
table and, if all have either failed or been built, build the metapackage.
Args:
package_message (dict): JSON message containing hte package state
within the 'body' key, ie. Complete or Failed
Returns:
(dict): HTTP return code containing the status of the fanout process
"""
package_state = json.loads(package_message['body'])
# The dynamoDB table containing the running status of each package
dynamo = get_dynamo_resource()
fanout_table = dynamo.Table(FANOUT_STATUS)
update_package_state(fanout_table, package_state)
# Delete failed items - this doesn't end the process as we still want
# a new metapackage to be built regardless of the failed packages.
delete_failed(fanout_table)
# Check remaining items
resp = fanout_table.scan()
still_building = packages_still_building(resp['Items'])
if still_building > 0:
print(f"{still_building} items still in table")
return return_code(200, {"status": "Items are still running"})
# Otherwise if everything has completed, invoke the meta-package
# building function
build_metapackage(fanout_table)
return return_code(200, {"status": "All packages built"})
def lambda_handler(event, context):
print(json.dumps(event))
# The dynamoDB table containing the running status of each package
dynamo = get_dynamo_resource()
package_table = dynamo.Table(PACKAGE_TABLE)
for record in event['Records']:
json_record = json.loads(record['body'])
# Get Dependencies
deps = json_record['dependencies']
git_url = json_record['git_url']
git_branch = json_record['git_branch']
stage = json_record['stage']
# Put each one in the FANOUT_STATUS table with an "Initialized"
# status and add it to the queue
build_packages = get_packages_to_build(package_table, deps, stage)
if len(build_packages) > 0:
print(f"Building the following packages: {build_packages}")
else:
print("No new packages to build")
process_packages(build_packages, git_url, git_branch, stage)
return return_code(200, {'packages': build_packages})
def update_repository(repo_name, url):
response_body = {}
# The dynamoDB table containing the packages
dynamo = get_dynamo_resource()
table = dynamo.Table(PACKAGE_TABLE)
# Get any new packages that have been added to the repositories and upload them
print("Getting all current items in the table")
response_body['current'] = get_current_package_count(table, repo_name)
new_packages = add_new_packages(repo_name, url, table)
print(f"Adding new packages to {repo_name}")
new_pkg_count = len(new_packages['packages'])
new_diff = new_pkg_count - response_body['current']
new_diff = new_diff if new_diff > 0 else 0
print(f"Added {new_diff} items to {repo_name}")
response_body['new'] = new_diff
# Remove any packages no longer contained within the repositories
print(f"Removing packages from {repo_name}")
resp = table.query(KeyConditionExpression=Key('Repository').eq(repo_name))
all_packages = [x['PackageName'] for x in resp['Items']]
to_delete = delete_old_packages(repo_name, all_packages,
new_packages['packages'], table)
print(f"Removed {len(to_delete)} items from {repo_name}")
response_body['deleted'] = len(to_delete)
# Return the number of changes being made
print(json.dumps(response_body))
return response_body
def get_built_packages():
dynamo = get_dynamo_resource()
fanout_table = dynamo.Table(FANOUT_STATUS)
resp = fanout_table.scan()
return [
x['PackageName'] for x in resp['Items']
if x['BuildStatus'] == Status.Complete.name
]
def clear_table():
""" Clears the fanout status table completely on completion. """
print("Clearing fanout status table")
dynamo = get_dynamo_resource()
fanout_table = dynamo.Table(FANOUT_STATUS)
resp = fanout_table.scan()
to_delete = [x['PackageName'] for x in resp['Items']]
print(f"Deleting the following items: {to_delete}")
with fanout_table.batch_writer() as batch:
for package in to_delete:
batch.delete_item(Key={'PackageName': package})