def start_deployment(config, connection, task_definitions):
"""Start the deployment.
The following steps are executed:
1. The task definitions are registered with AWS
2. The before_deploy tasks are started
3. The services are updated to reference the last task definitions
4. The client poll's AWS until all deployments are finished.
5. The after_deploy tasks are started.
"""
cluster_name = config['cluster_name']
services = config['services']
# Register the task definitions in ECS
register_task_definitions(connection, task_definitions)
# Run tasks before deploying services
tasks_before_deploy = config.get('before_deploy', [])
run_tasks(connection, cluster_name, task_definitions, tasks_before_deploy)
# Check if all services exist
existing_services = utils.describe_services(
connection.ecs, cluster=cluster_name, services=task_definitions.keys())
available_services = {
service['serviceName']
for service in existing_services
}
new_services = set(task_definitions.keys()) - available_services
# Update services
for service_name, service in services.items():
task_definition = task_definitions[service['task_definition']]
if service_name in new_services:
logger.info("Creating new service %s with task definition %s",
service_name, task_definition['name'])
connection.ecs.create_service(
cluster=cluster_name,
serviceName=service_name,
desiredCount=1,
taskDefinition=task_definition['name'])
else:
logger.info("Updating service %s with task defintion %s",
service_name, task_definition['name'])
connection.ecs.update_service(
cluster=cluster_name,
service=service_name,
taskDefinition=task_definition['name'])
is_finished = wait_for_deployments(connection, cluster_name,
services.keys())
if not is_finished:
raise DeploymentFailed("Timeout")
# Run tasks after deploying services
tasks_after_deploy = config.get('after_deploy', [])
run_tasks(connection, cluster_name, task_definitions, tasks_after_deploy)
def wait_for_deployments(connection, cluster_name, service_names):
"""Poll ECS until all deployments are finished (status = PRIMARY)
"""
logger.info("Waiting for deployments")
start_time = time.time()
def service_description(service):
"""Return string in format of 'name (0/2)'"""
name = service['serviceName']
for deployment in service['deployments']:
if deployment.get('status') != 'PRIMARY':
continue
desired = deployment['desiredCount']
pending = deployment['pendingCount']
running = deployment['runningCount']
return '%s (%s/%s)' % (name, pending + running, desired)
return name
# Wait till all service updates are deployed
time.sleep(POLL_TIME)
while True:
services = utils.describe_services(connection.ecs,
cluster=cluster_name,
services=service_names)
in_progress = [s for s in services if len(s['deployments']) > 1]
if in_progress:
logger.info(
"Waiting for services: %s",
', '.join([service_description(s) for s in in_progress]))
else:
logger.info("Deployment finished: %s",
', '.join([service_description(s) for s in services]))
break
time.sleep(5)
if time.time() - start_time > (60 * 15):
logger.error("Giving up after 15 minutes")
return False
return True
def wait_for_deployments(connection: Connection, cluster_name: str,
service_names: typing.List[str]) -> bool:
"""Poll ECS until all deployments are finished (status = PRIMARY)"""
logger.info("Waiting for deployments")
start_time = time.time()
def service_description(service):
"""Return string in format of 'name (0/2)'"""
name = service["serviceName"]
for deployment in service["deployments"]:
if deployment.get("status") != "PRIMARY":
continue
desired = deployment["desiredCount"]
pending = deployment["pendingCount"]
running = deployment["runningCount"]
return "%s (%s/%s)" % (name, pending + running, desired)
return name
# Wait till all service updates are deployed
time.sleep(5)
utc_timestamp = datetime.datetime.utcnow().replace(
tzinfo=pytz.utc) - datetime.timedelta(seconds=5)
last_event_timestamps = {name: utc_timestamp for name in service_names}
logged_message_ids: typing.Set[str] = set()
ready_timestamp = None
last_message = datetime.datetime.now()
while True:
services = utils.describe_services(connection.ecs,
cluster=cluster_name,
services=service_names)
in_progress = [s for s in services if len(s["deployments"]) > 1]
messages = extract_new_event_messages(services, last_event_timestamps,
logged_message_ids)
for message in messages:
logger.info("%s - %s", message["createdAt"].strftime("%H:%M:%S"),
message["message"])
last_message = datetime.datetime.now()
# 5 Seconds after the deployment is no longer in progress we mark it
# as done.
offset = datetime.datetime.utcnow() - datetime.timedelta(seconds=5)
if ready_timestamp and offset > ready_timestamp:
logger.info(
"Deployment finished: %s",
", ".join([service_description(s) for s in services]),
)
break
# Set is_ready after the previous check so that we can wait for x
# more seconds before ending the operation successfully.
if not in_progress:
ready_timestamp = datetime.datetime.utcnow()
# So we haven't printed something for a while, let's give some feedback
elif last_message < datetime.datetime.now() - datetime.timedelta(
seconds=10):
logger.info(
"Still waiting for: %s",
", ".join([s["serviceName"] for s in in_progress]),
)
time.sleep(5)
if time.time() - start_time > (60 * 15):
logger.error("Giving up after 15 minutes")
return False
return True