def _validate_alarm_names(services: list) -> None:
"""
All ecs alarm names must start with 'ecs:' prefix.
"""
for service_dict in services:
# TODO: Should be configurable
if service_dict['alarm']['AlarmName'].startswith('ecs:'):
continue
logger.error("Alarm names must start with 'ecs:' prefix %s" %
service_dict)
raise AeropressException()
def _load_config(root_path: Path,
image_url: str = None,
entrypoint: list = [],
command: list = [],
environment: str = None) -> list:
logger.info('Reading yaml config files from %s', root_path)
services = [] # type: List[Dict[str, Any]]
# Reading yaml services definitions into a list of dictionary.
for root, dirs, files in os.walk(root_path.as_posix()):
for name in files:
path = Path(os.path.join(root, name))
with open(path.as_posix()) as f:
_yaml_dict = yaml.safe_load(f.read())
for key, value in _yaml_dict.items():
# Handle service defnitions.
for service_k, service_v in value.items():
if service_k != 'task':
continue
# Override defaults for container definitions.
for container_definition in service_v[
'containerDefinitions']:
if image_url:
container_definition['image'] = image_url
if entrypoint:
container_definition['entryPoint'] = entrypoint
if command:
container_definition['command'] = command
if environment:
# Environment must be in format of ... '[{"name": "foo", "value": "bar"}]'
container_definition['environment'] = json.loads(
environment)
services.append(value)
# Validate definitions
if not _is_valid_config(services):
logger.error('Config is not valid!')
raise AeropressException()
return services
def _validate_log_definitions(tasks: list) -> None:
for task_dict in tasks:
for container_definition in task_dict['containerDefinitions']:
if not container_definition.get('logConfiguration'):
continue
if not container_definition['logConfiguration'].get('options'):
continue
options = container_definition['logConfiguration']['options']
if not options['awslogs-group'].startswith('/ecs'):
logger.error("log groups must start with '/ecs/' prefix: %s", options)
raise AeropressException()
if options['awslogs-stream-prefix'] != 'ecs':
logger.error("logstream prefixes must be 'ecs' : %s", options)
raise AeropressException()
def _is_valid_config(services: list) -> bool:
if not services:
logger.error('No service definition is found!')
return False
for service_dict in services:
# We might run only one task with run-task.
if not service_dict.get('taskDefinition'):
continue
if 'task' not in service_dict:
continue
if service_dict['taskDefinition'] != service_dict['task']['family']:
logger.error('Task definition is not found for service %s!',
service_dict['serviceName'])
return False
return True
def _register_task_definitions(tasks: list) -> None:
for task_dict in tasks:
# Create container definitions.
container_definitions = []
for container_definition in task_dict['containerDefinitions']:
d = {
'name': container_definition['name'],
'image': container_definition['image'],
'logConfiguration': container_definition['logConfiguration'],
'memoryReservation': container_definition['memoryReservation'],
'cpu': container_definition.get('cpu', 0),
'entryPoint': container_definition.get('entryPoint', []),
'command': container_definition.get('command', []),
'environment': container_definition.get('environment', []),
'portMappings': container_definition.get('portMappings', []),
'ulimits': container_definition.get('ulimits', []),
'mountPoints': container_definition.get('mountPoints', []),
'links': container_definition.get('links', []),
}
if container_definition.get('memory'):
if container_definition['memory'] < container_definition['memoryReservation']:
logger.error('memory must be equal or bigger than memoryReservation')
raise AeropressException()
d['memory'] = container_definition['memory']
container_definitions.append(d)
logger.info('Creating task definition: %s', task_dict['family'])
response = ecs_client.register_task_definition(
family=task_dict['family'],
taskRoleArn=task_dict['taskRoleArn'],
executionRoleArn=task_dict['executionRoleArn'],
networkMode=task_dict['networkMode'],
containerDefinitions=container_definitions,
requiresCompatibilities=task_dict['requiresCompatibilities'],
volumes=task_dict.get('volumes', []),
)
logger.debug('Created task definition details: %s', response)
def deploy(services: list, service_names: list) -> None:
selected_services = [] # type: List[Dict[str, Any]]
for service_name in service_names:
service_dict = _find_service(service_name, services)
if not service_dict:
logger.error("Service '%s' is not found! ", service_name)
raise AeropressException()
selected_services.append(service_dict)
# Register task definitions.
tasks = [
selected_service.get('task') for selected_service in selected_services
]
tasks = list(filter(None, tasks))
if tasks:
task.register_all(tasks)
# Update or create all services. (We might have tasks without services, eliminating them..)
filtered_selected_services = [
sd for sd in selected_services if sd.get('serviceName')
]
service.update_all(filtered_selected_services)
def main() -> None:
parser = argparse.ArgumentParser(
description='aeropress AWS ECS deployment helper')
subparsers = parser.add_subparsers(help='sub-command help',
dest='subparser_name')
parser_deploy = subparsers.add_parser('deploy',
help='Deploy docker image to ECS.')
parser_clean = subparsers.add_parser(
'clean', help='Clean commands for stale entitites on AWS.')
parser_register = subparsers.add_parser('register',
help='Register tasks on ECS.')
# deploy subcommand
parser_deploy.add_argument('--image-url',
type=str,
dest='deploy_image_url',
default=None,
help='Image URL for docker image.')
parser_deploy.add_argument('--environment',
type=str,
dest='environment',
help='Container environment.')
parser_deploy.add_argument(
'--service-names',
nargs='+',
dest='deploy_service_names',
help=
'Service name that will be updated. If not present, all services will be updated'
)
parser_deploy.add_argument(
'--path',
type=str,
dest='config_path',
help='Config path that includes service definitions.')
# clean sub command
parser_clean.add_argument(
'--stale-tasks',
action='store_true',
dest='clean_stale_tasks',
help='Cleans all stale tasks and leave only active revisions.')
parser_clean.add_argument(
'--stale-log-streams',
action='store_true',
dest='clean_stale_log_streams',
help='Cleans all stale log streams and leave only active revisions.')
parser_clean.add_argument('--days-ago',
type=int,
dest='log_stream_days_ago',
help='Timedelta for deleting stale log streams.')
parser_clean.add_argument(
'--path',
type=str,
dest='config_path',
help='Config path that includes service & task definitions.')
# register sub command
parser_register.add_argument(
'--task-definition',
type=str,
dest='task_definition',
help='Task definition that will be registered.')
parser_register.add_argument(
'--path',
type=str,
dest='config_path',
help='Config path that includes service & task definitions.')
parser_register.add_argument('--image-url',
type=str,
dest='image_url',
default=None,
help='Image URL for docker image.')
parser_register.add_argument(
'--entrypoint',
type=str,
dest='entrypoint',
default=[],
nargs='+',
help='Container entrypoint. Must be list of strings.')
parser_register.add_argument(
'--command',
type=str,
dest='command',
default=[],
nargs='+',
help='Container command. Must be list of strings.')
# Main command
parser.add_argument('--logging-level',
default='info',
choices=['debug', 'info', 'warning', 'error'],
type=str.lower,
dest='logging_level',
help='Print debug logs')
parser.add_argument('--version',
action='version',
dest='version',
version='{version}'.format(version=__version__))
args = parser.parse_args()
# Setup logger
setup_logging(args.logging_level)
# Create config dict, first.
config_path = Path(args.config_path)
# Clean stale tasks and exit.
if args.subparser_name == 'clean':
if args.clean_stale_tasks:
logger.info('Cleaning stale tasks...')
task.clean_stale_tasks()
return
if args.clean_stale_log_streams:
services = _load_config(config_path)
logger.info(
'Cleaning stale log streams from starting %s day(s) ago...',
args.log_stream_days_ago)
log.clean_stale_log_streams(services, args.log_stream_days_ago)
return
if args.subparser_name == 'deploy':
services = _load_config(config_path,
args.deploy_image_url,
environment=args.environment)
logger.info('Reading config from path %s', args.config_path)
if args.deploy_image_url:
logger.info("Deploying image '%s'", args.deploy_image_url)
sleep_time = 2
max_retry_count = 5
retry_count = 0
while True:
try:
deploy(services, args.deploy_service_names)
except botocore.exceptions.ClientError as e:
if "Rate exceeded" not in e.args[0]:
raise
if retry_count >= max_retry_count:
raise
logger.error(
'Rate limit exceeded. Sleeping for %s seconds...' %
sleep_time)
sleep(sleep_time)
sleep_time *= 2
retry_count += 1
else:
break
return
if args.subparser_name == 'register':
services = _load_config(config_path, args.image_url, args.entrypoint,
args.command)
task_dict = None
for service_dict in services:
if args.task_definition == service_dict.get('task',
{}).get('family'):
task_dict = service_dict['task']
break
if not task_dict:
logger.error("Could not find task definition '%s' on '%s'",
args.task_definition, args.config_path)
return
logger.info("Registering task definition '%s' fom path: %s",
args.task_definition, args.config_path)
task.register_all([task_dict], False)
return