def _validate_alarm_names(services: list) -> None: """ All ecs alarm names must start with 'ecs:' prefix. """ for service_dict in services: # TODO: Should be configurable if service_dict['alarm']['AlarmName'].startswith('ecs:'): continue logger.error("Alarm names must start with 'ecs:' prefix %s" % service_dict) raise AeropressException()
def _load_config(root_path: Path, image_url: str = None, entrypoint: list = [], command: list = [], environment: str = None) -> list: logger.info('Reading yaml config files from %s', root_path) services = [] # type: List[Dict[str, Any]] # Reading yaml services definitions into a list of dictionary. for root, dirs, files in os.walk(root_path.as_posix()): for name in files: path = Path(os.path.join(root, name)) with open(path.as_posix()) as f: _yaml_dict = yaml.safe_load(f.read()) for key, value in _yaml_dict.items(): # Handle service defnitions. for service_k, service_v in value.items(): if service_k != 'task': continue # Override defaults for container definitions. for container_definition in service_v[ 'containerDefinitions']: if image_url: container_definition['image'] = image_url if entrypoint: container_definition['entryPoint'] = entrypoint if command: container_definition['command'] = command if environment: # Environment must be in format of ... '[{"name": "foo", "value": "bar"}]' container_definition['environment'] = json.loads( environment) services.append(value) # Validate definitions if not _is_valid_config(services): logger.error('Config is not valid!') raise AeropressException() return services
def _validate_log_definitions(tasks: list) -> None: for task_dict in tasks: for container_definition in task_dict['containerDefinitions']: if not container_definition.get('logConfiguration'): continue if not container_definition['logConfiguration'].get('options'): continue options = container_definition['logConfiguration']['options'] if not options['awslogs-group'].startswith('/ecs'): logger.error("log groups must start with '/ecs/' prefix: %s", options) raise AeropressException() if options['awslogs-stream-prefix'] != 'ecs': logger.error("logstream prefixes must be 'ecs' : %s", options) raise AeropressException()
def _is_valid_config(services: list) -> bool: if not services: logger.error('No service definition is found!') return False for service_dict in services: # We might run only one task with run-task. if not service_dict.get('taskDefinition'): continue if 'task' not in service_dict: continue if service_dict['taskDefinition'] != service_dict['task']['family']: logger.error('Task definition is not found for service %s!', service_dict['serviceName']) return False return True
def _register_task_definitions(tasks: list) -> None: for task_dict in tasks: # Create container definitions. container_definitions = [] for container_definition in task_dict['containerDefinitions']: d = { 'name': container_definition['name'], 'image': container_definition['image'], 'logConfiguration': container_definition['logConfiguration'], 'memoryReservation': container_definition['memoryReservation'], 'cpu': container_definition.get('cpu', 0), 'entryPoint': container_definition.get('entryPoint', []), 'command': container_definition.get('command', []), 'environment': container_definition.get('environment', []), 'portMappings': container_definition.get('portMappings', []), 'ulimits': container_definition.get('ulimits', []), 'mountPoints': container_definition.get('mountPoints', []), 'links': container_definition.get('links', []), } if container_definition.get('memory'): if container_definition['memory'] < container_definition['memoryReservation']: logger.error('memory must be equal or bigger than memoryReservation') raise AeropressException() d['memory'] = container_definition['memory'] container_definitions.append(d) logger.info('Creating task definition: %s', task_dict['family']) response = ecs_client.register_task_definition( family=task_dict['family'], taskRoleArn=task_dict['taskRoleArn'], executionRoleArn=task_dict['executionRoleArn'], networkMode=task_dict['networkMode'], containerDefinitions=container_definitions, requiresCompatibilities=task_dict['requiresCompatibilities'], volumes=task_dict.get('volumes', []), ) logger.debug('Created task definition details: %s', response)
def deploy(services: list, service_names: list) -> None: selected_services = [] # type: List[Dict[str, Any]] for service_name in service_names: service_dict = _find_service(service_name, services) if not service_dict: logger.error("Service '%s' is not found! ", service_name) raise AeropressException() selected_services.append(service_dict) # Register task definitions. tasks = [ selected_service.get('task') for selected_service in selected_services ] tasks = list(filter(None, tasks)) if tasks: task.register_all(tasks) # Update or create all services. (We might have tasks without services, eliminating them..) filtered_selected_services = [ sd for sd in selected_services if sd.get('serviceName') ] service.update_all(filtered_selected_services)
def main() -> None: parser = argparse.ArgumentParser( description='aeropress AWS ECS deployment helper') subparsers = parser.add_subparsers(help='sub-command help', dest='subparser_name') parser_deploy = subparsers.add_parser('deploy', help='Deploy docker image to ECS.') parser_clean = subparsers.add_parser( 'clean', help='Clean commands for stale entitites on AWS.') parser_register = subparsers.add_parser('register', help='Register tasks on ECS.') # deploy subcommand parser_deploy.add_argument('--image-url', type=str, dest='deploy_image_url', default=None, help='Image URL for docker image.') parser_deploy.add_argument('--environment', type=str, dest='environment', help='Container environment.') parser_deploy.add_argument( '--service-names', nargs='+', dest='deploy_service_names', help= 'Service name that will be updated. If not present, all services will be updated' ) parser_deploy.add_argument( '--path', type=str, dest='config_path', help='Config path that includes service definitions.') # clean sub command parser_clean.add_argument( '--stale-tasks', action='store_true', dest='clean_stale_tasks', help='Cleans all stale tasks and leave only active revisions.') parser_clean.add_argument( '--stale-log-streams', action='store_true', dest='clean_stale_log_streams', help='Cleans all stale log streams and leave only active revisions.') parser_clean.add_argument('--days-ago', type=int, dest='log_stream_days_ago', help='Timedelta for deleting stale log streams.') parser_clean.add_argument( '--path', type=str, dest='config_path', help='Config path that includes service & task definitions.') # register sub command parser_register.add_argument( '--task-definition', type=str, dest='task_definition', help='Task definition that will be registered.') parser_register.add_argument( '--path', type=str, dest='config_path', help='Config path that includes service & task definitions.') parser_register.add_argument('--image-url', type=str, dest='image_url', default=None, help='Image URL for docker image.') parser_register.add_argument( '--entrypoint', type=str, dest='entrypoint', default=[], nargs='+', help='Container entrypoint. Must be list of strings.') parser_register.add_argument( '--command', type=str, dest='command', default=[], nargs='+', help='Container command. Must be list of strings.') # Main command parser.add_argument('--logging-level', default='info', choices=['debug', 'info', 'warning', 'error'], type=str.lower, dest='logging_level', help='Print debug logs') parser.add_argument('--version', action='version', dest='version', version='{version}'.format(version=__version__)) args = parser.parse_args() # Setup logger setup_logging(args.logging_level) # Create config dict, first. config_path = Path(args.config_path) # Clean stale tasks and exit. if args.subparser_name == 'clean': if args.clean_stale_tasks: logger.info('Cleaning stale tasks...') task.clean_stale_tasks() return if args.clean_stale_log_streams: services = _load_config(config_path) logger.info( 'Cleaning stale log streams from starting %s day(s) ago...', args.log_stream_days_ago) log.clean_stale_log_streams(services, args.log_stream_days_ago) return if args.subparser_name == 'deploy': services = _load_config(config_path, args.deploy_image_url, environment=args.environment) logger.info('Reading config from path %s', args.config_path) if args.deploy_image_url: logger.info("Deploying image '%s'", args.deploy_image_url) sleep_time = 2 max_retry_count = 5 retry_count = 0 while True: try: deploy(services, args.deploy_service_names) except botocore.exceptions.ClientError as e: if "Rate exceeded" not in e.args[0]: raise if retry_count >= max_retry_count: raise logger.error( 'Rate limit exceeded. Sleeping for %s seconds...' % sleep_time) sleep(sleep_time) sleep_time *= 2 retry_count += 1 else: break return if args.subparser_name == 'register': services = _load_config(config_path, args.image_url, args.entrypoint, args.command) task_dict = None for service_dict in services: if args.task_definition == service_dict.get('task', {}).get('family'): task_dict = service_dict['task'] break if not task_dict: logger.error("Could not find task definition '%s' on '%s'", args.task_definition, args.config_path) return logger.info("Registering task definition '%s' fom path: %s", args.task_definition, args.config_path) task.register_all([task_dict], False) return