def _pull_base_image(base_image, source_base_image_path, dockyard_alias): if base_image.is_remote(): did_print = False if not base_image.dockyard_address: if not base_image.exists(): if dockyard_alias == DOCKYARD_FALLBACK_ALIAS: was_fallback_dockyard = True else: print('Base image {base_image} not found. ' 'Searching in official Armada dockyard...'.format( **locals())) dockyard_alias = DOCKYARD_FALLBACK_ALIAS base_image = ArmadaImageFactory(source_base_image_path, dockyard_alias) was_fallback_dockyard = False if was_fallback_dockyard or not base_image.exists(): raise ArmadaCommandException( 'Base image {base_image} not found. Aborting.'.format( **locals())) dockyard_dict = dockyard.get_dockyard_dict(dockyard_alias) d = dockyard_factory(dockyard_dict.get('address'), dockyard_dict.get('user'), dockyard_dict.get('password')) if d.is_http(): did_print = print_http_dockyard_unavailability_warning( dockyard_dict['address'], dockyard_alias, "ERROR! Cannot pull from dockyard!", ) retries = 0 if did_print else 3 base_image_path = base_image.image_path_with_tag if is_verbose(): print('Fetching base image: "{base_image_path}".\n'.format( **locals())) pull_command = 'docker pull {base_image_path}'.format(**locals()) assert execute_local_command(pull_command, stream_output=True, retries=retries)[0] == 0 else: base_image = ArmadaImageFactory(base_image.image_name, 'local') if not base_image.exists(): raise ArmadaCommandException( 'Base image {base_image} not found. Aborting.'.format( **locals())) base_image_path = base_image.image_path_with_tag return base_image_path
def login_to_dockyard(dockyard_alias): dockyard_dict = dockyard.get_dockyard_dict(dockyard_alias) if not dockyard_dict: raise ArmadaCommandException("Couldn't read configuration for dockyard alias {0}.".format(dockyard_alias)) dockyard_user = dockyard_dict.get('user') dockyard_password = dockyard_dict.get('password') if dockyard_user and dockyard_password: dockyard_address = dockyard_dict.get('address') login_command = ('docker login --username="******" --password="******" ' '{dockyard_address}').format(**locals()) if execute_local_command(login_command)[0] != 0: raise ArmadaCommandException( 'ERROR: Could not login to dockyard with alias {dockyard_alias}.'.format(**locals()))
def command_build(args): dockerfile_path = args.file if not os.path.exists(dockerfile_path): raise ArmadaCommandException( 'ERROR: {} not found.'.format(dockerfile_path)) source_base_image_paths = _get_base_image_paths(dockerfile_path) dockyard_alias = args.dockyard or get_default_alias() try: image = ArmadaImageFactory(args.microservice_name, dockyard_alias, os.environ.get('MICROSERVICE_NAME')) except InvalidImagePathException: raise ArmadaCommandException('No microservice name supplied.') notify_about_detected_dev_environment(image.image_name) for source_base_image_path in source_base_image_paths: source_dockyard_address = split_image_path(source_base_image_path)[0] if source_dockyard_address: base_image = ArmadaImageFactory(source_base_image_path, args.dockyard) else: print_warning( 'Using image name only as base image ("FROM {}") has been deprecated. ' 'Consider providing full dockyard/docker registry address, ' 'e.g.: "FROM dockyard.armada.sh/microservice".'.format( source_base_image_path)) base_image = ArmadaImageFactory(source_base_image_path, dockyard_alias) base_image_path = _pull_base_image(base_image, source_base_image_path, dockyard_alias) if base_image_path != source_base_image_path: if is_verbose(): print( 'Tagging "{base_image_path}" as "{source_base_image_path}"\n' .format(**locals())) tag_command = "docker tag {} {}".format(base_image_path, source_base_image_path) assert execute_local_command(tag_command, stream_output=True, retries=1)[0] == 0 build_command = _generate_build_command(args, dockerfile_path, image) assert execute_local_command(' '.join(build_command), stream_output=True)[0] == 0
def _handle_result(result, is_restart): if not result: raise ArmadaCommandException("ERROR: armada API call failed.") if result.get('status') != 'ok': raise ArmadaCommandException('ERROR: {0}'.format(result['error'])) container_id = result['container_id'] if is_restart: print('Service has been restarted and is running in container {container_id} ' 'available at addresses:'.format(**locals())) else: print('Service is running in container {container_id} available at addresses:'.format(**locals())) for service_address, docker_port in result['endpoints'].iteritems(): print(' {0} ({1})'.format(service_address, docker_port))
def login_to_dockyard(dockyard_alias): dockyard_dict = dockyard.get_dockyard_dict(dockyard_alias) if not dockyard_dict: raise ArmadaCommandException("Couldn't read configuration for dockyard alias {0}.".format(dockyard_alias)) dockyard_user = dockyard_dict.get('user') dockyard_password = dockyard_dict.get('password') if dockyard_user and dockyard_password: dockyard_address = dockyard_dict.get('address') current_user_email = '{0}@{1}'.format(pwd.getpwuid(os.getuid()).pw_name, socket.gethostname()) login_command = ('docker login --username="******" --password="******" ' '--email="{current_user_email}" {dockyard_address}').format(**locals()) if execute_local_command(login_command)[0] != 0: raise ArmadaCommandException( 'ERROR: Could not login to dockyard with alias {dockyard_alias}.'.format(**locals()))
def command_push(args): if not args.image_path: raise ArmadaCommandException('ERROR: Please specify image_path argument' ' or set MICROSERVICE_NAME environment variable') dockyard_alias = args.dockyard image = ArmadaImage(args.image_path, dockyard_alias) if '/' not in args.image_path: if not ArmadaImage(image.image_name, 'local').exists(): raise Exception('Image {} does not exist. Typo?'.format(image.image_name)) dockyard_string = image.dockyard.url or '' dockyard_string += ' (alias: {})'.format(dockyard_alias) if dockyard_alias else '' print('Pushing image {} to dockyard: {}...'.format(image.image_name, dockyard_string)) tag_command = 'docker tag -f {} {}'.format(image.image_name, image.image_path) assert execute_local_command(tag_command, stream_output=True, retries=1)[0] == 0 else: # If command was called with [docker_registry_address]/[image_name] and no -d/--dockyard, then simply # mimic 'docker push' behavior (without tagging). print('Pushing image {}...'.format(image)) dockyard_dict = dockyard.get_dockyard_dict(dockyard_alias) did_print = False d = dockyard_factory(dockyard_dict.get('address'), dockyard_dict.get('user'), dockyard_dict.get('password')) if d.is_http(): did_print = print_http_dockyard_unavailability_warning( dockyard_dict['address'], dockyard_alias, "ERROR! Cannot push to dockyard!", ) retries = 0 if did_print else 3 login_to_dockyard(dockyard_alias) push_command = 'docker push {}'.format(image.image_path) assert execute_local_command(push_command, stream_output=True, retries=retries)[0] == 0
def _find_dockyard_with_image(vagrant_dev, is_restart, dockyard_alias, microservice_name): image = ArmadaImageFactory(microservice_name, dockyard_alias) if vagrant_dev and is_restart: local_image = ArmadaImageFactory(image.image_name_with_tag, 'local') image = select_latest_image(image, local_image) if image == local_image: dockyard_alias = 'local' if vagrant_dev and not image.exists(): print( 'Image {image} not found. Searching in default dockyard...'.format( **locals())) dockyard_alias = get_default() image = ArmadaImageFactory(image.image_name_with_tag, dockyard_alias) if not image.exists(): if dockyard_alias == DOCKYARD_FALLBACK_ALIAS: was_fallback_dockyard = True else: print( 'Image {} not found. Searching in official Armada dockyard...'. format(image.image_name_with_tag)) dockyard_alias = DOCKYARD_FALLBACK_ALIAS image = ArmadaImageFactory(image.image_name_with_tag, dockyard_alias) was_fallback_dockyard = False if was_fallback_dockyard or not image.exists(): raise ArmadaCommandException( 'Image {} not found. Aborting.'.format( image.image_path_with_tag)) return dockyard_alias, image
def update_dockyard(self, dockyard_alias): if dockyard_alias and dockyard_alias != 'local': dockyard_info = dockyard.alias.get_alias(dockyard_alias) if not dockyard_info: raise ArmadaCommandException( "Couldn't read configuration for dockyard alias {0}.".format(dockyard_alias)) self._payload['dockyard_user'] = dockyard_info.get('user') self._payload['dockyard_password'] = dockyard_info.get('password')
def update_ports(self, ports): for port_mapping in sum(ports or [], []): try: port_host, port_container = map( int, (port_mapping.split(':', 1) + [None])[:2]) self._payload['ports'][str(port_host)] = str(port_container) except (ValueError, TypeError): raise ArmadaCommandException( 'Invalid port mapping: {0}'.format(port_mapping))
def _ports_to_mapping_dict(self, ports): mapping_dict = {} for port_mapping in sum(ports or [], []): try: port_host, port_container = map(int, (port_mapping.split(':', 1) + [None])[:2]) mapping_dict[port_host] = port_container except (ValueError, TypeError): raise ArmadaCommandException('Invalid port mapping: {0}'.format(port_mapping)) return mapping_dict
def command_create(args): base_template = _get_template_name(args.base_template) service_name = args.name or base_template destination_dir = os.path.join(os.getcwd(), service_name) if os.path.exists(destination_dir): raise ArmadaCommandException( 'Destination dir {destination_dir} already exists.'.format( **locals())) command_list_code, command_list_out, command_list_err = execute_local_command( 'armada list armada -q | head -1') if command_list_code != 0: raise ArmadaCommandException( 'Could not get Armada container id:\n{command_list_err}'.format( **locals())) path_to_base_template = os.path.join('/opt/templates', base_template) armada_container_id = command_list_out.strip() temp_dir = tempfile.mkdtemp() try: command_cp = 'docker cp {armada_container_id}:{path_to_base_template} {temp_dir}'.format( **locals()) command_cp_code, command_cp_out, command_cp_err = execute_local_command( command_cp) if command_cp_code != 0: raise ArmadaCommandException( 'Could not get microservice template:\n{command_cp_err}'. format(**locals())) shutil.move(os.path.join(temp_dir, base_template), destination_dir) if service_name != base_template: upper_template = base_template.upper() template_name_variable = '_{upper_template}_'.format(**locals()) _replace_in_path(destination_dir, template_name_variable, service_name) dockyard_address = dockyard.get_dockyard_address() template_dockyard_variable = '_DOCKYARD_ADDRESS_' _replace_in_path(destination_dir, template_dockyard_variable, dockyard_address) print( 'Service {service_name} has been created in {destination_dir} from {args.base_template} template.' .format(**locals())) finally: shutil.rmtree(temp_dir)
def command_push(args): try: source_image = ArmadaImageFactory(args.image_path, 'local', os.environ.get('MICROSERVICE_NAME')) except InvalidImagePathException: raise ArmadaCommandException( 'ERROR: Please specify image_path argument' ' or set MICROSERVICE_NAME environment variable') notify_about_detected_dev_environment(source_image.image_name) dockyard_alias = args.dockyard if not source_image.dockyard_address: if not source_image.exists(): raise Exception('Image {} does not exist. Typo?'.format( source_image.image_name_with_tag)) destination_image = ArmadaImageFactory( source_image.image_name_with_tag, dockyard_alias) dockyard_string = destination_image.dockyard.url or '' dockyard_string += ' (alias: {})'.format( dockyard_alias) if dockyard_alias else '' print('Pushing image {} to dockyard: {}...'.format( source_image.image_name_with_tag, dockyard_string)) tag_command = "docker tag {} {}".format( source_image.image_name_with_tag, destination_image.image_path_with_tag) print(tag_command) assert execute_local_command(tag_command, stream_output=True, retries=1)[0] == 0 else: # If command was called with [docker_registry_address]/[image_name] and no -d/--dockyard, then simply # mimic 'docker push' behavior (without tagging). print('Pushing image {}...'.format(source_image.image_path_with_tag)) destination_image = source_image dockyard_dict = dockyard.get_dockyard_dict(dockyard_alias) did_print = False d = dockyard_factory(dockyard_dict.get('address'), dockyard_dict.get('user'), dockyard_dict.get('password')) if d.is_http(): did_print = print_http_dockyard_unavailability_warning( dockyard_dict['address'], dockyard_alias, "ERROR! Cannot push to dockyard!", ) retries = 0 if did_print else 3 login_to_dockyard(dockyard_alias) push_command = 'docker push {}'.format( destination_image.image_path_with_tag) assert execute_local_command(push_command, stream_output=True, retries=retries)[0] == 0
def command_stop(args): microservice_handle = args.microservice_handle or os.environ[ 'MICROSERVICE_NAME'] if not microservice_handle: raise ValueError('No microservice name or container id supplied.') instances = armada_utils.get_matched_containers(microservice_handle) instances_count = len(instances) if instances_count > 1: if not args.all: raise armada_utils.ArmadaCommandException( 'There are too many ({instances_count}) matching containers. ' 'Provide more specific container_id or microservice name or use -a/--all flag.' .format(**locals())) print('Stopping {instances_count} services {microservice_handle}...'. format(**locals())) else: microservice_name = instances[0]['ServiceName'] container_id = instances[0]["ServiceID"].split(':')[0] print( 'Stopping service {microservice_name} ({container_id})...'.format( **locals())) were_errors = False for i, instance in enumerate(instances): try: if instances_count > 1: print('[{0}/{1}]'.format(i + 1, instances_count)) if 'kv_index' in instance: kv.kv_remove('service/{}/{}'.format(instance['ServiceName'], instance['kv_index'])) print('Service {} has been removed.'.format( instance['ServiceName'])) else: container_id = instance['ServiceID'].split(':')[0] payload = {'container_id': container_id} ship_name = instance['Address'] result = armada_api.post('stop', payload, ship_name=ship_name) if result['status'] == 'ok': print('Service {container_id} has been stopped.'.format( **locals())) if instances_count > 1: print() else: raise ArmadaCommandException('Stopping error: {0}'.format( result['error'])) except: traceback.print_exc() were_errors = True if were_errors: sys.exit(1)
def command_stop(args): microservice_handles = args.microservice_handle or [os.environ['MICROSERVICE_NAME']] if not microservice_handles: raise ValueError('No microservice name or container id supplied.') armada_utils.notify_about_detected_dev_environment(microservice_handles[0]) services = {microservice_handle: armada_utils.get_matched_containers(microservice_handle) for microservice_handle in microservice_handles} for microservice_handle, instances in services.items(): instances_count = len(instances) if instances_count > 1 and not args.all: raise armada_utils.ArmadaCommandException( 'There are too many ({instances_count}) matching containers for service: {microservice_handle}. ' 'Provide more specific container_id or microservice name or use -a/--all flag.'.format(**locals())) were_errors = False for microservice_handle, instances in services.items(): instances_count = len(instances) if instances_count > 1: print('Stopping {instances_count} services {microservice_handle}...'.format(**locals())) else: microservice_name = instances[0]['ServiceName'] container_id = instances[0]["ServiceID"].split(':')[0] print('Stopping service {microservice_name} ({container_id})...'.format(**locals())) for i, instance in enumerate(instances): try: if instances_count > 1: print('[{0}/{1}]'.format(i + 1, instances_count)) container_id = instance['ServiceID'].split(':')[0] payload = {'container_id': container_id} ship_name = instance['Address'] result = armada_api.post('stop', payload, ship_name=ship_name) if result['status'] == 'error' and result['error'].startswith( 'armada API exception: ValueError - Cannot find ship:'): payload['force'] = True result = armada_api.post('stop', payload) if result['status'] == 'ok': print('Service {container_id} has been stopped.'.format(**locals())) print() else: raise ArmadaCommandException('Stopping error: {0}'.format(result['error'])) except: traceback.print_exc() were_errors = True if were_errors: sys.exit(1)
def command_run(args): if args.verbose: global verbose verbose = True microservice_name = args.microservice_name if not microservice_name: raise ArmadaCommandException( 'ERROR: Please specify microservice_name argument' ' or set MICROSERVICE_NAME environment variable') ship = args.ship is_run_locally = ship is None dockyard_alias = args.dockyard or dockyard.get_dockyard_alias( microservice_name, is_run_locally) vagrant_dev = _is_vagrant_dev(args.hidden_vagrant_dev, dockyard_alias, microservice_name) dockyard_alias, image = _find_dockyard_with_image(vagrant_dev, args.hidden_is_restart, dockyard_alias, microservice_name) _print_run_info(image, dockyard_alias, ship, args.rename) payload = RunPayload() payload.update_image_path(image.image_path) payload.update_dockyard(dockyard_alias) if vagrant_dev: payload.update_vagrant(args.dynamic_ports, args.use_latest_image_code, microservice_name) payload.update_environment(args.e) payload.update_ports(args.publish) payload.update_volumes(args.volumes) payload.update_microservice_vars(args.rename, args.env, args.app_id) payload.update_run_command(vagrant_dev) payload.update_resource_limits(args.cpu_shares, args.memory, args.memory_swap, args.cgroup_parent) payload.update_configs(args.configs) if verbose: print('payload: {0}'.format(payload)) warn_if_hit_crontab_environment_variable_length(payload.get('environment')) print( 'Checking if there is new image version. May take few minutes if download is needed...' ) result = armada_api.post('run', payload.data(), ship_name=ship) _handle_result(result, args.hidden_is_restart)
def __new__(cls, image_path, dockyard_alias=None, fallback_service_name=None): dockyard_address, image_name, image_tag = split_image_path(image_path) image_name = image_name or fallback_service_name if not image_name: raise InvalidImagePathException if dockyard_alias == 'local': return LocalArmadaImage(dockyard_address, image_name, image_tag) if dockyard_alias and dockyard_address: raise ArmadaCommandException( 'Ambiguous dockyard. Please specify either -d/--dockyard ' 'or dockyard_hostname[:port]/image_name') return RemoteArmadaImage(dockyard_address, image_name, image_tag, dockyard_alias)
def command_run(args): try: image = ArmadaImageFactory(args.microservice_name, 'local', os.environ.get('MICROSERVICE_NAME')) except InvalidImagePathException: raise ArmadaCommandException('ERROR: Please specify microservice_name argument' ' or set MICROSERVICE_NAME environment variable') ship = args.ship is_run_locally = ship is None dockyard_alias = args.dockyard or dockyard.get_dockyard_alias(image.image_name, is_run_locally) notify_about_detected_dev_environment(image.image_name) is_dev_env = _is_dev_environment(args.hidden_armada_develop, dockyard_alias, image.image_name) dockyard_alias, image = _find_dockyard_with_image(is_dev_env, args.hidden_is_restart, dockyard_alias, image.image_name_with_tag) _print_run_info(image, dockyard_alias, ship, args.rename) microservice_name = args.rename or image.image_name payload = RunPayload() payload.update_image_path(image.image_path_with_tag) payload.update_dockyard(dockyard_alias) if os.environ.get('MICROSERVICE_NAME') == image.image_name: payload.update_armada_develop_environment(image.image_name, microservice_name, args) payload.update_environment(args.e) payload.update_ports(args.publish) payload.update_volumes(args.volumes) payload.update_microservice_vars(args.rename, args.env, args.app_id) payload.update_run_command(is_dev_env, args.env, image.image_name) payload.update_resource_limits(args.cpu_shares, args.memory, args.memory_swap, args.cgroup_parent) payload.update_configs(args.configs) if is_verbose(): print('payload: {0}'.format(payload)) warn_if_hit_crontab_environment_variable_length(payload.get('environment')) print('Checking if there is new image version. May take few minutes if download is needed...') result = armada_api.post('run', payload.data(), ship_name=ship) _handle_result(result, args.hidden_is_restart)
def __init__(self, image_path, dockyard_alias=None): self.dockyard = None dockyard_address, self.image_name = ArmadaImage.__split_image_path( image_path) dockyard_dict = {} if dockyard_address: if dockyard_alias: raise ArmadaCommandException( 'Ambiguous dockyard. Please specify either -d/--dockyard ' 'or dockyard_hostname[:port]/image_name') elif dockyard_alias == 'local': pass else: dockyard_dict = dockyard.get_dockyard_dict(dockyard_alias) dockyard_address = dockyard_dict['address'] image_path = dockyard_address + '/' + self.image_name self.image_path = image_path self.dockyard = dockyard_factory(dockyard_address, dockyard_dict.get('user'), dockyard_dict.get('password'))