def save_container(ship, container_id, status, params=None): try: start_timestamp = kv_get('start_timestamp/{}'.format(container_id)) except: start_timestamp = None if status == 'crashed': service_name = params['microservice_name'] else: service_name = get_env(container_id, 'MICROSERVICE_NAME') params = json.loads( base64.b64decode( get_env(container_id, 'RESTART_CONTAINER_PARAMETERS'))) if not start_timestamp: start_timestamp = str(calendar.timegm(time.gmtime())) address = kv_get('ships/{}/ip'.format(ship)) or ship service_dict = { 'ServiceName': service_name, 'Status': status, 'container_id': container_id, 'params': params, 'start_timestamp': start_timestamp, 'ServiceID': container_id, 'Address': address } kv_set(create_consul_services_key(ship, service_name, container_id), service_dict)
def recover_containers_from_kv_store(): services_to_be_recovered = _get_crashed_services() for service in services_to_be_recovered: kv.update_container_status('recovering', key=service) recovery_retry_count = 0 while services_to_be_recovered and recovery_retry_count < RECOVERY_RETRY_LIMIT: get_logger().info("Recovering containers: %s", json.dumps(services_to_be_recovered)) services_not_recovered = [] for service in services_to_be_recovered: service_parameters = kv.kv_get(service)['params'] if not _recover_container(service_parameters): services_not_recovered.append(service) else: kv.kv_remove(service) sleep(DELAY_BETWEEN_RECOVER_RETRY_SECONDS) services_to_be_recovered = services_not_recovered recovery_retry_count += 1 for service in services_to_be_recovered: kv.update_container_status('not-recovered', key=service) return services_to_be_recovered
def _stop_service(self, container_id): ship = get_ship_name() service_dict = None service_list = kv_list('ships/{}/service/'.format(ship)) if service_list: key = fnmatch.filter(service_list, '*/{}'.format(container_id)) service_dict = kv_get(key[0]) if key else None if service_dict and service_dict['Status'] in ['crashed', 'not-recovered']: kv_remove(key[0]) else: run_command_in_container('supervisorctl stop armada_agent', container_id) # TODO: Compatibility with old microservice images. Should be removed in future armada version. run_command_in_container('supervisorctl stop register_in_service_discovery', container_id) docker_api = docker_client.api() last_exception = None try: deregister_services(container_id) except: traceback.print_exc() for i in range(3): try: docker_api.stop(container_id) kv_remove(key[0]) except Exception as e: last_exception = e traceback.print_exc() if not is_container_running(container_id): break if is_container_running(container_id): get_logger().error('Could not stop container: {}'.format(container_id)) raise last_exception
def _choose_active_instances(services_dicts): result = services_dicts running_services_with_single_active_instances = {} for microservice_id, service_dict in services_dicts.items(): if service_dict.get('single_active_instance' ) and service_dict['status'] in ('passing', 'warning'): key = 'chosen_active_instance/{},env={},app_id={}'.format( service_dict['name'], service_dict['tags'].get('env') or '', service_dict['tags'].get('app_id') or '') if key not in running_services_with_single_active_instances: running_services_with_single_active_instances[key] = set() running_services_with_single_active_instances[key].add( microservice_id) for key, running_microservice_ids in running_services_with_single_active_instances.items( ): currently_picked_instance = kv.kv_get(key) if currently_picked_instance not in running_microservice_ids: currently_picked_instance = random.choice( list(running_microservice_ids)) kv.kv_set(key, currently_picked_instance) for microservice_id in running_microservice_ids: if microservice_id != currently_picked_instance: result[microservice_id]['status'] = 'standby' return result
def _get_local_running_containers(): result = [] for container in get_local_services(): container_parameters = kv.kv_get(container)['params'] if container_parameters: result.append(container_parameters) return result
def main(): setup_sentry() args = _parse_args() saved_containers_path = args.saved_containers_path if not args.force and not _is_recovery_completed(): get_logger().info( 'Recovery is not completed. Aborting saving running containers.') return try: wait_for_consul_ready() saved_containers = get_local_services() containers_parameters_dict = {} for container in saved_containers: container_dict = kv.kv_get(container) containers_parameters_dict[container] = container_dict if not containers_parameters_dict: get_logger().info( 'Aborted saving container because list is empty.') return _save_containers_parameters_list_in_file(containers_parameters_dict, saved_containers_path) get_logger().info( 'Containers have been saved to {}.'.format(saved_containers_path)) except Exception as e: get_logger().exception(e) sys.exit(1)
def main(): setup_sentry() args = _parse_args() saved_containers_path = args.saved_containers_path try: wait_for_consul_ready() ship = get_ship_name() saved_containers = kv.kv_list('ships/{}/service/'.format(ship)) containers_parameters_dict = {} if saved_containers: for container in saved_containers: container_dict = kv.kv_get(container) containers_parameters_dict[container] = container_dict if containers_parameters_dict: try: _save_containers_parameters_list_in_kv_store(containers_parameters_dict) get_logger().info('Containers have been saved to kv store.') except Exception as e: get_logger().exception(e) if not args.force and not _is_recovery_completed(): get_logger().warning('Recovery is not completed. Aborting saving running containers.') return _save_containers_parameters_list_in_file(containers_parameters_dict, saved_containers_path) get_logger().info('Containers have been saved to {}.'.format(saved_containers_path)) else: get_logger().info('Aborted saving container because of errors.') except Exception as e: get_logger().exception(e) sys.exit(1)
def update_container_status(status, ship=None, service_name=None, container_id=None, key=None): if not key: key = create_consul_services_key(ship, service_name, container_id) service_dict = kv_get(key) if status == 'crashed' and service_dict['Status'] in ['not-recovered', 'recovering']: return service_dict['Status'] = status kv_set(key, service_dict)
def _get_services_list(filter_microservice_name, filter_env, filter_app_id, filter_local): if filter_local: ship_list = ['containers_parameters_list/{}'.format(get_ship_name())] else: ship_list = kv.kv_list('containers_parameters_list/') services_dict = {} if not ship_list: return {} for ship in ship_list: containers = kv.kv_get(ship) if containers: services_dict.update(containers) services_list = services_dict.keys() result = {} if not services_list: return result if filter_microservice_name: services_list = fnmatch.filter( services_list, 'ships/*/service/{}/*'.format(filter_microservice_name)) for service in services_list: service_dict = services_dict[service] microservice_name = service_dict['ServiceName'] microservice_status = service_dict['Status'] microservice_id = service_dict['ServiceID'] container_id = service_dict['container_id'] microservice_start_timestamp = service_dict['start_timestamp'] not_available = 'n/a' microservice_tags_dict = {} if service_dict['params']['microservice_env']: microservice_tags_dict['env'] = service_dict['params'][ 'microservice_env'] if service_dict['params']['microservice_app_id']: microservice_tags_dict['app_id'] = service_dict['params'][ 'microservice_app_id'] matches_env = (filter_env is None) or ( filter_env == microservice_tags_dict.get('env')) matches_app_id = (filter_app_id is None) or ( filter_app_id == microservice_tags_dict.get('app_id')) if matches_env and matches_app_id: microservice_dict = { 'name': microservice_name, 'status': microservice_status, 'address': not_available, 'microservice_id': microservice_id, 'container_id': container_id, 'tags': microservice_tags_dict, 'start_timestamp': microservice_start_timestamp, } result[microservice_id] = microservice_dict return result
def _get_local_running_containers(): result = [] ship = get_ship_name() local_containers = kv.kv_list('ships/{}/service/'.format(ship)) or [] for container in local_containers: container_parameters = kv.kv_get(container)['params'] if container_parameters: result.append(container_parameters) return result
def _get_services_list(filter_microservice_name, filter_env, filter_app_id, filter_local): if filter_local: ship_list = [get_ship_name()] else: ship_list = get_ship_names() services_dict = {} if not ship_list: return {} for ship in ship_list: containers = kv.kv_get('containers_parameters_list/{}'.format(ship)) if containers and isinstance(containers, dict): services_dict.update(containers) services_list = services_dict.keys() result = {} if not services_list: return result if filter_microservice_name: services_list = fnmatch.filter(services_list, 'ships/*/service/{}/*'.format(filter_microservice_name)) for service in services_list: service_dict = services_dict[service] microservice_name = service_dict['ServiceName'] microservice_status = service_dict['Status'] microservice_id = service_dict['ServiceID'] container_id = service_dict['container_id'] microservice_start_timestamp = service_dict['start_timestamp'] not_available = 'n/a' microservice_tags_dict = {} try: if service_dict['params']['microservice_env']: microservice_tags_dict['env'] = service_dict['params']['microservice_env'] if service_dict['params']['microservice_app_id']: microservice_tags_dict['app_id'] = service_dict['params']['microservice_app_id'] except KeyError as e: get_logger().warning(repr(e)) matches_env = (filter_env is None) or (filter_env == microservice_tags_dict.get('env')) matches_app_id = (filter_app_id is None) or (filter_app_id == microservice_tags_dict.get('app_id')) if matches_env and matches_app_id: microservice_dict = { 'name': microservice_name, 'status': microservice_status, 'address': not_available, 'microservice_id': microservice_id, 'container_id': container_id, 'tags': microservice_tags_dict, 'start_timestamp': microservice_start_timestamp, } result[microservice_id] = microservice_dict return result
def _get_crashed_services(): services_list = get_local_services() crashed_services = [] for service in services_list: service_dict = kv.kv_get(service) microservice_status = service_dict['Status'] if microservice_status in ['crashed', 'not-recovered']: crashed_services.append(service) return crashed_services
def _get_crashed_services(): ship = get_ship_name() services_list = kv.kv_list('ships/{}/service/'.format(ship)) crashed_services = [] if not services_list: return crashed_services for service in services_list: service_dict = kv.kv_get(service) microservice_status = service_dict['Status'] if microservice_status in ['crashed', 'not-recovered']: crashed_services.append(service) return crashed_services
def _get_restart_parameters(self, container_id): try: docker_api = docker_client.api() docker_inspect = docker_api.inspect_container(container_id) for env_var in docker_inspect['Config']['Env']: env_key, env_value = (env_var.strip('"').split('=', 1) + [''])[:2] if env_key == 'RESTART_CONTAINER_PARAMETERS': return json.loads(base64.b64decode(env_value)) except NotFound: service_list = kv_list('ships/') for service in service_list: if service.split('/')[-1] == container_id: return kv_get(service).get('params')
def _get_restart_parameters(self, container_id): try: docker_api = docker_client.api() docker_inspect = docker_api.inspect_container(container_id) for env_var in docker_inspect['Config']['Env']: env_key, env_value = (env_var.strip('"').split('=', 1) + [''])[:2] if env_key == 'RESTART_CONTAINER_PARAMETERS': return json.loads(base64.b64decode(env_value).decode()) except NotFound: for service in get_services_by_ship(ship=None): if service.split('/')[-1] == container_id: return kv_get(service).get('params')
def get_list(): result_list = [] # Contains dictionaries: # {'name': ..., 'is_default':..., 'address':..., ['user':...], ['password':...]) default_alias = kv.kv_get('dockyard/default') aliases_key = 'dockyard/aliases/' prefixed_aliases = kv.kv_list(aliases_key) or [] for prefixed_alias in sorted(prefixed_aliases): alias_name = prefixed_alias[len(aliases_key):] row = { 'name': alias_name, 'is_default': default_alias == alias_name, } row.update(get_alias(alias_name)) result_list.append(row) return result_list
def _get_inactive_services_list(filter_microservice_name, filter_env, filter_app_id): services_list = kv.kv_list("service/") result = [] if not services_list: return result names = set([service.split('/')[1] for service in services_list]) if filter_microservice_name: names = fnmatch.filter(names, filter_microservice_name) for name in names: instances = kv.kv_list('service/{}/'.format(name)) if instances is None: continue for instance in instances: instance_dict = kv.kv_get(instance) microservice_name = instance_dict['ServiceName'] microservice_status = instance_dict['Status'] not_available = 'n/a' container_id = instance_dict[ 'container_id'] if 'container_id' in instance_dict else not_available microservice_start_timestamp = instance_dict['start_timestamp'] microservice_tags_dict = {} if instance_dict['params']['microservice_env']: microservice_tags_dict['env'] = instance_dict['params'][ 'microservice_env'] if instance_dict['params']['microservice_app_id']: microservice_tags_dict['app_id'] = instance_dict['params'][ 'microservice_app_id'] matches_env = (filter_env is None) or ( filter_env == microservice_tags_dict.get('env')) matches_app_id = (filter_app_id is None) or ( filter_app_id == microservice_tags_dict.get('app_id')) if matches_env and matches_app_id: microservice_dict = { 'name': microservice_name, 'status': microservice_status, 'address': not_available, 'microservice_id': not_available, 'container_id': container_id, 'tags': microservice_tags_dict, 'start_timestamp': microservice_start_timestamp, } result.append(microservice_dict) return result
def on_post(self, req, resp): consul_host, error = self.get_post_parameter(req, 'host') if error: return self.status_error(resp, error) ship = get_ship_name() local_services_data = { key: kv.kv_get(key) for key in get_local_services_from_kv_store() } armada_size = _get_armada_size() if armada_size > 1: return self.status_error( resp, 'Currently only single ship armadas can join the others. ' 'Your armada has size: {0}.'.format(armada_size)) try: agent_self_dict = consul_query( 'agent/self', consul_address='{0}:8500'.format(consul_host)) datacenter = agent_self_dict['Config']['Datacenter'] except Exception as e: get_logger().exception(e) return self.status_error( resp, 'Could not read remote host datacenter address.') current_consul_mode = _get_current_consul_mode() if current_consul_mode == consul_config.ConsulMode.BOOTSTRAP: override_runtime_settings( consul_mode=consul_config.ConsulMode.CLIENT, ship_ips=[consul_host], datacenter=datacenter) else: override_runtime_settings(ship_ips=[consul_host] + get_other_ship_ips(), datacenter=datacenter) if _restart_consul(): supervisor_server = xmlrpc.client.Server( 'http://localhost:9001/RPC2') hermes_init_output = supervisor_server.supervisor.startProcessGroup( 'hermes_init') get_logger().info('hermes_init start: %s', hermes_init_output) set_ship_name(ship) for key, data in six.iteritems(local_services_data): kv.kv_set(key, data) return self.status_ok(resp) return self.status_error(resp, 'Waiting for armada restart timed out.')
def set_ship_name(new_name): ship_ip = get_ship_ip() old_name = get_ship_name(ship_ip) saved_containers = kv.kv_list('ships/{}/service/'.format(old_name)) if saved_containers: for container in saved_containers: new_key = 'ships/{}/service/{}/{}'.format(new_name, container.split('/')[-2], container.split('/')[-1]) container_dict = kv.kv_get(container) kv.kv_set(new_key, container_dict) kv.kv_remove(container) kv.kv_set('ships/{}/name'.format(ship_ip), new_name) kv.kv_set('ships/{}/ip'.format(new_name), ship_ip) os.system('sed -i \'s|ships/{}/|ships/{}/|\' /etc/consul.config'.format(old_name, new_name)) try: os.system('/usr/local/bin/consul reload') except Exception as e: get_logger().exception(e) kv.kv_remove('containers_parameters_list/{}'.format(old_name))
def set_ship_name(new_name): ship_ip = get_ship_ip() old_name = get_ship_name(ship_ip) saved_containers = kv.kv_list('ships/{}/service/'.format(old_name)) if saved_containers: for container in saved_containers: new_key = 'ships/{}/service/{}/{}'.format(new_name, container.split('/')[-2], container.split('/')[-1]) container_dict = kv.kv_get(container) kv.kv_set(new_key, container_dict) kv.kv_remove(container) kv.kv_set('ships/{}/name'.format(ship_ip), new_name) kv.kv_set('ships/{}/ip'.format(new_name), ship_ip) os.system('sed -i \'s|ships/{}/|ships/{}/|\' /etc/consul.config'.format(old_name, new_name)) try: os.system('/usr/local/bin/consul reload') except Exception as e: traceback.print_exc() kv.kv_remove('containers_parameters_list/{}'.format(old_name))
def _get_inactive_services_list(filter_microservice_name, filter_env, filter_app_id): services_list = kv.kv_list("service/") result = [] if not services_list: return result names = set([service.split('/')[1] for service in services_list]) if filter_microservice_name: names = fnmatch.filter(names, filter_microservice_name) for name in names: instances = kv.kv_list('service/{}/'.format(name)) if instances is None: continue for instance in instances: instance_dict = kv.kv_get(instance) microservice_name = instance_dict['ServiceName'] microservice_status = instance_dict['Status'] not_available = 'n/a' container_id = instance_dict['container_id'] if 'container_id' in instance_dict else not_available microservice_start_timestamp = instance_dict['start_timestamp'] microservice_tags_dict = {} if instance_dict['params']['microservice_env']: microservice_tags_dict['env'] = instance_dict['params']['microservice_env'] if instance_dict['params']['microservice_app_id']: microservice_tags_dict['app_id'] = instance_dict['params']['microservice_app_id'] matches_env = (filter_env is None) or (filter_env == microservice_tags_dict.get('env')) matches_app_id = (filter_app_id is None) or (filter_app_id == microservice_tags_dict.get('app_id')) if matches_env and matches_app_id: microservice_dict = { 'name': microservice_name, 'status': microservice_status, 'address': not_available, 'microservice_id': not_available, 'container_id': container_id, 'tags': microservice_tags_dict, 'start_timestamp': microservice_start_timestamp, } result.append(microservice_dict) return result
def _choose_active_instances(services_dicts): result = services_dicts running_services_with_single_active_instances = {} for microservice_id, service_dict in services_dicts.items(): if service_dict.get('single_active_instance') and service_dict['status'] in ('passing', 'warning'): key = 'chosen_active_instance/{},env={},app_id={}'.format(service_dict['name'], service_dict['tags'].get('env') or '', service_dict['tags'].get('app_id') or '') if key not in running_services_with_single_active_instances: running_services_with_single_active_instances[key] = set() running_services_with_single_active_instances[key].add(microservice_id) for key, running_microservice_ids in running_services_with_single_active_instances.items(): currently_picked_instance = kv.kv_get(key) if currently_picked_instance not in running_microservice_ids: currently_picked_instance = random.choice(list(running_microservice_ids)) kv.kv_set(key, currently_picked_instance) for microservice_id in running_microservice_ids: if microservice_id != currently_picked_instance: result[microservice_id]['status'] = 'standby' return result
def set_ship_name(new_name): from armada_backend.models.services import get_services_by_ship, create_consul_services_key ship_ip = get_ship_ip() old_name = get_ship_name(ship_ip) saved_containers = get_services_by_ship(old_name) if saved_containers: for container in saved_containers: new_key = create_consul_services_key( ship=new_name, service_name=container.split('/')[-2], container_id=container.split('/')[-1]) container_dict = kv.kv_get(container) kv.kv_set(new_key, container_dict) kv.kv_remove(container) kv.kv_set('ships/{}/name'.format(ship_ip), new_name) kv.kv_set('ships/{}/ip'.format(new_name), ship_ip) os.system('sed -i \'s|ships/{}/|ships/{}/|\' /etc/consul.config'.format( old_name, new_name)) try: os.system('/usr/local/bin/consul reload') except Exception as e: get_logger().exception(e)
def _stop_service(self, container_id): ship = get_ship_name() service_dict = None service_list = kv_list('ships/{}/service/'.format(ship)) if service_list: key = fnmatch.filter(service_list, '*/{}'.format(container_id)) service_dict = kv_get(key[0]) if key else None if service_dict and service_dict['Status'] in [ 'crashed', 'not-recovered' ]: kv_remove(key[0]) else: run_command_in_container('supervisorctl stop armada_agent', container_id) # TODO: Compatibility with old microservice images. Should be removed in future armada version. run_command_in_container( 'supervisorctl stop register_in_service_discovery', container_id) docker_api = docker_client.api() last_exception = None try: deregister_services(container_id) except: traceback.print_exc() for i in range(3): try: docker_api.stop(container_id) kv_remove(key[0]) except Exception as e: last_exception = e traceback.print_exc() if not is_container_running(container_id): break if is_container_running(container_id): get_logger().error( 'Could not stop container: {}'.format(container_id)) raise last_exception
def POST(self): consul_host, error = self.get_post_parameter('host') if error: return self.status_error(error) ship = get_ship_name() local_services = kv.kv_list('ships/{}/service/'.format(ship)) or [] local_services_data = {key: kv.kv_get(key) for key in local_services} armada_size = _get_armada_size() if armada_size > 1: return self.status_error('Currently only single ship armadas can join the others. ' 'Your armada has size: {0}.'.format(armada_size)) try: agent_self_dict = consul_query('agent/self', consul_address='{0}:8500'.format(consul_host)) datacenter = agent_self_dict['Config']['Datacenter'] except: return self.status_error('Could not read remote host datacenter address.') current_consul_mode = _get_current_consul_mode() if current_consul_mode == consul_config.ConsulMode.BOOTSTRAP: override_runtime_settings(consul_mode=consul_config.ConsulMode.CLIENT, ship_ips=[consul_host], datacenter=datacenter) else: override_runtime_settings(ship_ips=[consul_host] + get_other_ship_ips(), datacenter=datacenter) if _restart_consul(): supervisor_server = xmlrpclib.Server('http://localhost:9001/RPC2') hermes_init_output = supervisor_server.supervisor.startProcessGroup('hermes_init') get_logger().info('hermes_init start: {}'.format(hermes_init_output)) set_ship_name(ship) for key, data in local_services_data.items(): kv.kv_set(key, data) return self.status_ok() return self.status_error('Waiting for armada restart timed out.')
def get_default(): return kv.kv_get('dockyard/default')
def GET(self): try: get_args = web.input(local=False, microservice_name=None, env=None, app_id=None) filter_local = bool(get_args.local and strtobool(str(get_args.local))) filter_microservice_name = get_args.microservice_name filter_env = get_args.env filter_app_id = get_args.app_id if filter_local: local_microservices_ids = set(consul_query('agent/services').keys()) if filter_microservice_name: names = list(consul_query('catalog/services').keys()) microservices_names = fnmatch.filter(names, filter_microservice_name) else: microservices_names = list(consul_query('catalog/services').keys()) result = [] for microservice_name in microservices_names: if microservice_name == 'consul': continue query = 'health/service/{microservice_name}'.format(**locals()) instances = consul_query(query) for instance in instances: microservice_checks_statuses = set(check['Status'] for check in (instance['Checks'] or [])) microservice_computed_status = '-' for possible_status in ['passing', 'warning', 'critical']: if possible_status in microservice_checks_statuses: microservice_computed_status = possible_status microservice_ip = instance['Node']['Address'] microservice_port = str(instance['Service']['Port']) microservice_id = instance['Service']['ID'] container_id = microservice_id.split(':')[0] microservice_tags = instance['Service']['Tags'] or [] microservice_tags_dict = self.__create_dict_from_tags(microservice_tags) matches_env = (filter_env is None) or (filter_env == microservice_tags_dict.get('env')) matches_app_id = (filter_app_id is None) or (filter_app_id == microservice_tags_dict.get('app_id')) if (matches_env and matches_app_id and (not filter_local or microservice_id in local_microservices_ids)): microservice_address = microservice_ip + ':' + microservice_port try: microservice_start_timestamp = kv.kv_get("start_timestamp/" + container_id) except: microservice_start_timestamp = None microservice_dict = { 'name': microservice_name, 'address': microservice_address, 'microservice_id': microservice_id, 'container_id': container_id, 'status': microservice_computed_status, 'tags': microservice_tags_dict, 'start_timestamp': microservice_start_timestamp, } result.append(microservice_dict) inactive_services_list = _get_inactive_services_list(filter_microservice_name, filter_env, filter_app_id) result.extend(inactive_services_list) return self.status_ok({'result': result}) except Exception as e: traceback.print_exc() return self.status_exception("Cannot get the list of services.", e)
def update_service_dict(ship, service_name, container_id, key, value): consul_key = create_consul_services_key(ship, service_name, container_id) service_dict = kv_get(consul_key) service_dict[key] = value kv_set(consul_key, service_dict)
def ship_ip_to_name(ip): return kv.kv_get('ships/{}/name'.format(ip))
def GET(self): try: get_args = web.input(local=False, microservice_name=None, env=None, app_id=None) filter_local = bool(get_args.local and strtobool(str(get_args.local))) filter_microservice_name = get_args.microservice_name filter_env = get_args.env filter_app_id = get_args.app_id services_list = _get_services_list(filter_microservice_name, filter_env, filter_app_id, filter_local) if filter_local: local_microservices_ids = set( consul_query('agent/services').keys()) if filter_microservice_name: names = list(consul_query('catalog/services').keys()) microservices_names = fnmatch.filter(names, filter_microservice_name) else: microservices_names = list( consul_query('catalog/services').keys()) services_list_from_catalog = {} for microservice_name in microservices_names: if microservice_name == 'consul': continue query = 'health/service/{microservice_name}'.format(**locals()) instances = consul_query(query) for instance in instances: microservice_checks_statuses = set( check['Status'] for check in (instance['Checks'] or [])) microservice_computed_status = '-' for possible_status in ['passing', 'warning', 'critical']: if possible_status in microservice_checks_statuses: microservice_computed_status = possible_status microservice_ip = instance['Node']['Address'] microservice_port = str(instance['Service']['Port']) microservice_id = instance['Service']['ID'] container_id = microservice_id.split(':')[0] microservice_tags = instance['Service']['Tags'] or [] microservice_tags_dict = self.__create_dict_from_tags( microservice_tags) matches_env = (filter_env is None) or ( filter_env == microservice_tags_dict.get('env')) matches_app_id = (filter_app_id is None) or ( filter_app_id == microservice_tags_dict.get('app_id')) if (matches_env and matches_app_id and (not filter_local or microservice_id in local_microservices_ids)): microservice_address = microservice_ip + ':' + microservice_port try: microservice_start_timestamp = kv.kv_get( "start_timestamp/" + container_id) except: microservice_start_timestamp = None try: single_active_instance = kv.kv_get( "single_active_instance/" + microservice_id) except: single_active_instance = False microservice_dict = { 'name': microservice_name, 'address': microservice_address, 'microservice_id': microservice_id, 'container_id': container_id, 'status': microservice_computed_status, 'tags': microservice_tags_dict, 'start_timestamp': microservice_start_timestamp, 'single_active_instance': single_active_instance, } services_list_from_catalog[ microservice_id] = microservice_dict result = services_list result.update(services_list_from_catalog) result = _choose_active_instances(result) return self.status_ok({'result': result.values()}) except Exception as e: return self.status_exception("Cannot get the list of services.", e)
def _get_services_list(filter_microservice_name, filter_env, filter_app_id, filter_local): if filter_local: ship_list = [get_ship_name()] else: ship_list = get_ship_names() services_dict = {} if not ship_list: return {} for ship in ship_list: containers = kv.kv_get('containers_parameters_list/{}'.format(ship)) if containers and isinstance(containers, dict): services_dict.update(containers) services_list = services_dict.keys() result = {} if not services_list: return result if filter_microservice_name: services_list = fnmatch.filter( services_list, 'ships/*/service/{}/*'.format(filter_microservice_name)) for service in services_list: service_dict = services_dict[service] microservice_name = service_dict['ServiceName'] microservice_status = service_dict['Status'] microservice_id = service_dict['ServiceID'] container_id = service_dict['container_id'] microservice_start_timestamp = service_dict['start_timestamp'] single_active_instance = service_dict.get('single_active_instance', False) not_available = 'n/a' microservice_tags_dict = {} try: if service_dict['params']['microservice_env']: microservice_tags_dict['env'] = service_dict['params'][ 'microservice_env'] if service_dict['params']['microservice_app_id']: microservice_tags_dict['app_id'] = service_dict['params'][ 'microservice_app_id'] except KeyError as e: get_logger().warning(repr(e)) matches_env = (filter_env is None) or ( filter_env == microservice_tags_dict.get('env')) matches_app_id = (filter_app_id is None) or ( filter_app_id == microservice_tags_dict.get('app_id')) if matches_env and matches_app_id: microservice_dict = { 'name': microservice_name, 'status': microservice_status, 'address': not_available, 'microservice_id': microservice_id, 'container_id': container_id, 'tags': microservice_tags_dict, 'start_timestamp': microservice_start_timestamp, 'single_active_instance': single_active_instance, } result[microservice_id] = microservice_dict return result
def ship_name_to_ip(name): return kv.kv_get('ships/{}/ip'.format(name))
def get_ship_name(ship_ip=None): if ship_ip is None: ship_ip = get_ship_ip() ship_name = kv.kv_get('ships/{}/name'.format(ship_ip)) or ship_ip return ship_name
def get_initialized(): return kv.kv_get('dockyard/initialized') == '1'
def get_alias(name): key = 'dockyard/aliases/{name}'.format(**locals()) return kv.kv_get(key)
def get_matched_containers(microservice_name_or_container_id_prefix): service_names = list(consul_query('catalog/services').keys()) matched_containers_by_name = [] matched_containers_by_id = [] for service_name in service_names: query = 'catalog/service/{service_name}'.format(**locals()) try: instances = consul_query(query) except Exception as e: print_err( 'WARNING: query "{query}" failed ({exception_class}: {exception})' .format(query=query, exception_class=type(e).__name__, exception=e)) instances = [] for instance in instances: container_id = instance['ServiceID'].split(':')[0] service_name = instance['ServiceName'] if microservice_name_or_container_id_prefix == service_name: matched_containers_by_name.append(instance) if container_id.startswith(microservice_name_or_container_id_prefix ) and ":" not in instance['ServiceID']: matched_containers_by_id.append(instance) services_list = kv.kv_list('services/') if services_list: for service in services_list: service_dict = kv.kv_get(service) container_id = service_dict['container_id'] service_name = service_dict['ServiceName'] if service_dict['Status'] == 'started': try: instances = consul_query( 'catalog/service/{}'.format(service_name)) if container_id in [ i['ServiceID'].split(':')[0] for i in instances ]: continue except Exception as e: logging.exception(e) if microservice_name_or_container_id_prefix == service_name: matched_containers_by_name.append(service_dict) if container_id.startswith(microservice_name_or_container_id_prefix) \ and ":" not in service_dict['ServiceID']: matched_containers_by_id.append(service_dict) matched_containers_by_name_count = len(matched_containers_by_name) matched_containers_by_id_count = len(matched_containers_by_id) if matched_containers_by_name_count and matched_containers_by_id_count: raise ArmadaCommandException( 'Found matching containers with both microservice name ({matched_containers_by_name_count}) ' 'and container_id ({matched_containers_by_id_count}). ' 'Please provide more specific criteria.'.format(**locals())) if matched_containers_by_id_count > 1 and len( microservice_name_or_container_id_prefix) < 12: raise ArmadaCommandException( 'There are too many ({matched_containers_by_id_count}) matching containers. ' 'Please provide more specific container_id.'.format(**locals())) matched_containers = matched_containers_by_name + matched_containers_by_id matches_count = len(matched_containers) if matches_count == 0: raise ArmadaCommandException( 'There are no running containers with microservice: ' '{microservice_name_or_container_id_prefix}'.format(**locals())) return matched_containers