def save_container(ship, container_id, status, params=None):
try:
start_timestamp = kv_get('start_timestamp/{}'.format(container_id))
except:
start_timestamp = None
if status == 'crashed':
service_name = params['microservice_name']
else:
service_name = get_env(container_id, 'MICROSERVICE_NAME')
params = json.loads(
base64.b64decode(
get_env(container_id, 'RESTART_CONTAINER_PARAMETERS')))
if not start_timestamp:
start_timestamp = str(calendar.timegm(time.gmtime()))
address = kv_get('ships/{}/ip'.format(ship)) or ship
service_dict = {
'ServiceName': service_name,
'Status': status,
'container_id': container_id,
'params': params,
'start_timestamp': start_timestamp,
'ServiceID': container_id,
'Address': address
}
kv_set(create_consul_services_key(ship, service_name, container_id),
service_dict)
def recover_containers_from_kv_store():
services_to_be_recovered = _get_crashed_services()
for service in services_to_be_recovered:
kv.update_container_status('recovering', key=service)
recovery_retry_count = 0
while services_to_be_recovered and recovery_retry_count < RECOVERY_RETRY_LIMIT:
get_logger().info("Recovering containers: %s",
json.dumps(services_to_be_recovered))
services_not_recovered = []
for service in services_to_be_recovered:
service_parameters = kv.kv_get(service)['params']
if not _recover_container(service_parameters):
services_not_recovered.append(service)
else:
kv.kv_remove(service)
sleep(DELAY_BETWEEN_RECOVER_RETRY_SECONDS)
services_to_be_recovered = services_not_recovered
recovery_retry_count += 1
for service in services_to_be_recovered:
kv.update_container_status('not-recovered', key=service)
return services_to_be_recovered
def _stop_service(self, container_id):
ship = get_ship_name()
service_dict = None
service_list = kv_list('ships/{}/service/'.format(ship))
if service_list:
key = fnmatch.filter(service_list, '*/{}'.format(container_id))
service_dict = kv_get(key[0]) if key else None
if service_dict and service_dict['Status'] in ['crashed', 'not-recovered']:
kv_remove(key[0])
else:
run_command_in_container('supervisorctl stop armada_agent', container_id)
# TODO: Compatibility with old microservice images. Should be removed in future armada version.
run_command_in_container('supervisorctl stop register_in_service_discovery', container_id)
docker_api = docker_client.api()
last_exception = None
try:
deregister_services(container_id)
except:
traceback.print_exc()
for i in range(3):
try:
docker_api.stop(container_id)
kv_remove(key[0])
except Exception as e:
last_exception = e
traceback.print_exc()
if not is_container_running(container_id):
break
if is_container_running(container_id):
get_logger().error('Could not stop container: {}'.format(container_id))
raise last_exception
def recover_containers_from_kv_store():
services_to_be_recovered = _get_crashed_services()
for service in services_to_be_recovered:
kv.update_container_status('recovering', key=service)
recovery_retry_count = 0
while services_to_be_recovered and recovery_retry_count < RECOVERY_RETRY_LIMIT:
get_logger().info("Recovering containers: %s", json.dumps(services_to_be_recovered))
services_not_recovered = []
for service in services_to_be_recovered:
service_parameters = kv.kv_get(service)['params']
if not _recover_container(service_parameters):
services_not_recovered.append(service)
else:
kv.kv_remove(service)
sleep(DELAY_BETWEEN_RECOVER_RETRY_SECONDS)
services_to_be_recovered = services_not_recovered
recovery_retry_count += 1
for service in services_to_be_recovered:
kv.update_container_status('not-recovered', key=service)
return services_to_be_recovered
def _choose_active_instances(services_dicts):
result = services_dicts
running_services_with_single_active_instances = {}
for microservice_id, service_dict in services_dicts.items():
if service_dict.get('single_active_instance'
) and service_dict['status'] in ('passing',
'warning'):
key = 'chosen_active_instance/{},env={},app_id={}'.format(
service_dict['name'], service_dict['tags'].get('env') or '',
service_dict['tags'].get('app_id') or '')
if key not in running_services_with_single_active_instances:
running_services_with_single_active_instances[key] = set()
running_services_with_single_active_instances[key].add(
microservice_id)
for key, running_microservice_ids in running_services_with_single_active_instances.items(
):
currently_picked_instance = kv.kv_get(key)
if currently_picked_instance not in running_microservice_ids:
currently_picked_instance = random.choice(
list(running_microservice_ids))
kv.kv_set(key, currently_picked_instance)
for microservice_id in running_microservice_ids:
if microservice_id != currently_picked_instance:
result[microservice_id]['status'] = 'standby'
return result
def _get_local_running_containers():
result = []
for container in get_local_services():
container_parameters = kv.kv_get(container)['params']
if container_parameters:
result.append(container_parameters)
return result
def main():
setup_sentry()
args = _parse_args()
saved_containers_path = args.saved_containers_path
if not args.force and not _is_recovery_completed():
get_logger().info(
'Recovery is not completed. Aborting saving running containers.')
return
try:
wait_for_consul_ready()
saved_containers = get_local_services()
containers_parameters_dict = {}
for container in saved_containers:
container_dict = kv.kv_get(container)
containers_parameters_dict[container] = container_dict
if not containers_parameters_dict:
get_logger().info(
'Aborted saving container because list is empty.')
return
_save_containers_parameters_list_in_file(containers_parameters_dict,
saved_containers_path)
get_logger().info(
'Containers have been saved to {}.'.format(saved_containers_path))
except Exception as e:
get_logger().exception(e)
sys.exit(1)
def main():
setup_sentry()
args = _parse_args()
saved_containers_path = args.saved_containers_path
try:
wait_for_consul_ready()
ship = get_ship_name()
saved_containers = kv.kv_list('ships/{}/service/'.format(ship))
containers_parameters_dict = {}
if saved_containers:
for container in saved_containers:
container_dict = kv.kv_get(container)
containers_parameters_dict[container] = container_dict
if containers_parameters_dict:
try:
_save_containers_parameters_list_in_kv_store(containers_parameters_dict)
get_logger().info('Containers have been saved to kv store.')
except Exception as e:
get_logger().exception(e)
if not args.force and not _is_recovery_completed():
get_logger().warning('Recovery is not completed. Aborting saving running containers.')
return
_save_containers_parameters_list_in_file(containers_parameters_dict, saved_containers_path)
get_logger().info('Containers have been saved to {}.'.format(saved_containers_path))
else:
get_logger().info('Aborted saving container because of errors.')
except Exception as e:
get_logger().exception(e)
sys.exit(1)
def update_container_status(status, ship=None, service_name=None, container_id=None, key=None):
if not key:
key = create_consul_services_key(ship, service_name, container_id)
service_dict = kv_get(key)
if status == 'crashed' and service_dict['Status'] in ['not-recovered', 'recovering']:
return
service_dict['Status'] = status
kv_set(key, service_dict)
def _get_services_list(filter_microservice_name, filter_env, filter_app_id,
filter_local):
if filter_local:
ship_list = ['containers_parameters_list/{}'.format(get_ship_name())]
else:
ship_list = kv.kv_list('containers_parameters_list/')
services_dict = {}
if not ship_list:
return {}
for ship in ship_list:
containers = kv.kv_get(ship)
if containers:
services_dict.update(containers)
services_list = services_dict.keys()
result = {}
if not services_list:
return result
if filter_microservice_name:
services_list = fnmatch.filter(
services_list,
'ships/*/service/{}/*'.format(filter_microservice_name))
for service in services_list:
service_dict = services_dict[service]
microservice_name = service_dict['ServiceName']
microservice_status = service_dict['Status']
microservice_id = service_dict['ServiceID']
container_id = service_dict['container_id']
microservice_start_timestamp = service_dict['start_timestamp']
not_available = 'n/a'
microservice_tags_dict = {}
if service_dict['params']['microservice_env']:
microservice_tags_dict['env'] = service_dict['params'][
'microservice_env']
if service_dict['params']['microservice_app_id']:
microservice_tags_dict['app_id'] = service_dict['params'][
'microservice_app_id']
matches_env = (filter_env is None) or (
filter_env == microservice_tags_dict.get('env'))
matches_app_id = (filter_app_id is None) or (
filter_app_id == microservice_tags_dict.get('app_id'))
if matches_env and matches_app_id:
microservice_dict = {
'name': microservice_name,
'status': microservice_status,
'address': not_available,
'microservice_id': microservice_id,
'container_id': container_id,
'tags': microservice_tags_dict,
'start_timestamp': microservice_start_timestamp,
}
result[microservice_id] = microservice_dict
return result
def _get_local_running_containers():
result = []
ship = get_ship_name()
local_containers = kv.kv_list('ships/{}/service/'.format(ship)) or []
for container in local_containers:
container_parameters = kv.kv_get(container)['params']
if container_parameters:
result.append(container_parameters)
return result
def _get_local_running_containers():
result = []
ship = get_ship_name()
local_containers = kv.kv_list('ships/{}/service/'.format(ship)) or []
for container in local_containers:
container_parameters = kv.kv_get(container)['params']
if container_parameters:
result.append(container_parameters)
return result
def _get_services_list(filter_microservice_name, filter_env, filter_app_id, filter_local):
if filter_local:
ship_list = [get_ship_name()]
else:
ship_list = get_ship_names()
services_dict = {}
if not ship_list:
return {}
for ship in ship_list:
containers = kv.kv_get('containers_parameters_list/{}'.format(ship))
if containers and isinstance(containers, dict):
services_dict.update(containers)
services_list = services_dict.keys()
result = {}
if not services_list:
return result
if filter_microservice_name:
services_list = fnmatch.filter(services_list, 'ships/*/service/{}/*'.format(filter_microservice_name))
for service in services_list:
service_dict = services_dict[service]
microservice_name = service_dict['ServiceName']
microservice_status = service_dict['Status']
microservice_id = service_dict['ServiceID']
container_id = service_dict['container_id']
microservice_start_timestamp = service_dict['start_timestamp']
not_available = 'n/a'
microservice_tags_dict = {}
try:
if service_dict['params']['microservice_env']:
microservice_tags_dict['env'] = service_dict['params']['microservice_env']
if service_dict['params']['microservice_app_id']:
microservice_tags_dict['app_id'] = service_dict['params']['microservice_app_id']
except KeyError as e:
get_logger().warning(repr(e))
matches_env = (filter_env is None) or (filter_env == microservice_tags_dict.get('env'))
matches_app_id = (filter_app_id is None) or (filter_app_id == microservice_tags_dict.get('app_id'))
if matches_env and matches_app_id:
microservice_dict = {
'name': microservice_name,
'status': microservice_status,
'address': not_available,
'microservice_id': microservice_id,
'container_id': container_id,
'tags': microservice_tags_dict,
'start_timestamp': microservice_start_timestamp,
}
result[microservice_id] = microservice_dict
return result
def _get_crashed_services():
services_list = get_local_services()
crashed_services = []
for service in services_list:
service_dict = kv.kv_get(service)
microservice_status = service_dict['Status']
if microservice_status in ['crashed', 'not-recovered']:
crashed_services.append(service)
return crashed_services
def _get_crashed_services():
ship = get_ship_name()
services_list = kv.kv_list('ships/{}/service/'.format(ship))
crashed_services = []
if not services_list:
return crashed_services
for service in services_list:
service_dict = kv.kv_get(service)
microservice_status = service_dict['Status']
if microservice_status in ['crashed', 'not-recovered']:
crashed_services.append(service)
return crashed_services
def _get_crashed_services():
ship = get_ship_name()
services_list = kv.kv_list('ships/{}/service/'.format(ship))
crashed_services = []
if not services_list:
return crashed_services
for service in services_list:
service_dict = kv.kv_get(service)
microservice_status = service_dict['Status']
if microservice_status in ['crashed', 'not-recovered']:
crashed_services.append(service)
return crashed_services
def _get_restart_parameters(self, container_id):
try:
docker_api = docker_client.api()
docker_inspect = docker_api.inspect_container(container_id)
for env_var in docker_inspect['Config']['Env']:
env_key, env_value = (env_var.strip('"').split('=', 1) + [''])[:2]
if env_key == 'RESTART_CONTAINER_PARAMETERS':
return json.loads(base64.b64decode(env_value))
except NotFound:
service_list = kv_list('ships/')
for service in service_list:
if service.split('/')[-1] == container_id:
return kv_get(service).get('params')
def _get_restart_parameters(self, container_id):
try:
docker_api = docker_client.api()
docker_inspect = docker_api.inspect_container(container_id)
for env_var in docker_inspect['Config']['Env']:
env_key, env_value = (env_var.strip('"').split('=', 1) +
[''])[:2]
if env_key == 'RESTART_CONTAINER_PARAMETERS':
return json.loads(base64.b64decode(env_value).decode())
except NotFound:
for service in get_services_by_ship(ship=None):
if service.split('/')[-1] == container_id:
return kv_get(service).get('params')
def get_list():
result_list = [] # Contains dictionaries:
# {'name': ..., 'is_default':..., 'address':..., ['user':...], ['password':...])
default_alias = kv.kv_get('dockyard/default')
aliases_key = 'dockyard/aliases/'
prefixed_aliases = kv.kv_list(aliases_key) or []
for prefixed_alias in sorted(prefixed_aliases):
alias_name = prefixed_alias[len(aliases_key):]
row = {
'name': alias_name,
'is_default': default_alias == alias_name,
}
row.update(get_alias(alias_name))
result_list.append(row)
return result_list
def get_list():
result_list = [] # Contains dictionaries:
# {'name': ..., 'is_default':..., 'address':..., ['user':...], ['password':...])
default_alias = kv.kv_get('dockyard/default')
aliases_key = 'dockyard/aliases/'
prefixed_aliases = kv.kv_list(aliases_key) or []
for prefixed_alias in sorted(prefixed_aliases):
alias_name = prefixed_alias[len(aliases_key):]
row = {
'name': alias_name,
'is_default': default_alias == alias_name,
}
row.update(get_alias(alias_name))
result_list.append(row)
return result_list
def _get_inactive_services_list(filter_microservice_name, filter_env,
filter_app_id):
services_list = kv.kv_list("service/")
result = []
if not services_list:
return result
names = set([service.split('/')[1] for service in services_list])
if filter_microservice_name:
names = fnmatch.filter(names, filter_microservice_name)
for name in names:
instances = kv.kv_list('service/{}/'.format(name))
if instances is None:
continue
for instance in instances:
instance_dict = kv.kv_get(instance)
microservice_name = instance_dict['ServiceName']
microservice_status = instance_dict['Status']
not_available = 'n/a'
container_id = instance_dict[
'container_id'] if 'container_id' in instance_dict else not_available
microservice_start_timestamp = instance_dict['start_timestamp']
microservice_tags_dict = {}
if instance_dict['params']['microservice_env']:
microservice_tags_dict['env'] = instance_dict['params'][
'microservice_env']
if instance_dict['params']['microservice_app_id']:
microservice_tags_dict['app_id'] = instance_dict['params'][
'microservice_app_id']
matches_env = (filter_env is None) or (
filter_env == microservice_tags_dict.get('env'))
matches_app_id = (filter_app_id is None) or (
filter_app_id == microservice_tags_dict.get('app_id'))
if matches_env and matches_app_id:
microservice_dict = {
'name': microservice_name,
'status': microservice_status,
'address': not_available,
'microservice_id': not_available,
'container_id': container_id,
'tags': microservice_tags_dict,
'start_timestamp': microservice_start_timestamp,
}
result.append(microservice_dict)
return result
def on_post(self, req, resp):
consul_host, error = self.get_post_parameter(req, 'host')
if error:
return self.status_error(resp, error)
ship = get_ship_name()
local_services_data = {
key: kv.kv_get(key)
for key in get_local_services_from_kv_store()
}
armada_size = _get_armada_size()
if armada_size > 1:
return self.status_error(
resp,
'Currently only single ship armadas can join the others. '
'Your armada has size: {0}.'.format(armada_size))
try:
agent_self_dict = consul_query(
'agent/self', consul_address='{0}:8500'.format(consul_host))
datacenter = agent_self_dict['Config']['Datacenter']
except Exception as e:
get_logger().exception(e)
return self.status_error(
resp, 'Could not read remote host datacenter address.')
current_consul_mode = _get_current_consul_mode()
if current_consul_mode == consul_config.ConsulMode.BOOTSTRAP:
override_runtime_settings(
consul_mode=consul_config.ConsulMode.CLIENT,
ship_ips=[consul_host],
datacenter=datacenter)
else:
override_runtime_settings(ship_ips=[consul_host] +
get_other_ship_ips(),
datacenter=datacenter)
if _restart_consul():
supervisor_server = xmlrpc.client.Server(
'http://localhost:9001/RPC2')
hermes_init_output = supervisor_server.supervisor.startProcessGroup(
'hermes_init')
get_logger().info('hermes_init start: %s', hermes_init_output)
set_ship_name(ship)
for key, data in six.iteritems(local_services_data):
kv.kv_set(key, data)
return self.status_ok(resp)
return self.status_error(resp, 'Waiting for armada restart timed out.')
def set_ship_name(new_name):
ship_ip = get_ship_ip()
old_name = get_ship_name(ship_ip)
saved_containers = kv.kv_list('ships/{}/service/'.format(old_name))
if saved_containers:
for container in saved_containers:
new_key = 'ships/{}/service/{}/{}'.format(new_name, container.split('/')[-2], container.split('/')[-1])
container_dict = kv.kv_get(container)
kv.kv_set(new_key, container_dict)
kv.kv_remove(container)
kv.kv_set('ships/{}/name'.format(ship_ip), new_name)
kv.kv_set('ships/{}/ip'.format(new_name), ship_ip)
os.system('sed -i \'s|ships/{}/|ships/{}/|\' /etc/consul.config'.format(old_name, new_name))
try:
os.system('/usr/local/bin/consul reload')
except Exception as e:
get_logger().exception(e)
kv.kv_remove('containers_parameters_list/{}'.format(old_name))
def set_ship_name(new_name):
ship_ip = get_ship_ip()
old_name = get_ship_name(ship_ip)
saved_containers = kv.kv_list('ships/{}/service/'.format(old_name))
if saved_containers:
for container in saved_containers:
new_key = 'ships/{}/service/{}/{}'.format(new_name, container.split('/')[-2], container.split('/')[-1])
container_dict = kv.kv_get(container)
kv.kv_set(new_key, container_dict)
kv.kv_remove(container)
kv.kv_set('ships/{}/name'.format(ship_ip), new_name)
kv.kv_set('ships/{}/ip'.format(new_name), ship_ip)
os.system('sed -i \'s|ships/{}/|ships/{}/|\' /etc/consul.config'.format(old_name, new_name))
try:
os.system('/usr/local/bin/consul reload')
except Exception as e:
traceback.print_exc()
kv.kv_remove('containers_parameters_list/{}'.format(old_name))
def _get_inactive_services_list(filter_microservice_name, filter_env, filter_app_id):
services_list = kv.kv_list("service/")
result = []
if not services_list:
return result
names = set([service.split('/')[1] for service in services_list])
if filter_microservice_name:
names = fnmatch.filter(names, filter_microservice_name)
for name in names:
instances = kv.kv_list('service/{}/'.format(name))
if instances is None:
continue
for instance in instances:
instance_dict = kv.kv_get(instance)
microservice_name = instance_dict['ServiceName']
microservice_status = instance_dict['Status']
not_available = 'n/a'
container_id = instance_dict['container_id'] if 'container_id' in instance_dict else not_available
microservice_start_timestamp = instance_dict['start_timestamp']
microservice_tags_dict = {}
if instance_dict['params']['microservice_env']:
microservice_tags_dict['env'] = instance_dict['params']['microservice_env']
if instance_dict['params']['microservice_app_id']:
microservice_tags_dict['app_id'] = instance_dict['params']['microservice_app_id']
matches_env = (filter_env is None) or (filter_env == microservice_tags_dict.get('env'))
matches_app_id = (filter_app_id is None) or (filter_app_id == microservice_tags_dict.get('app_id'))
if matches_env and matches_app_id:
microservice_dict = {
'name': microservice_name,
'status': microservice_status,
'address': not_available,
'microservice_id': not_available,
'container_id': container_id,
'tags': microservice_tags_dict,
'start_timestamp': microservice_start_timestamp,
}
result.append(microservice_dict)
return result
def _choose_active_instances(services_dicts):
result = services_dicts
running_services_with_single_active_instances = {}
for microservice_id, service_dict in services_dicts.items():
if service_dict.get('single_active_instance') and service_dict['status'] in ('passing', 'warning'):
key = 'chosen_active_instance/{},env={},app_id={}'.format(service_dict['name'],
service_dict['tags'].get('env') or '',
service_dict['tags'].get('app_id') or '')
if key not in running_services_with_single_active_instances:
running_services_with_single_active_instances[key] = set()
running_services_with_single_active_instances[key].add(microservice_id)
for key, running_microservice_ids in running_services_with_single_active_instances.items():
currently_picked_instance = kv.kv_get(key)
if currently_picked_instance not in running_microservice_ids:
currently_picked_instance = random.choice(list(running_microservice_ids))
kv.kv_set(key, currently_picked_instance)
for microservice_id in running_microservice_ids:
if microservice_id != currently_picked_instance:
result[microservice_id]['status'] = 'standby'
return result
def set_ship_name(new_name):
from armada_backend.models.services import get_services_by_ship, create_consul_services_key
ship_ip = get_ship_ip()
old_name = get_ship_name(ship_ip)
saved_containers = get_services_by_ship(old_name)
if saved_containers:
for container in saved_containers:
new_key = create_consul_services_key(
ship=new_name,
service_name=container.split('/')[-2],
container_id=container.split('/')[-1])
container_dict = kv.kv_get(container)
kv.kv_set(new_key, container_dict)
kv.kv_remove(container)
kv.kv_set('ships/{}/name'.format(ship_ip), new_name)
kv.kv_set('ships/{}/ip'.format(new_name), ship_ip)
os.system('sed -i \'s|ships/{}/|ships/{}/|\' /etc/consul.config'.format(
old_name, new_name))
try:
os.system('/usr/local/bin/consul reload')
except Exception as e:
get_logger().exception(e)
def _stop_service(self, container_id):
ship = get_ship_name()
service_dict = None
service_list = kv_list('ships/{}/service/'.format(ship))
if service_list:
key = fnmatch.filter(service_list, '*/{}'.format(container_id))
service_dict = kv_get(key[0]) if key else None
if service_dict and service_dict['Status'] in [
'crashed', 'not-recovered'
]:
kv_remove(key[0])
else:
run_command_in_container('supervisorctl stop armada_agent',
container_id)
# TODO: Compatibility with old microservice images. Should be removed in future armada version.
run_command_in_container(
'supervisorctl stop register_in_service_discovery',
container_id)
docker_api = docker_client.api()
last_exception = None
try:
deregister_services(container_id)
except:
traceback.print_exc()
for i in range(3):
try:
docker_api.stop(container_id)
kv_remove(key[0])
except Exception as e:
last_exception = e
traceback.print_exc()
if not is_container_running(container_id):
break
if is_container_running(container_id):
get_logger().error(
'Could not stop container: {}'.format(container_id))
raise last_exception
def POST(self):
consul_host, error = self.get_post_parameter('host')
if error:
return self.status_error(error)
ship = get_ship_name()
local_services = kv.kv_list('ships/{}/service/'.format(ship)) or []
local_services_data = {key: kv.kv_get(key) for key in local_services}
armada_size = _get_armada_size()
if armada_size > 1:
return self.status_error('Currently only single ship armadas can join the others. '
'Your armada has size: {0}.'.format(armada_size))
try:
agent_self_dict = consul_query('agent/self', consul_address='{0}:8500'.format(consul_host))
datacenter = agent_self_dict['Config']['Datacenter']
except:
return self.status_error('Could not read remote host datacenter address.')
current_consul_mode = _get_current_consul_mode()
if current_consul_mode == consul_config.ConsulMode.BOOTSTRAP:
override_runtime_settings(consul_mode=consul_config.ConsulMode.CLIENT,
ship_ips=[consul_host],
datacenter=datacenter)
else:
override_runtime_settings(ship_ips=[consul_host] + get_other_ship_ips(),
datacenter=datacenter)
if _restart_consul():
supervisor_server = xmlrpclib.Server('http://localhost:9001/RPC2')
hermes_init_output = supervisor_server.supervisor.startProcessGroup('hermes_init')
get_logger().info('hermes_init start: {}'.format(hermes_init_output))
set_ship_name(ship)
for key, data in local_services_data.items():
kv.kv_set(key, data)
return self.status_ok()
return self.status_error('Waiting for armada restart timed out.')
def get_default():
return kv.kv_get('dockyard/default')
def GET(self):
try:
get_args = web.input(local=False, microservice_name=None, env=None, app_id=None)
filter_local = bool(get_args.local and strtobool(str(get_args.local)))
filter_microservice_name = get_args.microservice_name
filter_env = get_args.env
filter_app_id = get_args.app_id
if filter_local:
local_microservices_ids = set(consul_query('agent/services').keys())
if filter_microservice_name:
names = list(consul_query('catalog/services').keys())
microservices_names = fnmatch.filter(names, filter_microservice_name)
else:
microservices_names = list(consul_query('catalog/services').keys())
result = []
for microservice_name in microservices_names:
if microservice_name == 'consul':
continue
query = 'health/service/{microservice_name}'.format(**locals())
instances = consul_query(query)
for instance in instances:
microservice_checks_statuses = set(check['Status'] for check in (instance['Checks'] or []))
microservice_computed_status = '-'
for possible_status in ['passing', 'warning', 'critical']:
if possible_status in microservice_checks_statuses:
microservice_computed_status = possible_status
microservice_ip = instance['Node']['Address']
microservice_port = str(instance['Service']['Port'])
microservice_id = instance['Service']['ID']
container_id = microservice_id.split(':')[0]
microservice_tags = instance['Service']['Tags'] or []
microservice_tags_dict = self.__create_dict_from_tags(microservice_tags)
matches_env = (filter_env is None) or (filter_env == microservice_tags_dict.get('env'))
matches_app_id = (filter_app_id is None) or (filter_app_id == microservice_tags_dict.get('app_id'))
if (matches_env and matches_app_id and
(not filter_local or microservice_id in local_microservices_ids)):
microservice_address = microservice_ip + ':' + microservice_port
try:
microservice_start_timestamp = kv.kv_get("start_timestamp/" + container_id)
except:
microservice_start_timestamp = None
microservice_dict = {
'name': microservice_name,
'address': microservice_address,
'microservice_id': microservice_id,
'container_id': container_id,
'status': microservice_computed_status,
'tags': microservice_tags_dict,
'start_timestamp': microservice_start_timestamp,
}
result.append(microservice_dict)
inactive_services_list = _get_inactive_services_list(filter_microservice_name, filter_env, filter_app_id)
result.extend(inactive_services_list)
return self.status_ok({'result': result})
except Exception as e:
traceback.print_exc()
return self.status_exception("Cannot get the list of services.", e)
def update_service_dict(ship, service_name, container_id, key, value):
consul_key = create_consul_services_key(ship, service_name, container_id)
service_dict = kv_get(consul_key)
service_dict[key] = value
kv_set(consul_key, service_dict)
def ship_ip_to_name(ip):
return kv.kv_get('ships/{}/name'.format(ip))
def GET(self):
try:
get_args = web.input(local=False,
microservice_name=None,
env=None,
app_id=None)
filter_local = bool(get_args.local
and strtobool(str(get_args.local)))
filter_microservice_name = get_args.microservice_name
filter_env = get_args.env
filter_app_id = get_args.app_id
services_list = _get_services_list(filter_microservice_name,
filter_env, filter_app_id,
filter_local)
if filter_local:
local_microservices_ids = set(
consul_query('agent/services').keys())
if filter_microservice_name:
names = list(consul_query('catalog/services').keys())
microservices_names = fnmatch.filter(names,
filter_microservice_name)
else:
microservices_names = list(
consul_query('catalog/services').keys())
services_list_from_catalog = {}
for microservice_name in microservices_names:
if microservice_name == 'consul':
continue
query = 'health/service/{microservice_name}'.format(**locals())
instances = consul_query(query)
for instance in instances:
microservice_checks_statuses = set(
check['Status']
for check in (instance['Checks'] or []))
microservice_computed_status = '-'
for possible_status in ['passing', 'warning', 'critical']:
if possible_status in microservice_checks_statuses:
microservice_computed_status = possible_status
microservice_ip = instance['Node']['Address']
microservice_port = str(instance['Service']['Port'])
microservice_id = instance['Service']['ID']
container_id = microservice_id.split(':')[0]
microservice_tags = instance['Service']['Tags'] or []
microservice_tags_dict = self.__create_dict_from_tags(
microservice_tags)
matches_env = (filter_env is None) or (
filter_env == microservice_tags_dict.get('env'))
matches_app_id = (filter_app_id is None) or (
filter_app_id == microservice_tags_dict.get('app_id'))
if (matches_env and matches_app_id and
(not filter_local
or microservice_id in local_microservices_ids)):
microservice_address = microservice_ip + ':' + microservice_port
try:
microservice_start_timestamp = kv.kv_get(
"start_timestamp/" + container_id)
except:
microservice_start_timestamp = None
try:
single_active_instance = kv.kv_get(
"single_active_instance/" + microservice_id)
except:
single_active_instance = False
microservice_dict = {
'name': microservice_name,
'address': microservice_address,
'microservice_id': microservice_id,
'container_id': container_id,
'status': microservice_computed_status,
'tags': microservice_tags_dict,
'start_timestamp': microservice_start_timestamp,
'single_active_instance': single_active_instance,
}
services_list_from_catalog[
microservice_id] = microservice_dict
result = services_list
result.update(services_list_from_catalog)
result = _choose_active_instances(result)
return self.status_ok({'result': result.values()})
except Exception as e:
return self.status_exception("Cannot get the list of services.", e)
def _get_services_list(filter_microservice_name, filter_env, filter_app_id,
filter_local):
if filter_local:
ship_list = [get_ship_name()]
else:
ship_list = get_ship_names()
services_dict = {}
if not ship_list:
return {}
for ship in ship_list:
containers = kv.kv_get('containers_parameters_list/{}'.format(ship))
if containers and isinstance(containers, dict):
services_dict.update(containers)
services_list = services_dict.keys()
result = {}
if not services_list:
return result
if filter_microservice_name:
services_list = fnmatch.filter(
services_list,
'ships/*/service/{}/*'.format(filter_microservice_name))
for service in services_list:
service_dict = services_dict[service]
microservice_name = service_dict['ServiceName']
microservice_status = service_dict['Status']
microservice_id = service_dict['ServiceID']
container_id = service_dict['container_id']
microservice_start_timestamp = service_dict['start_timestamp']
single_active_instance = service_dict.get('single_active_instance',
False)
not_available = 'n/a'
microservice_tags_dict = {}
try:
if service_dict['params']['microservice_env']:
microservice_tags_dict['env'] = service_dict['params'][
'microservice_env']
if service_dict['params']['microservice_app_id']:
microservice_tags_dict['app_id'] = service_dict['params'][
'microservice_app_id']
except KeyError as e:
get_logger().warning(repr(e))
matches_env = (filter_env is None) or (
filter_env == microservice_tags_dict.get('env'))
matches_app_id = (filter_app_id is None) or (
filter_app_id == microservice_tags_dict.get('app_id'))
if matches_env and matches_app_id:
microservice_dict = {
'name': microservice_name,
'status': microservice_status,
'address': not_available,
'microservice_id': microservice_id,
'container_id': container_id,
'tags': microservice_tags_dict,
'start_timestamp': microservice_start_timestamp,
'single_active_instance': single_active_instance,
}
result[microservice_id] = microservice_dict
return result
def ship_name_to_ip(name):
return kv.kv_get('ships/{}/ip'.format(name))
def get_ship_name(ship_ip=None):
if ship_ip is None:
ship_ip = get_ship_ip()
ship_name = kv.kv_get('ships/{}/name'.format(ship_ip)) or ship_ip
return ship_name
def get_initialized():
return kv.kv_get('dockyard/initialized') == '1'
def get_default():
return kv.kv_get('dockyard/default')
def get_alias(name):
key = 'dockyard/aliases/{name}'.format(**locals())
return kv.kv_get(key)
def get_initialized():
return kv.kv_get('dockyard/initialized') == '1'
def get_alias(name):
key = 'dockyard/aliases/{name}'.format(**locals())
return kv.kv_get(key)
def get_ship_name(ship_ip=None):
if ship_ip is None:
ship_ip = get_ship_ip()
ship_name = kv.kv_get('ships/{}/name'.format(ship_ip)) or ship_ip
return ship_name
def get_matched_containers(microservice_name_or_container_id_prefix):
service_names = list(consul_query('catalog/services').keys())
matched_containers_by_name = []
matched_containers_by_id = []
for service_name in service_names:
query = 'catalog/service/{service_name}'.format(**locals())
try:
instances = consul_query(query)
except Exception as e:
print_err(
'WARNING: query "{query}" failed ({exception_class}: {exception})'
.format(query=query,
exception_class=type(e).__name__,
exception=e))
instances = []
for instance in instances:
container_id = instance['ServiceID'].split(':')[0]
service_name = instance['ServiceName']
if microservice_name_or_container_id_prefix == service_name:
matched_containers_by_name.append(instance)
if container_id.startswith(microservice_name_or_container_id_prefix
) and ":" not in instance['ServiceID']:
matched_containers_by_id.append(instance)
services_list = kv.kv_list('services/')
if services_list:
for service in services_list:
service_dict = kv.kv_get(service)
container_id = service_dict['container_id']
service_name = service_dict['ServiceName']
if service_dict['Status'] == 'started':
try:
instances = consul_query(
'catalog/service/{}'.format(service_name))
if container_id in [
i['ServiceID'].split(':')[0] for i in instances
]:
continue
except Exception as e:
logging.exception(e)
if microservice_name_or_container_id_prefix == service_name:
matched_containers_by_name.append(service_dict)
if container_id.startswith(microservice_name_or_container_id_prefix) \
and ":" not in service_dict['ServiceID']:
matched_containers_by_id.append(service_dict)
matched_containers_by_name_count = len(matched_containers_by_name)
matched_containers_by_id_count = len(matched_containers_by_id)
if matched_containers_by_name_count and matched_containers_by_id_count:
raise ArmadaCommandException(
'Found matching containers with both microservice name ({matched_containers_by_name_count}) '
'and container_id ({matched_containers_by_id_count}). '
'Please provide more specific criteria.'.format(**locals()))
if matched_containers_by_id_count > 1 and len(
microservice_name_or_container_id_prefix) < 12:
raise ArmadaCommandException(
'There are too many ({matched_containers_by_id_count}) matching containers. '
'Please provide more specific container_id.'.format(**locals()))
matched_containers = matched_containers_by_name + matched_containers_by_id
matches_count = len(matched_containers)
if matches_count == 0:
raise ArmadaCommandException(
'There are no running containers with microservice: '
'{microservice_name_or_container_id_prefix}'.format(**locals()))
return matched_containers
