def _get_local_running_containers():
result = []
for container in get_local_services_from_kv_store():
container_parameters = kv.kv_get(container)['params']
if container_parameters:
result.append(container_parameters)
return result
def _add_running_services_at_startup():
wait_for_consul_ready()
try:
ship_ip, ship_name = get_ship_ip_and_name()
containers_saved_in_kv = get_local_services_from_kv_store()
sleep(10)
all_services = consul_query('agent/services')
if 'consul' in all_services:
del all_services['consul']
for service_id, service_dict in six.iteritems(all_services):
if ':' in service_id:
continue
if service_dict['Service'] == 'armada':
continue
key = create_consul_services_key(ship_name,
service_dict['Service'],
service_id)
if not containers_saved_in_kv or key not in containers_saved_in_kv:
save_container(ship_name,
service_id,
'started',
ship_ip=ship_ip)
get_logger().info(
'Added running service: {}'.format(service_id))
except Exception:
get_logger().exception('Unable to add running services.')
def _get_crashed_services():
services_list = get_local_services_from_kv_store()
crashed_services = []
for service in services_list:
service_dict = kv.kv_get(service)
microservice_status = service_dict['Status']
if microservice_status in ['crashed', 'not-recovered']:
crashed_services.append(service)
return crashed_services
def on_post(self, req, resp):
consul_host, error = self.get_post_parameter(req, 'host')
if error:
return self.status_error(resp, error)
ship = get_ship_name()
local_services_data = {
key: kv.kv_get(key)
for key in get_local_services_from_kv_store()
}
armada_size = _get_armada_size()
if armada_size > 1:
return self.status_error(
resp,
'Currently only single ship armadas can join the others. '
'Your armada has size: {0}.'.format(armada_size))
try:
agent_self_dict = consul_query(
'agent/self', consul_address='{0}:8500'.format(consul_host))
datacenter = agent_self_dict['Config']['Datacenter']
except Exception as e:
get_logger().exception(e)
return self.status_error(
resp, 'Could not read remote host datacenter address.')
current_consul_mode = _get_current_consul_mode()
if current_consul_mode == consul_config.ConsulMode.BOOTSTRAP:
override_runtime_settings(
consul_mode=consul_config.ConsulMode.CLIENT,
ship_ips=[consul_host],
datacenter=datacenter)
else:
override_runtime_settings(ship_ips=[consul_host] +
get_other_ship_ips(),
datacenter=datacenter)
if _restart_consul():
supervisor_server = xmlrpc.client.Server(
'http://localhost:9001/RPC2')
hermes_init_output = supervisor_server.supervisor.startProcessGroup(
'hermes_init')
get_logger().info('hermes_init start: %s', hermes_init_output)
set_ship_name(ship)
for key, data in six.iteritems(local_services_data):
kv.kv_set(key, data)
return self.status_ok(resp)
return self.status_error(resp, 'Waiting for armada restart timed out.')
def _stop_service(self, container_id, force=False):
if force:
service_list = get_services_by_ship()
else:
service_list = get_local_services_from_kv_store()
try:
keys = fnmatch.filter(service_list, '*/{}'.format(container_id))
except (IndexError, TypeError) as e:
get_logger().exception(e)
keys = []
if not is_container_running(container_id):
for key in keys:
kv_remove(key)
try:
deregister_services(container_id)
except Exception as e:
get_logger().exception(e)
else:
run_command_in_container('supervisorctl stop armada_agent',
container_id)
trigger_hook('pre-stop', container_id)
docker_api = docker_client.api()
last_exception = None
try:
deregister_services(container_id)
except Exception as e:
get_logger().exception(e)
for i in range(3):
try:
docker_api.stop(container_id)
except Exception as e:
get_logger().debug(e, exc_info=True)
last_exception = e
if not is_container_running(container_id):
for key in keys:
kv_remove(key)
break
if is_container_running(container_id):
get_logger().error('Could not stop container: %s',
container_id)
raise last_exception
def _update_running_services():
ship_ip = get_ship_ip()
try:
ship_name = get_ship_name(ship_ip)
except:
ship_name = ship_ip
services = _get_local_services_from_catalog()
running_containers_ids = _get_running_container_ids()
local_services = get_local_services_from_kv_store()
local_services_container_ids = [it.split('/')[-1] for it in local_services]
for service_id in services.keys():
container_id, is_subservice = _get_container_id_with_subservice(
service_id)
if container_id not in local_services_container_ids:
save_container(ship_name, container_id, 'started', ship_ip=ship_ip)
get_logger().info(
'Saved container in kv-store: {container_id}'.format(
container_id=container_id))
if container_id in running_containers_ids:
continue
if not is_subservice:
name = services[service_id]['Service']
update_container_status('crashed',
ship=ship_name,
service_name=name,
container_id=container_id)
get_logger().info('Set status to "crashed": {container_id}'.format(
container_id=container_id))
deregister_services(container_id)
for service_key in local_services:
container_id = service_key.split('/')[-1]
if container_id not in running_containers_ids:
update_container_status('crashed', key=service_key)
get_logger().info('Set status to "crashed": {container_id}'.format(
container_id=container_id))
deregister_services(container_id)
