def _get_local_running_containers(): result = [] for container in get_local_services_from_kv_store(): container_parameters = kv.kv_get(container)['params'] if container_parameters: result.append(container_parameters) return result
def _add_running_services_at_startup(): wait_for_consul_ready() try: ship_ip, ship_name = get_ship_ip_and_name() containers_saved_in_kv = get_local_services_from_kv_store() sleep(10) all_services = consul_query('agent/services') if 'consul' in all_services: del all_services['consul'] for service_id, service_dict in six.iteritems(all_services): if ':' in service_id: continue if service_dict['Service'] == 'armada': continue key = create_consul_services_key(ship_name, service_dict['Service'], service_id) if not containers_saved_in_kv or key not in containers_saved_in_kv: save_container(ship_name, service_id, 'started', ship_ip=ship_ip) get_logger().info( 'Added running service: {}'.format(service_id)) except Exception: get_logger().exception('Unable to add running services.')
def _get_crashed_services(): services_list = get_local_services_from_kv_store() crashed_services = [] for service in services_list: service_dict = kv.kv_get(service) microservice_status = service_dict['Status'] if microservice_status in ['crashed', 'not-recovered']: crashed_services.append(service) return crashed_services
def on_post(self, req, resp): consul_host, error = self.get_post_parameter(req, 'host') if error: return self.status_error(resp, error) ship = get_ship_name() local_services_data = { key: kv.kv_get(key) for key in get_local_services_from_kv_store() } armada_size = _get_armada_size() if armada_size > 1: return self.status_error( resp, 'Currently only single ship armadas can join the others. ' 'Your armada has size: {0}.'.format(armada_size)) try: agent_self_dict = consul_query( 'agent/self', consul_address='{0}:8500'.format(consul_host)) datacenter = agent_self_dict['Config']['Datacenter'] except Exception as e: get_logger().exception(e) return self.status_error( resp, 'Could not read remote host datacenter address.') current_consul_mode = _get_current_consul_mode() if current_consul_mode == consul_config.ConsulMode.BOOTSTRAP: override_runtime_settings( consul_mode=consul_config.ConsulMode.CLIENT, ship_ips=[consul_host], datacenter=datacenter) else: override_runtime_settings(ship_ips=[consul_host] + get_other_ship_ips(), datacenter=datacenter) if _restart_consul(): supervisor_server = xmlrpc.client.Server( 'http://localhost:9001/RPC2') hermes_init_output = supervisor_server.supervisor.startProcessGroup( 'hermes_init') get_logger().info('hermes_init start: %s', hermes_init_output) set_ship_name(ship) for key, data in six.iteritems(local_services_data): kv.kv_set(key, data) return self.status_ok(resp) return self.status_error(resp, 'Waiting for armada restart timed out.')
def _stop_service(self, container_id, force=False): if force: service_list = get_services_by_ship() else: service_list = get_local_services_from_kv_store() try: keys = fnmatch.filter(service_list, '*/{}'.format(container_id)) except (IndexError, TypeError) as e: get_logger().exception(e) keys = [] if not is_container_running(container_id): for key in keys: kv_remove(key) try: deregister_services(container_id) except Exception as e: get_logger().exception(e) else: run_command_in_container('supervisorctl stop armada_agent', container_id) trigger_hook('pre-stop', container_id) docker_api = docker_client.api() last_exception = None try: deregister_services(container_id) except Exception as e: get_logger().exception(e) for i in range(3): try: docker_api.stop(container_id) except Exception as e: get_logger().debug(e, exc_info=True) last_exception = e if not is_container_running(container_id): for key in keys: kv_remove(key) break if is_container_running(container_id): get_logger().error('Could not stop container: %s', container_id) raise last_exception
def _update_running_services(): ship_ip = get_ship_ip() try: ship_name = get_ship_name(ship_ip) except: ship_name = ship_ip services = _get_local_services_from_catalog() running_containers_ids = _get_running_container_ids() local_services = get_local_services_from_kv_store() local_services_container_ids = [it.split('/')[-1] for it in local_services] for service_id in services.keys(): container_id, is_subservice = _get_container_id_with_subservice( service_id) if container_id not in local_services_container_ids: save_container(ship_name, container_id, 'started', ship_ip=ship_ip) get_logger().info( 'Saved container in kv-store: {container_id}'.format( container_id=container_id)) if container_id in running_containers_ids: continue if not is_subservice: name = services[service_id]['Service'] update_container_status('crashed', ship=ship_name, service_name=name, container_id=container_id) get_logger().info('Set status to "crashed": {container_id}'.format( container_id=container_id)) deregister_services(container_id) for service_key in local_services: container_id = service_key.split('/')[-1] if container_id not in running_containers_ids: update_container_status('crashed', key=service_key) get_logger().info('Set status to "crashed": {container_id}'.format( container_id=container_id)) deregister_services(container_id)