def _load_from_list(saved_containers, ship): wait_for_consul_ready() running_containers = _get_local_running_containers() containers_to_be_added = _multiset_difference(saved_containers, running_containers) for container_parameters in containers_to_be_added: get_logger().info('Added service: {}'.format(container_parameters)) kv.save_service(ship, _generate_id(), 'crashed', params=container_parameters)
def _load_from_list(saved_containers, ship): wait_for_consul_ready() running_containers = _get_local_running_containers() containers_to_be_added = _multiset_difference(saved_containers, running_containers) index = 0 for container_parameters in containers_to_be_added: kv.save_service(ship, str(index), 'crashed', params=container_parameters) index += 1
def _add_running_services_at_startup(containers_saved_in_kv, ship): wait_for_consul_ready() # wait for registering services sleep(10) all_services = consul_query('agent/services') del all_services['consul'] for service_id, service_dict in all_services.items(): if ':' in service_id: continue if service_dict['Service'] == 'armada': continue key = 'ships/{}/service/{}/{}'.format(ship, service_dict['Service'], service_id) if not containers_saved_in_kv or key not in containers_saved_in_kv: kv.save_service(ship, service_id, 'started')
def deregister_not_running_services(): services_ids = get_local_services_ids() containers_ids = get_running_container_ids() for service_id in services_ids: if service_id != 'consul': container_id = service_id.split(':')[0] if container_id not in containers_ids: name = consul_query('agent/services')[service_id]['Service'] params = get_container_parameters(container_id) kv_index = 0 if kv.kv_list('service/{}/'.format(name)): kv_index = int( kv.kv_list( 'service/{}/'.format(name))[-1].split('/')[2]) + 1 kv.save_service(name, kv_index, 'crashed', params, container_id) deregister_services(container_id)
def _start_container(self, long_container_id): docker_api = docker_client.api(timeout=30) docker_api.start(long_container_id) service_endpoints = {} agent_self_dict = consul_query('agent/self') service_ip = agent_self_dict['Config']['AdvertiseAddr'] docker_inspect = docker_api.inspect_container(long_container_id) ship = get_ship_name() container_id = shorten_container_id(long_container_id) save_service(ship, container_id, status='started') for container_port, host_address in docker_inspect['NetworkSettings']['Ports'].items(): service_endpoints['{0}:{1}'.format(service_ip, host_address[0]['HostPort'])] = container_port return service_endpoints
def _add_running_services_at_startup(): wait_for_consul_ready() try: ship = get_ship_name() containers_saved_in_kv = kv.kv_list('ships/{}/service/'.format(ship)) sleep(10) all_services = consul_query('agent/services') del all_services['consul'] for service_id, service_dict in all_services.items(): if ':' in service_id: continue if service_dict['Service'] == 'armada': continue key = 'ships/{}/service/{}/{}'.format(ship, service_dict['Service'], service_id) if not containers_saved_in_kv or key not in containers_saved_in_kv: kv.save_service(ship, service_id, 'started') get_logger().info('Added running service: {}'.format(service_id)) except: get_logger().exception('Unable to add running services.')
def _start_container(self, long_container_id): docker_api = docker_client.api(timeout=30) docker_api.start(long_container_id) service_endpoints = {} agent_self_dict = consul_query('agent/self') service_ip = agent_self_dict['Config']['AdvertiseAddr'] docker_inspect = docker_api.inspect_container(long_container_id) ship = get_ship_name() container_id = shorten_container_id(long_container_id) save_service(ship, container_id, status='started') for container_port, host_address in docker_inspect['NetworkSettings'][ 'Ports'].items(): service_endpoints['{0}:{1}'.format( service_ip, host_address[0]['HostPort'])] = container_port return service_endpoints
def recover_saved_containers(saved_containers): wait_for_consul_ready() running_containers = _get_local_running_containers() containers_to_be_recovered = _multiset_difference(saved_containers, running_containers) recovery_retry_count = 0 while containers_to_be_recovered and recovery_retry_count < RECOVERY_RETRY_LIMIT: get_logger().info("Recovering containers: {}".format( json.dumps(containers_to_be_recovered))) containers_not_recovered = [] counter_to_be_recovered = Counter( json.dumps(x, sort_keys=True) for x in containers_to_be_recovered) to_be_recovered = [] for container_parameters in counter_to_be_recovered.elements(): try: if to_be_recovered[-1][0] == container_parameters: index = to_be_recovered[-1][1] + 1 else: index = 0 except IndexError: index = 0 to_be_recovered.append((container_parameters, index)) name = json.loads(container_parameters)['microservice_name'] kv.save_service(name, index, 'recovering', json.loads(container_parameters)) for container_parameters, index in to_be_recovered: container_parameters = json.loads(container_parameters) name = container_parameters['microservice_name'] if not _recover_container(container_parameters): containers_not_recovered.append(container_parameters) if recovery_retry_count == (RECOVERY_RETRY_LIMIT - 1): kv.save_service(name, index, 'not-recovered', json.loads(container_parameters)) else: kv.kv_remove('service/{}/{}'.format(name, index)) sleep(DELAY_BETWEEN_RECOVER_RETRY_SECONDS) running_containers = _get_local_running_containers() containers_to_be_recovered = _multiset_difference( containers_not_recovered, running_containers) recovery_retry_count += 1 return containers_to_be_recovered
def _add_running_services_at_startup(): wait_for_consul_ready() try: ship = get_ship_name() containers_saved_in_kv = kv.kv_list('ships/{}/service/'.format(ship)) sleep(10) all_services = consul_query('agent/services') del all_services['consul'] for service_id, service_dict in all_services.items(): if ':' in service_id: continue if service_dict['Service'] == 'armada': continue key = 'ships/{}/service/{}/{}'.format(ship, service_dict['Service'], service_id) if not containers_saved_in_kv or key not in containers_saved_in_kv: kv.save_service(ship, service_id, 'started') get_logger().info( 'Added running service: {}'.format(service_id)) except: get_logger().exception('Unable to add running services.')
def recover_saved_containers(saved_containers): wait_for_consul_ready() running_containers = _get_local_running_containers() containers_to_be_recovered = _multiset_difference(saved_containers, running_containers) recovery_retry_count = 0 while containers_to_be_recovered and recovery_retry_count < RECOVERY_RETRY_LIMIT: get_logger().info("Recovering containers: {}".format(json.dumps(containers_to_be_recovered))) containers_not_recovered = [] counter_to_be_recovered = Counter(json.dumps(x, sort_keys=True) for x in containers_to_be_recovered) to_be_recovered = [] for container_parameters in counter_to_be_recovered.elements(): try: if to_be_recovered[-1][0] == container_parameters: index = to_be_recovered[-1][1] + 1 else: index = 0 except IndexError: index = 0 to_be_recovered.append((container_parameters, index)) name = json.loads(container_parameters)['microservice_name'] kv.save_service(name, index, 'recovering', json.loads(container_parameters)) for container_parameters, index in to_be_recovered: container_parameters = json.loads(container_parameters) name = container_parameters['microservice_name'] if not _recover_container(container_parameters): containers_not_recovered.append(container_parameters) if recovery_retry_count == (RECOVERY_RETRY_LIMIT - 1): kv.save_service(name, index, 'not-recovered', json.loads(container_parameters)) else: kv.kv_remove('service/{}/{}'.format(name, index)) sleep(DELAY_BETWEEN_RECOVER_RETRY_SECONDS) running_containers = _get_local_running_containers() containers_to_be_recovered = _multiset_difference(containers_not_recovered, running_containers) recovery_retry_count += 1 return containers_to_be_recovered
def _mark_service_as_crashed(container_id, service_name): params = get_container_parameters(container_id) kv_index = 0 if kv.kv_list('service/{}/'.format(service_name)): kv_index = int(kv.kv_list('service/{}/'.format(service_name))[-1].split('/')[2]) + 1 kv.save_service(service_name, kv_index, 'crashed', params, container_id)