def _load_from_list(saved_containers, ship):
    wait_for_consul_ready()
    running_containers = _get_local_running_containers()
    containers_to_be_added = _multiset_difference(saved_containers, running_containers)
    for container_parameters in containers_to_be_added:
        get_logger().info('Added service: {}'.format(container_parameters))
        kv.save_service(ship, _generate_id(), 'crashed', params=container_parameters)
def _load_from_list(saved_containers, ship):
    wait_for_consul_ready()
    running_containers = _get_local_running_containers()
    containers_to_be_added = _multiset_difference(saved_containers, running_containers)
    index = 0
    for container_parameters in containers_to_be_added:
        kv.save_service(ship, str(index), 'crashed', params=container_parameters)
        index += 1
Example #3
0
def _load_from_list(saved_containers, ship):
    wait_for_consul_ready()
    running_containers = _get_local_running_containers()
    containers_to_be_added = _multiset_difference(saved_containers,
                                                  running_containers)
    for container_parameters in containers_to_be_added:
        get_logger().info('Added service: {}'.format(container_parameters))
        kv.save_service(ship,
                        _generate_id(),
                        'crashed',
                        params=container_parameters)
def _add_running_services_at_startup(containers_saved_in_kv, ship):
    wait_for_consul_ready()
    # wait for registering services
    sleep(10)
    all_services = consul_query('agent/services')
    del all_services['consul']
    for service_id, service_dict in all_services.items():
        if ':' in service_id:
            continue
        if service_dict['Service'] == 'armada':
            continue
        key = 'ships/{}/service/{}/{}'.format(ship, service_dict['Service'], service_id)
        if not containers_saved_in_kv or key not in containers_saved_in_kv:
            kv.save_service(ship, service_id, 'started')
Example #5
0
def deregister_not_running_services():
    services_ids = get_local_services_ids()
    containers_ids = get_running_container_ids()
    for service_id in services_ids:
        if service_id != 'consul':
            container_id = service_id.split(':')[0]
            if container_id not in containers_ids:
                name = consul_query('agent/services')[service_id]['Service']
                params = get_container_parameters(container_id)
                kv_index = 0
                if kv.kv_list('service/{}/'.format(name)):
                    kv_index = int(
                        kv.kv_list(
                            'service/{}/'.format(name))[-1].split('/')[2]) + 1
                kv.save_service(name, kv_index, 'crashed', params,
                                container_id)
                deregister_services(container_id)
Example #6
0
    def _start_container(self, long_container_id):
        docker_api = docker_client.api(timeout=30)
        docker_api.start(long_container_id)

        service_endpoints = {}
        agent_self_dict = consul_query('agent/self')
        service_ip = agent_self_dict['Config']['AdvertiseAddr']

        docker_inspect = docker_api.inspect_container(long_container_id)

        ship = get_ship_name()
        container_id = shorten_container_id(long_container_id)
        save_service(ship, container_id, status='started')

        for container_port, host_address in docker_inspect['NetworkSettings']['Ports'].items():
            service_endpoints['{0}:{1}'.format(service_ip, host_address[0]['HostPort'])] = container_port
        return service_endpoints
def _add_running_services_at_startup():
    wait_for_consul_ready()
    try:
        ship = get_ship_name()
        containers_saved_in_kv = kv.kv_list('ships/{}/service/'.format(ship))
        sleep(10)
        all_services = consul_query('agent/services')
        del all_services['consul']
        for service_id, service_dict in all_services.items():
            if ':' in service_id:
                continue
            if service_dict['Service'] == 'armada':
                continue
            key = 'ships/{}/service/{}/{}'.format(ship, service_dict['Service'], service_id)
            if not containers_saved_in_kv or key not in containers_saved_in_kv:
                kv.save_service(ship, service_id, 'started')
                get_logger().info('Added running service: {}'.format(service_id))
    except:
        get_logger().exception('Unable to add running services.')
Example #8
0
    def _start_container(self, long_container_id):
        docker_api = docker_client.api(timeout=30)
        docker_api.start(long_container_id)

        service_endpoints = {}
        agent_self_dict = consul_query('agent/self')
        service_ip = agent_self_dict['Config']['AdvertiseAddr']

        docker_inspect = docker_api.inspect_container(long_container_id)

        ship = get_ship_name()
        container_id = shorten_container_id(long_container_id)
        save_service(ship, container_id, status='started')

        for container_port, host_address in docker_inspect['NetworkSettings'][
                'Ports'].items():
            service_endpoints['{0}:{1}'.format(
                service_ip, host_address[0]['HostPort'])] = container_port
        return service_endpoints
def recover_saved_containers(saved_containers):
    wait_for_consul_ready()
    running_containers = _get_local_running_containers()
    containers_to_be_recovered = _multiset_difference(saved_containers,
                                                      running_containers)
    recovery_retry_count = 0
    while containers_to_be_recovered and recovery_retry_count < RECOVERY_RETRY_LIMIT:
        get_logger().info("Recovering containers: {}".format(
            json.dumps(containers_to_be_recovered)))
        containers_not_recovered = []
        counter_to_be_recovered = Counter(
            json.dumps(x, sort_keys=True) for x in containers_to_be_recovered)
        to_be_recovered = []
        for container_parameters in counter_to_be_recovered.elements():
            try:
                if to_be_recovered[-1][0] == container_parameters:
                    index = to_be_recovered[-1][1] + 1
                else:
                    index = 0
            except IndexError:
                index = 0
            to_be_recovered.append((container_parameters, index))
            name = json.loads(container_parameters)['microservice_name']
            kv.save_service(name, index, 'recovering',
                            json.loads(container_parameters))

        for container_parameters, index in to_be_recovered:
            container_parameters = json.loads(container_parameters)
            name = container_parameters['microservice_name']
            if not _recover_container(container_parameters):
                containers_not_recovered.append(container_parameters)
                if recovery_retry_count == (RECOVERY_RETRY_LIMIT - 1):
                    kv.save_service(name, index, 'not-recovered',
                                    json.loads(container_parameters))
            else:
                kv.kv_remove('service/{}/{}'.format(name, index))
        sleep(DELAY_BETWEEN_RECOVER_RETRY_SECONDS)
        running_containers = _get_local_running_containers()
        containers_to_be_recovered = _multiset_difference(
            containers_not_recovered, running_containers)
        recovery_retry_count += 1

    return containers_to_be_recovered
def _add_running_services_at_startup():
    wait_for_consul_ready()
    try:
        ship = get_ship_name()
        containers_saved_in_kv = kv.kv_list('ships/{}/service/'.format(ship))
        sleep(10)
        all_services = consul_query('agent/services')
        del all_services['consul']
        for service_id, service_dict in all_services.items():
            if ':' in service_id:
                continue
            if service_dict['Service'] == 'armada':
                continue
            key = 'ships/{}/service/{}/{}'.format(ship,
                                                  service_dict['Service'],
                                                  service_id)
            if not containers_saved_in_kv or key not in containers_saved_in_kv:
                kv.save_service(ship, service_id, 'started')
                get_logger().info(
                    'Added running service: {}'.format(service_id))
    except:
        get_logger().exception('Unable to add running services.')
def recover_saved_containers(saved_containers):
    wait_for_consul_ready()
    running_containers = _get_local_running_containers()
    containers_to_be_recovered = _multiset_difference(saved_containers, running_containers)
    recovery_retry_count = 0
    while containers_to_be_recovered and recovery_retry_count < RECOVERY_RETRY_LIMIT:
        get_logger().info("Recovering containers: {}".format(json.dumps(containers_to_be_recovered)))
        containers_not_recovered = []
        counter_to_be_recovered = Counter(json.dumps(x, sort_keys=True) for x in containers_to_be_recovered)
        to_be_recovered = []
        for container_parameters in counter_to_be_recovered.elements():
            try:
                if to_be_recovered[-1][0] == container_parameters:
                    index = to_be_recovered[-1][1] + 1
                else:
                    index = 0
            except IndexError:
                index = 0
            to_be_recovered.append((container_parameters, index))
            name = json.loads(container_parameters)['microservice_name']
            kv.save_service(name, index, 'recovering', json.loads(container_parameters))

        for container_parameters, index in to_be_recovered:
            container_parameters = json.loads(container_parameters)
            name = container_parameters['microservice_name']
            if not _recover_container(container_parameters):
                containers_not_recovered.append(container_parameters)
                if recovery_retry_count == (RECOVERY_RETRY_LIMIT - 1):
                    kv.save_service(name, index, 'not-recovered', json.loads(container_parameters))
            else:
                kv.kv_remove('service/{}/{}'.format(name, index))
        sleep(DELAY_BETWEEN_RECOVER_RETRY_SECONDS)
        running_containers = _get_local_running_containers()
        containers_to_be_recovered = _multiset_difference(containers_not_recovered, running_containers)
        recovery_retry_count += 1

    return containers_to_be_recovered
Example #12
0
def _mark_service_as_crashed(container_id, service_name):
    params = get_container_parameters(container_id)
    kv_index = 0
    if kv.kv_list('service/{}/'.format(service_name)):
        kv_index = int(kv.kv_list('service/{}/'.format(service_name))[-1].split('/')[2]) + 1
    kv.save_service(service_name, kv_index, 'crashed', params, container_id)