def recover_containers_from_kv_store():
services_to_be_recovered = _get_crashed_services()
for service in services_to_be_recovered:
kv.update_service_status('recovering', key=service)
recovery_retry_count = 0
while services_to_be_recovered and recovery_retry_count < RECOVERY_RETRY_LIMIT:
get_logger().info("Recovering containers: %s", json.dumps(services_to_be_recovered))
services_not_recovered = []
for service in services_to_be_recovered:
service_parameters = kv.kv_get(service)['params']
if not _recover_container(service_parameters):
services_not_recovered.append(service)
else:
kv.kv_remove(service)
sleep(DELAY_BETWEEN_RECOVER_RETRY_SECONDS)
services_to_be_recovered = services_not_recovered
recovery_retry_count += 1
for service in services_to_be_recovered:
kv.update_service_status('not-recovered', key=service)
return services_to_be_recovered
def recover_containers_from_kv_store():
services_to_be_recovered = _get_crashed_services()
for service in services_to_be_recovered:
kv.update_service_status('recovering', key=service)
recovery_retry_count = 0
while services_to_be_recovered and recovery_retry_count < RECOVERY_RETRY_LIMIT:
get_logger().info("Recovering containers: {}".format(
json.dumps(services_to_be_recovered)))
services_not_recovered = []
for service in services_to_be_recovered:
service_parameters = kv.kv_get(service)['params']
if not _recover_container(service_parameters):
services_not_recovered.append(service)
if recovery_retry_count == (RECOVERY_RETRY_LIMIT - 1):
kv.update_service_status('not-recovered', key=service)
else:
kv.kv_remove(service)
sleep(DELAY_BETWEEN_RECOVER_RETRY_SECONDS)
services_to_be_recovered = _get_crashed_services()
recovery_retry_count += 1
return services_to_be_recovered
def _deregister_not_running_services():
try:
ship = get_ship_name()
except:
ship = get_ship_ip()
services = _get_local_services()
running_containers_ids = _get_running_container_ids()
for service_id in services.keys():
container_id, is_subservice = _get_container_id_with_subservice(service_id)
if container_id in running_containers_ids:
continue
if not is_subservice:
name = services[service_id]['Service']
kv.update_service_status('crashed', ship=ship, name=name, container_id=container_id)
deregister_services(container_id)
services_ids = kv.kv_list('ships/{}/service/'.format(ship)) or []
for service_id in services_ids:
container_id = service_id.split('/')[-1]
if container_id not in running_containers_ids:
kv.update_service_status('crashed', key=service_id)
deregister_services(container_id)
def _deregister_not_running_services():
try:
ship = get_ship_name()
except:
ship = get_ship_ip()
services = _get_local_services()
running_containers_ids = _get_running_container_ids()
for service_id in services.keys():
container_id, is_subservice = _get_container_id_with_subservice(service_id)
if container_id in running_containers_ids:
continue
if not is_subservice:
name = services[service_id]['Service']
kv.update_service_status('crashed', ship=ship, name=name, container_id=container_id)
deregister_services(container_id)
services_ids = kv.kv_list('ships/{}/service/'.format(ship)) or []
containers_ids = _get_running_container_ids()
for service_id in services_ids:
container_id = service_id.split('/')[-1]
if container_id not in containers_ids:
kv.update_service_status('crashed', key=service_id)
deregister_services(container_id)
