def _check_if_we_should_recover(saved_containers_path):
try:
if int(os.environ.get('DOCKER_START_TIMESTAMP')) > int(os.path.getmtime(saved_containers_path)):
print_err('Docker daemon restart detected.')
return True
else:
print_err('No need to recover.')
return False
except:
return False
def _check_if_we_should_recover(saved_containers_path):
try:
if int(os.environ.get('DOCKER_START_TIMESTAMP')) > int(
os.path.getmtime(saved_containers_path)):
print_err('Docker daemon restart detected.')
return True
else:
print_err('No need to recover.')
return False
except:
return False
def _recover_saved_containers_from_path(saved_containers_path):
try:
saved_containers = _load_saved_containers_parameters_list(saved_containers_path)
not_recovered = recover_saved_containers(saved_containers)
if not_recovered:
print_err('Following containers were not recovered: ', not_recovered)
return False
else:
return True
except:
traceback.print_exc()
print_err('Unable to recover from {saved_containers_path}.'.format(**locals()))
return False
def recover_saved_containers(saved_containers):
wait_for_consul_ready()
running_containers = _get_local_running_containers()
containers_to_be_recovered = _multiset_difference(saved_containers, running_containers)
recovery_retry_count = 0
while containers_to_be_recovered and recovery_retry_count < RECOVERY_RETRY_LIMIT:
print_err("Recovering containers: {}".format(json.dumps(containers_to_be_recovered)))
for container_parameters in containers_to_be_recovered:
_recover_container(container_parameters)
sleep(DELAY_BETWEEN_RECOVER_RETRY_SECONDS)
running_containers = _get_local_running_containers()
containers_to_be_recovered = _multiset_difference(saved_containers, running_containers)
recovery_retry_count += 1
return containers_to_be_recovered
def _recover_saved_containers_from_path(saved_containers_path):
try:
saved_containers = _load_saved_containers_parameters_list(
saved_containers_path)
not_recovered = recover_saved_containers(saved_containers)
if not_recovered:
print_err('Following containers were not recovered: ',
not_recovered)
return False
else:
return True
except:
traceback.print_exc()
print_err('Unable to recover from {saved_containers_path}.'.format(
**locals()))
return False
def recover_saved_containers(saved_containers):
wait_for_consul_ready()
running_containers = _get_local_running_containers()
containers_to_be_recovered = _multiset_difference(saved_containers,
running_containers)
recovery_retry_count = 0
while containers_to_be_recovered and recovery_retry_count < RECOVERY_RETRY_LIMIT:
print_err("Recovering containers: {}".format(
json.dumps(containers_to_be_recovered)))
for container_parameters in containers_to_be_recovered:
_recover_container(container_parameters)
sleep(DELAY_BETWEEN_RECOVER_RETRY_SECONDS)
running_containers = _get_local_running_containers()
containers_to_be_recovered = _multiset_difference(
saved_containers, running_containers)
recovery_retry_count += 1
return containers_to_be_recovered
def _recover_container(container_parameters):
print_err('Recovering: {}...\n'.format(json.dumps(container_parameters)))
recovery_result = armada_api.post('run', container_parameters)
print_err('Recovered container: {}'.format(json.dumps(recovery_result)))
def _recover_container(container_parameters):
print_err('Recovering: {}...\n'.format(json.dumps(container_parameters)))
recovery_result = armada_api.post('run', container_parameters)
print_err('Recovered container: {}'.format(json.dumps(recovery_result)))
