def _check_if_we_should_recover(saved_containers_path): try: if int(os.environ.get('DOCKER_START_TIMESTAMP')) > int(os.path.getmtime(saved_containers_path)): print_err('Docker daemon restart detected.') return True else: print_err('No need to recover.') return False except: return False
def _check_if_we_should_recover(saved_containers_path): try: if int(os.environ.get('DOCKER_START_TIMESTAMP')) > int( os.path.getmtime(saved_containers_path)): print_err('Docker daemon restart detected.') return True else: print_err('No need to recover.') return False except: return False
def _recover_saved_containers_from_path(saved_containers_path): try: saved_containers = _load_saved_containers_parameters_list(saved_containers_path) not_recovered = recover_saved_containers(saved_containers) if not_recovered: print_err('Following containers were not recovered: ', not_recovered) return False else: return True except: traceback.print_exc() print_err('Unable to recover from {saved_containers_path}.'.format(**locals())) return False
def recover_saved_containers(saved_containers): wait_for_consul_ready() running_containers = _get_local_running_containers() containers_to_be_recovered = _multiset_difference(saved_containers, running_containers) recovery_retry_count = 0 while containers_to_be_recovered and recovery_retry_count < RECOVERY_RETRY_LIMIT: print_err("Recovering containers: {}".format(json.dumps(containers_to_be_recovered))) for container_parameters in containers_to_be_recovered: _recover_container(container_parameters) sleep(DELAY_BETWEEN_RECOVER_RETRY_SECONDS) running_containers = _get_local_running_containers() containers_to_be_recovered = _multiset_difference(saved_containers, running_containers) recovery_retry_count += 1 return containers_to_be_recovered
def _recover_saved_containers_from_path(saved_containers_path): try: saved_containers = _load_saved_containers_parameters_list( saved_containers_path) not_recovered = recover_saved_containers(saved_containers) if not_recovered: print_err('Following containers were not recovered: ', not_recovered) return False else: return True except: traceback.print_exc() print_err('Unable to recover from {saved_containers_path}.'.format( **locals())) return False
def recover_saved_containers(saved_containers): wait_for_consul_ready() running_containers = _get_local_running_containers() containers_to_be_recovered = _multiset_difference(saved_containers, running_containers) recovery_retry_count = 0 while containers_to_be_recovered and recovery_retry_count < RECOVERY_RETRY_LIMIT: print_err("Recovering containers: {}".format( json.dumps(containers_to_be_recovered))) for container_parameters in containers_to_be_recovered: _recover_container(container_parameters) sleep(DELAY_BETWEEN_RECOVER_RETRY_SECONDS) running_containers = _get_local_running_containers() containers_to_be_recovered = _multiset_difference( saved_containers, running_containers) recovery_retry_count += 1 return containers_to_be_recovered
def _recover_container(container_parameters): print_err('Recovering: {}...\n'.format(json.dumps(container_parameters))) recovery_result = armada_api.post('run', container_parameters) print_err('Recovered container: {}'.format(json.dumps(recovery_result)))