def _deregister_not_running_services():
try:
ship = get_ship_name()
except:
ship = get_ship_ip()
services = _get_local_services()
running_containers_ids = _get_running_container_ids()
for service_id in services.keys():
container_id, is_subservice = _get_container_id_with_subservice(
service_id)
if container_id in running_containers_ids:
continue
if not is_subservice:
name = services[service_id]['Service']
kv.update_container_status('crashed',
ship=ship,
name=name,
container_id=container_id)
deregister_services(container_id)
services_keys = kv.kv_list('ships/{}/service/'.format(ship)) or []
for service_key in services_keys:
container_id = service_key.split('/')[-1]
if container_id not in running_containers_ids:
kv.update_container_status('crashed', key=service_key)
deregister_services(container_id)
def POST(self):
try:
# 'startsecs=0' is to avoid restarting consul after `consul leave`.
os.system(
'sed -i \'/autorestart=true/cautorestart=false\' /etc/supervisor/conf.d/consul.conf'
)
os.system('echo startsecs=0 >> /etc/supervisor/conf.d/consul.conf')
os.system('supervisorctl update consul')
# As 'supervisorctl update' will restart Consul, we have to wait for it to be running.
deadline = time.time() + 15
while time.time() < deadline:
try:
get_current_datacenter()
break
except:
time.sleep(1)
deregister_services(gethostname())
os.system('consul leave')
finally:
post_data = json.loads(web.data() or '{}')
runtime_settings_path = '/opt/armada/runtime_settings.json'
if not post_data.get('keep-joined') and os.path.isfile(
runtime_settings_path):
with open(runtime_settings_path) as f:
runtime_settings = json.load(f)
runtime_settings['ships'] = []
runtime_settings['is_commander'] = True
with open(runtime_settings_path, 'w') as f:
json.dump(runtime_settings, f, sort_keys=True, indent=4)
return self.status_ok({'message': 'Shutdown complete.'})
def deregister_not_running_services():
services_ids = get_local_services_ids()
containers_ids = get_running_container_ids()
for service_id in services_ids:
if service_id != 'consul':
container_id = service_id.split(':')[0]
if container_id not in containers_ids:
deregister_services(container_id)
def deregister_not_running_services():
services_ids = get_local_services_ids()
containers_ids = get_running_container_ids()
for service_id in services_ids:
if service_id != 'consul':
container_id = service_id.split(':')[0]
if container_id not in containers_ids:
deregister_services(container_id)
def _deregister_not_running_services():
services = _get_local_services()
running_containers_ids = _get_running_container_ids()
for service_id in services.keys():
container_id, is_subservice = _get_container_id_with_subservice(service_id)
if container_id in running_containers_ids:
continue
if not is_subservice:
_mark_service_as_crashed(container_id, services[service_id]['Service'])
deregister_services(container_id)
def deregister_not_running_services():
services_ids = get_local_services_ids()
containers_ids = get_running_container_ids()
for service_id in services_ids:
if service_id != 'consul':
container_id = service_id.split(':')[0]
if container_id not in containers_ids:
name = consul_query('agent/services')[service_id]['Service']
params = get_container_parameters(container_id)
kv_index = 0
if kv.kv_list('service/{}/'.format(name)):
kv_index = int(
kv.kv_list(
'service/{}/'.format(name))[-1].split('/')[2]) + 1
kv.save_service(name, kv_index, 'crashed', params,
container_id)
deregister_services(container_id)
def POST(self):
# 'startsecs=0' is to avoid restarting consul after `consul leave`.
os.system('sed -i \'/autorestart=true/cautorestart=false\' /etc/supervisor/conf.d/consul.conf')
os.system('echo startsecs=0 >> /etc/supervisor/conf.d/consul.conf')
os.system('supervisorctl update consul')
# As 'supervisorctl update' will restart Consul, we have to wait for it to be running.
while True:
try:
get_current_datacenter()
break
except:
pass
deregister_services(gethostname())
os.system('consul leave')
return self.status_ok({'message': 'Shutdown complete.'})
def _stop_service(self, container_id):
ship = get_ship_name()
service_list = kv_list('ships/{}/service/'.format(ship))
try:
key = fnmatch.filter(service_list, '*/{}'.format(container_id))[0]
except (IndexError, TypeError):
key = None
if not is_container_running(container_id):
if key:
kv_remove(key)
try:
deregister_services(container_id)
except Exception as e:
get_logger().exception(e)
else:
run_command_in_container('supervisorctl stop armada_agent',
container_id)
# TODO: Compatibility with old microservice images. Should be removed in future armada version.
run_command_in_container(
'supervisorctl stop register_in_service_discovery',
container_id)
docker_api = docker_client.api()
last_exception = None
try:
deregister_services(container_id)
except Exception as e:
get_logger().exception(e)
for i in range(3):
try:
docker_api.stop(container_id)
except Exception as e:
get_logger().debug(e, exc_info=True)
last_exception = e
if not is_container_running(container_id):
if key:
kv_remove(key)
break
if is_container_running(container_id):
get_logger().error('Could not stop container: %s',
container_id)
raise last_exception
def _stop_service(self, container_id, force=False):
if force:
service_list = get_services_by_ship()
else:
service_list = get_local_services_from_kv_store()
try:
keys = fnmatch.filter(service_list, '*/{}'.format(container_id))
except (IndexError, TypeError) as e:
get_logger().exception(e)
keys = []
if not is_container_running(container_id):
for key in keys:
kv_remove(key)
try:
deregister_services(container_id)
except Exception as e:
get_logger().exception(e)
else:
run_command_in_container('supervisorctl stop armada_agent',
container_id)
trigger_hook('pre-stop', container_id)
docker_api = docker_client.api()
last_exception = None
try:
deregister_services(container_id)
except Exception as e:
get_logger().exception(e)
for i in range(3):
try:
docker_api.stop(container_id)
except Exception as e:
get_logger().debug(e, exc_info=True)
last_exception = e
if not is_container_running(container_id):
for key in keys:
kv_remove(key)
break
if is_container_running(container_id):
get_logger().error('Could not stop container: %s',
container_id)
raise last_exception
def POST(self):
# Wpisujemy 'startsecs=0', zeby ubicie consula przez 'consul leave' nie spowodowalo jego restartu.
os.system('sed -i \'/autorestart=true/cautorestart=false\' /etc/supervisor/conf.d/consul.conf')
os.system('echo startsecs=0 >> /etc/supervisor/conf.d/consul.conf')
os.system('supervisorctl update consul')
# 'supervisorctl update' zrestartuje consula. Czekamy az wystartuje na nowo.
while True:
try:
get_current_datacenter()
break
except:
pass
deregister_services(gethostname())
os.system('consul leave')
return self.status_ok({'message': 'Shutdown complete.'})
def POST(self):
# 'startsecs=0' is to avoid restarting consul after `consul leave`.
os.system(
'sed -i \'/autorestart=true/cautorestart=false\' /etc/supervisor/conf.d/consul.conf'
)
os.system('echo startsecs=0 >> /etc/supervisor/conf.d/consul.conf')
os.system('supervisorctl update consul')
# As 'supervisorctl update' will restart Consul, we have to wait for it to be running.
while True:
try:
get_current_datacenter()
break
except:
pass
deregister_services(gethostname())
os.system('consul leave')
return self.status_ok({'message': 'Shutdown complete.'})
def POST(self):
# Wpisujemy 'startsecs=0', zeby ubicie consula przez 'consul leave' nie spowodowalo jego restartu.
os.system(
'sed -i \'/autorestart=true/cautorestart=false\' /etc/supervisor/conf.d/consul.conf'
)
os.system('echo startsecs=0 >> /etc/supervisor/conf.d/consul.conf')
os.system('supervisorctl update consul')
# 'supervisorctl update' zrestartuje consula. Czekamy az wystartuje na nowo.
while True:
try:
get_current_datacenter()
break
except:
pass
deregister_services(gethostname())
os.system('consul leave')
return self.status_ok({'message': 'Shutdown complete.'})
def _stop_service(self, container_id):
ship = get_ship_name()
service_list = kv_list("ships/{}/service/".format(ship))
try:
key = fnmatch.filter(service_list, "*/{}".format(container_id))[0]
except (IndexError, TypeError):
key = None
if not is_container_running(container_id):
if key:
kv_remove(key)
try:
deregister_services(container_id)
except Exception as e:
get_logger().exception(e)
else:
run_command_in_container("supervisorctl stop armada_agent", container_id)
# TODO: Compatibility with old microservice images. Should be removed in future armada version.
run_command_in_container("supervisorctl stop register_in_service_discovery", container_id)
docker_api = docker_client.api()
last_exception = None
try:
deregister_services(container_id)
except Exception as e:
get_logger().exception(e)
for i in range(3):
try:
docker_api.stop(container_id)
except Exception as e:
get_logger().debug(e, exc_info=True)
last_exception = e
if not is_container_running(container_id):
if key:
kv_remove(key)
break
if is_container_running(container_id):
get_logger().error("Could not stop container: %s", container_id)
raise last_exception
def _deregister_not_running_services():
try:
ship = get_ship_name()
except:
ship = get_ship_ip()
services = _get_local_services()
running_containers_ids = _get_running_container_ids()
for service_id in services.keys():
container_id, is_subservice = _get_container_id_with_subservice(service_id)
if container_id in running_containers_ids:
continue
if not is_subservice:
name = services[service_id]['Service']
kv.update_container_status('crashed', ship=ship, name=name, container_id=container_id)
deregister_services(container_id)
services_keys = kv.kv_list('ships/{}/service/'.format(ship)) or []
for service_key in services_keys:
container_id = service_key.split('/')[-1]
if container_id not in running_containers_ids:
kv.update_container_status('crashed', key=service_key)
deregister_services(container_id)
def on_post(self, req, resp):
try:
# 'startsecs=0' is to avoid restarting consul after `consul leave`.
check_call(
'sed -i \'/autorestart=true/cautorestart=false\' /etc/supervisor/conf.d/consul.conf',
shell=True)
check_call(
'echo startsecs=0 >> /etc/supervisor/conf.d/consul.conf',
shell=True)
check_call(['supervisorctl', 'update', 'consul'])
# As 'supervisorctl update' will restart Consul, we have to wait for it to be running.
deadline = time.time() + 15
ok = False
while time.time() < deadline:
try:
get_current_datacenter()
ok = True
break
except Exception:
time.sleep(1)
if not ok:
get_logger().warn('Restarting consul timed out.')
deregister_services(gethostname())
check_call(['consul', 'leave'])
finally:
post_data = req.json
runtime_settings_path = '/opt/armada/runtime_settings.json'
if not post_data.get('keep-joined') and os.path.isfile(
runtime_settings_path):
with open(runtime_settings_path) as f:
runtime_settings = json.load(f)
runtime_settings['ships'] = []
runtime_settings['is_commander'] = True
with open(runtime_settings_path, 'w') as f:
json.dump(runtime_settings, f, sort_keys=True, indent=4)
return self.status_ok(resp, {'message': 'Shutdown complete.'})
def _update_running_services():
ship_ip = get_ship_ip()
try:
ship_name = get_ship_name(ship_ip)
except:
ship_name = ship_ip
services = _get_local_services_from_catalog()
running_containers_ids = _get_running_container_ids()
local_services = get_local_services_from_kv_store()
local_services_container_ids = [it.split('/')[-1] for it in local_services]
for service_id in services.keys():
container_id, is_subservice = _get_container_id_with_subservice(
service_id)
if container_id not in local_services_container_ids:
save_container(ship_name, container_id, 'started', ship_ip=ship_ip)
get_logger().info(
'Saved container in kv-store: {container_id}'.format(
container_id=container_id))
if container_id in running_containers_ids:
continue
if not is_subservice:
name = services[service_id]['Service']
update_container_status('crashed',
ship=ship_name,
service_name=name,
container_id=container_id)
get_logger().info('Set status to "crashed": {container_id}'.format(
container_id=container_id))
deregister_services(container_id)
for service_key in local_services:
container_id = service_key.split('/')[-1]
if container_id not in running_containers_ids:
update_container_status('crashed', key=service_key)
get_logger().info('Set status to "crashed": {container_id}'.format(
container_id=container_id))
deregister_services(container_id)