def _deregister_not_running_services(): try: ship = get_ship_name() except: ship = get_ship_ip() services = _get_local_services() running_containers_ids = _get_running_container_ids() for service_id in services.keys(): container_id, is_subservice = _get_container_id_with_subservice( service_id) if container_id in running_containers_ids: continue if not is_subservice: name = services[service_id]['Service'] kv.update_container_status('crashed', ship=ship, name=name, container_id=container_id) deregister_services(container_id) services_keys = kv.kv_list('ships/{}/service/'.format(ship)) or [] for service_key in services_keys: container_id = service_key.split('/')[-1] if container_id not in running_containers_ids: kv.update_container_status('crashed', key=service_key) deregister_services(container_id)
def POST(self): try: # 'startsecs=0' is to avoid restarting consul after `consul leave`. os.system( 'sed -i \'/autorestart=true/cautorestart=false\' /etc/supervisor/conf.d/consul.conf' ) os.system('echo startsecs=0 >> /etc/supervisor/conf.d/consul.conf') os.system('supervisorctl update consul') # As 'supervisorctl update' will restart Consul, we have to wait for it to be running. deadline = time.time() + 15 while time.time() < deadline: try: get_current_datacenter() break except: time.sleep(1) deregister_services(gethostname()) os.system('consul leave') finally: post_data = json.loads(web.data() or '{}') runtime_settings_path = '/opt/armada/runtime_settings.json' if not post_data.get('keep-joined') and os.path.isfile( runtime_settings_path): with open(runtime_settings_path) as f: runtime_settings = json.load(f) runtime_settings['ships'] = [] runtime_settings['is_commander'] = True with open(runtime_settings_path, 'w') as f: json.dump(runtime_settings, f, sort_keys=True, indent=4) return self.status_ok({'message': 'Shutdown complete.'})
def deregister_not_running_services(): services_ids = get_local_services_ids() containers_ids = get_running_container_ids() for service_id in services_ids: if service_id != 'consul': container_id = service_id.split(':')[0] if container_id not in containers_ids: deregister_services(container_id)
def _deregister_not_running_services(): services = _get_local_services() running_containers_ids = _get_running_container_ids() for service_id in services.keys(): container_id, is_subservice = _get_container_id_with_subservice(service_id) if container_id in running_containers_ids: continue if not is_subservice: _mark_service_as_crashed(container_id, services[service_id]['Service']) deregister_services(container_id)
def deregister_not_running_services(): services_ids = get_local_services_ids() containers_ids = get_running_container_ids() for service_id in services_ids: if service_id != 'consul': container_id = service_id.split(':')[0] if container_id not in containers_ids: name = consul_query('agent/services')[service_id]['Service'] params = get_container_parameters(container_id) kv_index = 0 if kv.kv_list('service/{}/'.format(name)): kv_index = int( kv.kv_list( 'service/{}/'.format(name))[-1].split('/')[2]) + 1 kv.save_service(name, kv_index, 'crashed', params, container_id) deregister_services(container_id)
def POST(self): # 'startsecs=0' is to avoid restarting consul after `consul leave`. os.system('sed -i \'/autorestart=true/cautorestart=false\' /etc/supervisor/conf.d/consul.conf') os.system('echo startsecs=0 >> /etc/supervisor/conf.d/consul.conf') os.system('supervisorctl update consul') # As 'supervisorctl update' will restart Consul, we have to wait for it to be running. while True: try: get_current_datacenter() break except: pass deregister_services(gethostname()) os.system('consul leave') return self.status_ok({'message': 'Shutdown complete.'})
def _stop_service(self, container_id): ship = get_ship_name() service_list = kv_list('ships/{}/service/'.format(ship)) try: key = fnmatch.filter(service_list, '*/{}'.format(container_id))[0] except (IndexError, TypeError): key = None if not is_container_running(container_id): if key: kv_remove(key) try: deregister_services(container_id) except Exception as e: get_logger().exception(e) else: run_command_in_container('supervisorctl stop armada_agent', container_id) # TODO: Compatibility with old microservice images. Should be removed in future armada version. run_command_in_container( 'supervisorctl stop register_in_service_discovery', container_id) docker_api = docker_client.api() last_exception = None try: deregister_services(container_id) except Exception as e: get_logger().exception(e) for i in range(3): try: docker_api.stop(container_id) except Exception as e: get_logger().debug(e, exc_info=True) last_exception = e if not is_container_running(container_id): if key: kv_remove(key) break if is_container_running(container_id): get_logger().error('Could not stop container: %s', container_id) raise last_exception
def _stop_service(self, container_id, force=False): if force: service_list = get_services_by_ship() else: service_list = get_local_services_from_kv_store() try: keys = fnmatch.filter(service_list, '*/{}'.format(container_id)) except (IndexError, TypeError) as e: get_logger().exception(e) keys = [] if not is_container_running(container_id): for key in keys: kv_remove(key) try: deregister_services(container_id) except Exception as e: get_logger().exception(e) else: run_command_in_container('supervisorctl stop armada_agent', container_id) trigger_hook('pre-stop', container_id) docker_api = docker_client.api() last_exception = None try: deregister_services(container_id) except Exception as e: get_logger().exception(e) for i in range(3): try: docker_api.stop(container_id) except Exception as e: get_logger().debug(e, exc_info=True) last_exception = e if not is_container_running(container_id): for key in keys: kv_remove(key) break if is_container_running(container_id): get_logger().error('Could not stop container: %s', container_id) raise last_exception
def POST(self): # Wpisujemy 'startsecs=0', zeby ubicie consula przez 'consul leave' nie spowodowalo jego restartu. os.system('sed -i \'/autorestart=true/cautorestart=false\' /etc/supervisor/conf.d/consul.conf') os.system('echo startsecs=0 >> /etc/supervisor/conf.d/consul.conf') os.system('supervisorctl update consul') # 'supervisorctl update' zrestartuje consula. Czekamy az wystartuje na nowo. while True: try: get_current_datacenter() break except: pass deregister_services(gethostname()) os.system('consul leave') return self.status_ok({'message': 'Shutdown complete.'})
def POST(self): # 'startsecs=0' is to avoid restarting consul after `consul leave`. os.system( 'sed -i \'/autorestart=true/cautorestart=false\' /etc/supervisor/conf.d/consul.conf' ) os.system('echo startsecs=0 >> /etc/supervisor/conf.d/consul.conf') os.system('supervisorctl update consul') # As 'supervisorctl update' will restart Consul, we have to wait for it to be running. while True: try: get_current_datacenter() break except: pass deregister_services(gethostname()) os.system('consul leave') return self.status_ok({'message': 'Shutdown complete.'})
def POST(self): # Wpisujemy 'startsecs=0', zeby ubicie consula przez 'consul leave' nie spowodowalo jego restartu. os.system( 'sed -i \'/autorestart=true/cautorestart=false\' /etc/supervisor/conf.d/consul.conf' ) os.system('echo startsecs=0 >> /etc/supervisor/conf.d/consul.conf') os.system('supervisorctl update consul') # 'supervisorctl update' zrestartuje consula. Czekamy az wystartuje na nowo. while True: try: get_current_datacenter() break except: pass deregister_services(gethostname()) os.system('consul leave') return self.status_ok({'message': 'Shutdown complete.'})
def _stop_service(self, container_id): ship = get_ship_name() service_list = kv_list("ships/{}/service/".format(ship)) try: key = fnmatch.filter(service_list, "*/{}".format(container_id))[0] except (IndexError, TypeError): key = None if not is_container_running(container_id): if key: kv_remove(key) try: deregister_services(container_id) except Exception as e: get_logger().exception(e) else: run_command_in_container("supervisorctl stop armada_agent", container_id) # TODO: Compatibility with old microservice images. Should be removed in future armada version. run_command_in_container("supervisorctl stop register_in_service_discovery", container_id) docker_api = docker_client.api() last_exception = None try: deregister_services(container_id) except Exception as e: get_logger().exception(e) for i in range(3): try: docker_api.stop(container_id) except Exception as e: get_logger().debug(e, exc_info=True) last_exception = e if not is_container_running(container_id): if key: kv_remove(key) break if is_container_running(container_id): get_logger().error("Could not stop container: %s", container_id) raise last_exception
def _deregister_not_running_services(): try: ship = get_ship_name() except: ship = get_ship_ip() services = _get_local_services() running_containers_ids = _get_running_container_ids() for service_id in services.keys(): container_id, is_subservice = _get_container_id_with_subservice(service_id) if container_id in running_containers_ids: continue if not is_subservice: name = services[service_id]['Service'] kv.update_container_status('crashed', ship=ship, name=name, container_id=container_id) deregister_services(container_id) services_keys = kv.kv_list('ships/{}/service/'.format(ship)) or [] for service_key in services_keys: container_id = service_key.split('/')[-1] if container_id not in running_containers_ids: kv.update_container_status('crashed', key=service_key) deregister_services(container_id)
def on_post(self, req, resp): try: # 'startsecs=0' is to avoid restarting consul after `consul leave`. check_call( 'sed -i \'/autorestart=true/cautorestart=false\' /etc/supervisor/conf.d/consul.conf', shell=True) check_call( 'echo startsecs=0 >> /etc/supervisor/conf.d/consul.conf', shell=True) check_call(['supervisorctl', 'update', 'consul']) # As 'supervisorctl update' will restart Consul, we have to wait for it to be running. deadline = time.time() + 15 ok = False while time.time() < deadline: try: get_current_datacenter() ok = True break except Exception: time.sleep(1) if not ok: get_logger().warn('Restarting consul timed out.') deregister_services(gethostname()) check_call(['consul', 'leave']) finally: post_data = req.json runtime_settings_path = '/opt/armada/runtime_settings.json' if not post_data.get('keep-joined') and os.path.isfile( runtime_settings_path): with open(runtime_settings_path) as f: runtime_settings = json.load(f) runtime_settings['ships'] = [] runtime_settings['is_commander'] = True with open(runtime_settings_path, 'w') as f: json.dump(runtime_settings, f, sort_keys=True, indent=4) return self.status_ok(resp, {'message': 'Shutdown complete.'})
def _update_running_services(): ship_ip = get_ship_ip() try: ship_name = get_ship_name(ship_ip) except: ship_name = ship_ip services = _get_local_services_from_catalog() running_containers_ids = _get_running_container_ids() local_services = get_local_services_from_kv_store() local_services_container_ids = [it.split('/')[-1] for it in local_services] for service_id in services.keys(): container_id, is_subservice = _get_container_id_with_subservice( service_id) if container_id not in local_services_container_ids: save_container(ship_name, container_id, 'started', ship_ip=ship_ip) get_logger().info( 'Saved container in kv-store: {container_id}'.format( container_id=container_id)) if container_id in running_containers_ids: continue if not is_subservice: name = services[service_id]['Service'] update_container_status('crashed', ship=ship_name, service_name=name, container_id=container_id) get_logger().info('Set status to "crashed": {container_id}'.format( container_id=container_id)) deregister_services(container_id) for service_key in local_services: container_id = service_key.split('/')[-1] if container_id not in running_containers_ids: update_container_status('crashed', key=service_key) get_logger().info('Set status to "crashed": {container_id}'.format( container_id=container_id)) deregister_services(container_id)