Example #1
0
def _deregister_not_running_services():
    try:
        ship = get_ship_name()
    except:
        ship = get_ship_ip()
    services = _get_local_services()
    running_containers_ids = _get_running_container_ids()
    for service_id in services.keys():
        container_id, is_subservice = _get_container_id_with_subservice(
            service_id)
        if container_id in running_containers_ids:
            continue
        if not is_subservice:
            name = services[service_id]['Service']
            kv.update_container_status('crashed',
                                       ship=ship,
                                       name=name,
                                       container_id=container_id)
        deregister_services(container_id)

    services_keys = kv.kv_list('ships/{}/service/'.format(ship)) or []
    for service_key in services_keys:
        container_id = service_key.split('/')[-1]
        if container_id not in running_containers_ids:
            kv.update_container_status('crashed', key=service_key)
            deregister_services(container_id)
Example #2
0
    def POST(self):
        try:
            # 'startsecs=0' is to avoid restarting consul after `consul leave`.
            os.system(
                'sed -i \'/autorestart=true/cautorestart=false\' /etc/supervisor/conf.d/consul.conf'
            )
            os.system('echo startsecs=0 >> /etc/supervisor/conf.d/consul.conf')

            os.system('supervisorctl update consul')

            # As 'supervisorctl update' will restart Consul, we have to wait for it to be running.
            deadline = time.time() + 15
            while time.time() < deadline:
                try:
                    get_current_datacenter()
                    break
                except:
                    time.sleep(1)

            deregister_services(gethostname())
            os.system('consul leave')
        finally:
            post_data = json.loads(web.data() or '{}')
            runtime_settings_path = '/opt/armada/runtime_settings.json'
            if not post_data.get('keep-joined') and os.path.isfile(
                    runtime_settings_path):
                with open(runtime_settings_path) as f:
                    runtime_settings = json.load(f)
                runtime_settings['ships'] = []
                runtime_settings['is_commander'] = True
                with open(runtime_settings_path, 'w') as f:
                    json.dump(runtime_settings, f, sort_keys=True, indent=4)
        return self.status_ok({'message': 'Shutdown complete.'})
Example #3
0
def deregister_not_running_services():
    services_ids = get_local_services_ids()
    containers_ids = get_running_container_ids()
    for service_id in services_ids:
        if service_id != 'consul':
            container_id = service_id.split(':')[0]
            if container_id not in containers_ids:
                deregister_services(container_id)
Example #4
0
def deregister_not_running_services():
    services_ids = get_local_services_ids()
    containers_ids = get_running_container_ids()
    for service_id in services_ids:
        if service_id != 'consul':
            container_id = service_id.split(':')[0]
            if container_id not in containers_ids:
                deregister_services(container_id)
Example #5
0
def _deregister_not_running_services():
    services = _get_local_services()
    running_containers_ids = _get_running_container_ids()
    for service_id in services.keys():
        container_id, is_subservice = _get_container_id_with_subservice(service_id)
        if container_id in running_containers_ids:
            continue
        if not is_subservice:
            _mark_service_as_crashed(container_id, services[service_id]['Service'])
        deregister_services(container_id)
Example #6
0
def deregister_not_running_services():
    services_ids = get_local_services_ids()
    containers_ids = get_running_container_ids()
    for service_id in services_ids:
        if service_id != 'consul':
            container_id = service_id.split(':')[0]
            if container_id not in containers_ids:
                name = consul_query('agent/services')[service_id]['Service']
                params = get_container_parameters(container_id)
                kv_index = 0
                if kv.kv_list('service/{}/'.format(name)):
                    kv_index = int(
                        kv.kv_list(
                            'service/{}/'.format(name))[-1].split('/')[2]) + 1
                kv.save_service(name, kv_index, 'crashed', params,
                                container_id)
                deregister_services(container_id)
Example #7
0
    def POST(self):
        # 'startsecs=0' is to avoid restarting consul after `consul leave`.
        os.system('sed -i \'/autorestart=true/cautorestart=false\' /etc/supervisor/conf.d/consul.conf')
        os.system('echo startsecs=0 >> /etc/supervisor/conf.d/consul.conf')

        os.system('supervisorctl update consul')

        # As 'supervisorctl update' will restart Consul, we have to wait for it to be running.
        while True:
            try:
                get_current_datacenter()
                break
            except:
                pass

        deregister_services(gethostname())
        os.system('consul leave')
        return self.status_ok({'message': 'Shutdown complete.'})
Example #8
0
    def _stop_service(self, container_id):
        ship = get_ship_name()
        service_list = kv_list('ships/{}/service/'.format(ship))
        try:
            key = fnmatch.filter(service_list, '*/{}'.format(container_id))[0]
        except (IndexError, TypeError):
            key = None

        if not is_container_running(container_id):
            if key:
                kv_remove(key)
            try:
                deregister_services(container_id)
            except Exception as e:
                get_logger().exception(e)
        else:
            run_command_in_container('supervisorctl stop armada_agent',
                                     container_id)

            # TODO: Compatibility with old microservice images. Should be removed in future armada version.
            run_command_in_container(
                'supervisorctl stop register_in_service_discovery',
                container_id)

            docker_api = docker_client.api()
            last_exception = None
            try:
                deregister_services(container_id)
            except Exception as e:
                get_logger().exception(e)
            for i in range(3):
                try:
                    docker_api.stop(container_id)
                except Exception as e:
                    get_logger().debug(e, exc_info=True)
                    last_exception = e
                if not is_container_running(container_id):
                    if key:
                        kv_remove(key)
                    break
            if is_container_running(container_id):
                get_logger().error('Could not stop container: %s',
                                   container_id)
                raise last_exception
Example #9
0
    def _stop_service(self, container_id, force=False):
        if force:
            service_list = get_services_by_ship()
        else:
            service_list = get_local_services_from_kv_store()

        try:
            keys = fnmatch.filter(service_list, '*/{}'.format(container_id))
        except (IndexError, TypeError) as e:
            get_logger().exception(e)
            keys = []

        if not is_container_running(container_id):
            for key in keys:
                kv_remove(key)
            try:
                deregister_services(container_id)
            except Exception as e:
                get_logger().exception(e)
        else:
            run_command_in_container('supervisorctl stop armada_agent',
                                     container_id)
            trigger_hook('pre-stop', container_id)

            docker_api = docker_client.api()
            last_exception = None
            try:
                deregister_services(container_id)
            except Exception as e:
                get_logger().exception(e)
            for i in range(3):
                try:
                    docker_api.stop(container_id)
                except Exception as e:
                    get_logger().debug(e, exc_info=True)
                    last_exception = e
                if not is_container_running(container_id):
                    for key in keys:
                        kv_remove(key)
                    break
            if is_container_running(container_id):
                get_logger().error('Could not stop container: %s',
                                   container_id)
                raise last_exception
Example #10
0
    def POST(self):

        # Wpisujemy 'startsecs=0', zeby ubicie consula przez 'consul leave' nie spowodowalo jego restartu.
        os.system('sed -i \'/autorestart=true/cautorestart=false\' /etc/supervisor/conf.d/consul.conf')
        os.system('echo startsecs=0 >> /etc/supervisor/conf.d/consul.conf')

        os.system('supervisorctl update consul')

        # 'supervisorctl update' zrestartuje consula. Czekamy az wystartuje na nowo.
        while True:
            try:
                get_current_datacenter()
                break
            except:
                pass

        deregister_services(gethostname())
        os.system('consul leave')
        return self.status_ok({'message': 'Shutdown complete.'})
Example #11
0
    def POST(self):
        # 'startsecs=0' is to avoid restarting consul after `consul leave`.
        os.system(
            'sed -i \'/autorestart=true/cautorestart=false\' /etc/supervisor/conf.d/consul.conf'
        )
        os.system('echo startsecs=0 >> /etc/supervisor/conf.d/consul.conf')

        os.system('supervisorctl update consul')

        # As 'supervisorctl update' will restart Consul, we have to wait for it to be running.
        while True:
            try:
                get_current_datacenter()
                break
            except:
                pass

        deregister_services(gethostname())
        os.system('consul leave')
        return self.status_ok({'message': 'Shutdown complete.'})
Example #12
0
    def POST(self):

        # Wpisujemy 'startsecs=0', zeby ubicie consula przez 'consul leave' nie spowodowalo jego restartu.
        os.system(
            'sed -i \'/autorestart=true/cautorestart=false\' /etc/supervisor/conf.d/consul.conf'
        )
        os.system('echo startsecs=0 >> /etc/supervisor/conf.d/consul.conf')

        os.system('supervisorctl update consul')

        # 'supervisorctl update' zrestartuje consula. Czekamy az wystartuje na nowo.
        while True:
            try:
                get_current_datacenter()
                break
            except:
                pass

        deregister_services(gethostname())
        os.system('consul leave')
        return self.status_ok({'message': 'Shutdown complete.'})
Example #13
0
    def _stop_service(self, container_id):
        ship = get_ship_name()
        service_list = kv_list("ships/{}/service/".format(ship))
        try:
            key = fnmatch.filter(service_list, "*/{}".format(container_id))[0]
        except (IndexError, TypeError):
            key = None

        if not is_container_running(container_id):
            if key:
                kv_remove(key)
            try:
                deregister_services(container_id)
            except Exception as e:
                get_logger().exception(e)
        else:
            run_command_in_container("supervisorctl stop armada_agent", container_id)

            # TODO: Compatibility with old microservice images. Should be removed in future armada version.
            run_command_in_container("supervisorctl stop register_in_service_discovery", container_id)

            docker_api = docker_client.api()
            last_exception = None
            try:
                deregister_services(container_id)
            except Exception as e:
                get_logger().exception(e)
            for i in range(3):
                try:
                    docker_api.stop(container_id)
                except Exception as e:
                    get_logger().debug(e, exc_info=True)
                    last_exception = e
                if not is_container_running(container_id):
                    if key:
                        kv_remove(key)
                    break
            if is_container_running(container_id):
                get_logger().error("Could not stop container: %s", container_id)
                raise last_exception
Example #14
0
def _deregister_not_running_services():
    try:
        ship = get_ship_name()
    except:
        ship = get_ship_ip()
    services = _get_local_services()
    running_containers_ids = _get_running_container_ids()
    for service_id in services.keys():
        container_id, is_subservice = _get_container_id_with_subservice(service_id)
        if container_id in running_containers_ids:
            continue
        if not is_subservice:
            name = services[service_id]['Service']
            kv.update_container_status('crashed', ship=ship, name=name, container_id=container_id)
        deregister_services(container_id)

    services_keys = kv.kv_list('ships/{}/service/'.format(ship)) or []
    for service_key in services_keys:
        container_id = service_key.split('/')[-1]
        if container_id not in running_containers_ids:
            kv.update_container_status('crashed', key=service_key)
            deregister_services(container_id)
Example #15
0
    def on_post(self, req, resp):
        try:
            # 'startsecs=0' is to avoid restarting consul after `consul leave`.
            check_call(
                'sed -i \'/autorestart=true/cautorestart=false\' /etc/supervisor/conf.d/consul.conf',
                shell=True)
            check_call(
                'echo startsecs=0 >> /etc/supervisor/conf.d/consul.conf',
                shell=True)

            check_call(['supervisorctl', 'update', 'consul'])

            # As 'supervisorctl update' will restart Consul, we have to wait for it to be running.
            deadline = time.time() + 15
            ok = False
            while time.time() < deadline:
                try:
                    get_current_datacenter()
                    ok = True
                    break
                except Exception:
                    time.sleep(1)
            if not ok:
                get_logger().warn('Restarting consul timed out.')

            deregister_services(gethostname())
            check_call(['consul', 'leave'])
        finally:
            post_data = req.json
            runtime_settings_path = '/opt/armada/runtime_settings.json'
            if not post_data.get('keep-joined') and os.path.isfile(
                    runtime_settings_path):
                with open(runtime_settings_path) as f:
                    runtime_settings = json.load(f)
                runtime_settings['ships'] = []
                runtime_settings['is_commander'] = True
                with open(runtime_settings_path, 'w') as f:
                    json.dump(runtime_settings, f, sort_keys=True, indent=4)
        return self.status_ok(resp, {'message': 'Shutdown complete.'})
Example #16
0
def _update_running_services():
    ship_ip = get_ship_ip()
    try:
        ship_name = get_ship_name(ship_ip)
    except:
        ship_name = ship_ip
    services = _get_local_services_from_catalog()
    running_containers_ids = _get_running_container_ids()
    local_services = get_local_services_from_kv_store()
    local_services_container_ids = [it.split('/')[-1] for it in local_services]
    for service_id in services.keys():
        container_id, is_subservice = _get_container_id_with_subservice(
            service_id)
        if container_id not in local_services_container_ids:
            save_container(ship_name, container_id, 'started', ship_ip=ship_ip)
            get_logger().info(
                'Saved container in kv-store: {container_id}'.format(
                    container_id=container_id))
        if container_id in running_containers_ids:
            continue
        if not is_subservice:
            name = services[service_id]['Service']
            update_container_status('crashed',
                                    ship=ship_name,
                                    service_name=name,
                                    container_id=container_id)
            get_logger().info('Set status to "crashed": {container_id}'.format(
                container_id=container_id))
        deregister_services(container_id)

    for service_key in local_services:
        container_id = service_key.split('/')[-1]
        if container_id not in running_containers_ids:
            update_container_status('crashed', key=service_key)
            get_logger().info('Set status to "crashed": {container_id}'.format(
                container_id=container_id))
            deregister_services(container_id)