コード例 #1
0
ファイル: service.py プロジェクト: ssthom/anchore-engine
def start(services, no_auto_upgrade, anchore_module, skip_config_validate,
          skip_db_compat_check, all):
    """
    Startup and monitor service processes. Specify a list of service names or empty for all.
    """

    global config
    ecode = ExitCode.ok

    if not anchore_module:
        module_name = "anchore_engine"
    else:
        module_name = str(anchore_module)

    if os.environ.get('ANCHORE_ENGINE_SKIP_DB_COMPAT_CHECK',
                      str(skip_db_compat_check)).lower() in [
                          'true', 't', 'y', 'yes'
                      ]:
        skip_db_compat_check = True
    else:
        skip_db_compat_check = False

    if services:
        input_services = list(services)
    else:
        input_services = os.getenv('ANCHORE_ENGINE_SERVICES',
                                   '').strip().split()

    if not input_services and not all:
        raise click.exceptions.BadArgumentUsage(
            'No services defined to start. Must either provide service arguments, ANCHORE_ENGINE_SERVICES env var, or --all option'
        )

    try:
        validate_params = {
            'services': True,
            'webhooks': True,
            'credentials': True
        }
        if skip_config_validate:
            try:
                items = skip_config_validate.split(',')
                for item in items:
                    validate_params[item] = False
            except Exception as err:
                raise Exception(err)

        # find/set up configuration
        configdir = config['configdir']
        configfile = os.path.join(configdir, "config.yaml")

        localconfig = None
        if os.path.exists(configfile):
            try:
                localconfig = anchore_engine.configuration.localconfig.load_config(
                    configdir=configdir,
                    configfile=configfile,
                    validate_params=validate_params)
            except Exception as err:
                raise Exception("cannot load local configuration: " + str(err))
        else:
            raise Exception(
                "cannot locate configuration file ({})".format(configfile))

        # load the appropriate DB module
        try:
            logger.info(
                "Loading DB routines from module ({})".format(module_name))
            module = importlib.import_module(module_name +
                                             ".db.entities.upgrade")
        except Exception as err:
            raise Exception("Input anchore-module (" + str(module_name) +
                            ") cannot be found/imported - exception: " +
                            str(err))

        # get the list of local services to start
        startFailed = False
        if not input_services:
            config_services = localconfig.get('services', {})
            if not config_services:
                logger.warn(
                    'could not find any services to execute in the config file'
                )
                sys.exit(1)

            input_services = [
                name for name, srv_conf in list(config_services.items())
                if srv_conf.get('enabled')
            ]

        services = []
        for service_conf_name in input_services:
            if service_conf_name in list(service_map.values()):
                svc = service_conf_name
            else:
                svc = service_map.get(service_conf_name)

            if svc:
                services.append(svc)
            else:
                logger.warn(
                    'specified service {} not found in list of available services {} - removing from list of services to start'
                    .format(service_conf_name, list(service_map.keys())))

        if 'anchore-catalog' in services:
            services.remove('anchore-catalog')
            services.insert(0, 'anchore-catalog')

        if not services:
            logger.error(
                "No services found in ANCHORE_ENGINE_SERVICES or as enabled in config.yaml to start - exiting"
            )
            sys.exit(1)

        # preflight - db checks
        try:
            db_params = anchore_engine.db.entities.common.get_params(
                localconfig)

            # override db_timeout since upgrade might require longer db session timeout setting
            try:
                if 'timeout' in db_params.get('db_connect_args', {}):
                    db_params['db_connect_args']['timeout'] = 86400
                elif 'connect_timeout' in db_params.get('db_connect_args', {}):
                    db_params['db_connect_args']['connect_timeout'] = 86400
            except Exception as err:
                pass

            anchore_manager.util.db.connect_database(db_params, db_retries=300)
            code_versions, db_versions = anchore_manager.util.db.init_database(
                upgrade_module=module,
                localconfig=localconfig,
                do_db_compatibility_check=(not skip_db_compat_check))

            in_sync = False
            timed_out = False
            max_timeout = 3600

            timer = time.time()
            while not in_sync and not timed_out:
                code_versions, db_versions = module.get_versions()

                if code_versions and db_versions:
                    if code_versions['db_version'] != db_versions['db_version']:
                        if not no_auto_upgrade and 'anchore-catalog' in services:
                            logger.info("Performing upgrade.")
                            try:
                                # perform the upgrade logic here
                                rc = module.run_upgrade()
                                if rc:
                                    logger.info("Upgrade completed")
                                else:
                                    logger.info(
                                        "No upgrade necessary. Completed.")
                            except Exception as err:
                                raise err

                            in_sync = True
                        else:
                            logger.warn(
                                "this version of anchore-engine requires the anchore DB version ({}) but we discovered anchore DB version ({}) in the running DB - it is safe to run the upgrade while seeing this message - will retry for {} more seconds."
                                .format(
                                    str(code_versions['db_version']),
                                    str(db_versions['db_version']),
                                    str(max_timeout -
                                        int(time.time() - timer))))
                            time.sleep(5)
                    else:
                        logger.info("DB version and code version in sync.")
                        in_sync = True
                else:
                    logger.warn(
                        'no existing anchore DB data can be discovered, assuming bootstrap'
                    )
                    in_sync = True

                if (max_timeout - int(time.time() - timer)) < 0:
                    timed_out = True

            if not in_sync:
                raise Exception(
                    "this version of anchore-engine requires the anchore DB version ("
                    + str(code_versions['db_version']) +
                    ") but we discovered anchore DB version (" +
                    str(db_versions['db_version']) +
                    ") in the running DB - please perform the DB upgrade process and retry\n"
                    "See: https://docs.anchore.com/current/docs/engine/engine_installation/upgrade/#advanced--manual-upgrade-procedure"
                )

        except Exception as err:
            raise err

        finally:
            rc = anchore_engine.db.entities.common.do_disconnect()

        # start up services
        logger.info('Starting services: {}'.format(services))

        for supportdir in ["/var/log/anchore", "/var/run/anchore"]:
            try:
                if not os.path.exists(supportdir):
                    os.makedirs(supportdir, 0o755)
            except Exception as err:
                logger.error(
                    "cannot create log directory {} - exception: {}".format(
                        supportdir, str(err)))
                raise err

        pids = []
        keepalive_threads = []
        for service in services:
            pidfile = "/var/run/anchore/" + service + ".pid"
            try:
                terminate_service(service, flush_pidfile=True)

                service_thread = ServiceThread(startup_service,
                                               (service, configdir))
                keepalive_threads.append(service_thread)
                max_tries = 30
                tries = 0
                alive = True
                while not os.path.exists(pidfile) and tries < max_tries:
                    logger.info(
                        "waiting for service pidfile {} to exist {}/{}".format(
                            pidfile, tries, max_tries))

                    try:
                        alive = service_thread.thread.is_alive()
                    except:
                        pass
                    if not alive:
                        logger.info(
                            "service thread has stopped {}".format(service))
                        break

                    time.sleep(1)
                    tries = tries + 1

                logger.info("auto_restart_services setting: {}".format(
                    localconfig.get('auto_restart_services', False)))
                if not localconfig.get('auto_restart_services', False):
                    logger.info(
                        "checking for startup failure pidfile={}, is_alive={}".
                        format(os.path.exists(pidfile), alive))
                    if not os.path.exists(pidfile) or not alive:
                        raise Exception(
                            "service thread for ({}) failed to start".format(
                                service))

                time.sleep(1)
            except Exception as err:
                startFailed = True
                logger.warn("service start failed - exception: {}".format(
                    str(err)))
                break

        if startFailed:
            logger.fatal(
                "one or more services failed to start. cleanly terminating the others"
            )
            for service in services:
                terminate_service(service, flush_pidfile=True)
            sys.exit(1)
        else:
            # start up the log watchers
            try:
                observer = Observer()
                observer.schedule(AnchoreLogWatcher(),
                                  path="/var/log/anchore/")
                observer.start()

                try:
                    while True:
                        time.sleep(1)
                        if localconfig.get(
                                'auto_restart_services', False
                        ):  # 'auto_restart_services' in localconfig and localconfig['auto_restart_services']:
                            for service_thread in keepalive_threads:
                                if not service_thread.thread.is_alive():
                                    logger.info(
                                        "restarting service: {}".format(
                                            service_thread.thread.name))
                                    service_thread.start()

                except KeyboardInterrupt:
                    observer.stop()
                observer.join()

            except Exception as err:
                logger.error(
                    "failed to startup log watchers - exception: {}".format(
                        str(err)))
                raise err

    except Exception as err:
        log_error('servicestart', err)
        ecode = ExitCode.failed

    doexit(ecode)
コード例 #2
0
def check(configfile, analysis_archive):
    """
    Test the configuration in the expected anchore-engine config location or override that and use the configuration file provided as an option.

    To test, the system will read and write a very small data document to the driver and then delete it on completion.
    """

    db_conf = db_context()
    db_preflight(db_conf['params'], db_conf['retries'])

    logger.info('Using config file {}'.format(configfile))
    sys_config = load_config(configfile=configfile)

    if sys_config:
        service_config = sys_config['services']['catalog']
    else:
        service_config = None

    if not service_config:
        logger.info(
            'No configuration file or content available. Cannot test archive driver configuration'
        )
        fail_exit()

    if analysis_archive:
        try:
            object_store.initialize(service_config,
                                    manager_id=ANALYSIS_ARCHIVE_MANAGER_ID,
                                    config_keys=[ANALYSIS_ARCHIVE_MANAGER_ID])
        except:
            logger.error(
                'No "analysis_archive" configuration section found in the configuration. To check a config that uses the default backend for analysis archive data, use the regular object storage check'
            )
            fail_exit()

        mgr = object_store.get_manager(ANALYSIS_ARCHIVE_MANAGER_ID)
    else:
        object_store.initialize(service_config,
                                manager_id=DEFAULT_OBJECT_STORE_MANAGER_ID,
                                config_keys=[
                                    DEFAULT_OBJECT_STORE_MANAGER_ID,
                                    ALT_OBJECT_STORE_CONFIG_KEY
                                ])
        mgr = object_store.get_manager()

    test_user_id = 'test'
    test_bucket = 'anchorecliconfigtest'
    test_archive_id = 'cliconfigtest'
    test_data = 'clitesting at {}'.format(
        datetime.datetime.utcnow().isoformat())

    logger.info(
        'Checking existence of test document with user_id = {}, bucket = {} and archive_id = {}'
        .format(test_user_id, test_bucket, test_archive_id))
    if mgr.exists(test_user_id, test_bucket, test_archive_id):
        test_archive_id = 'cliconfigtest2'
        if mgr.exists(test_user_id, test_bucket, test_archive_id):
            logger.error(
                'Found existing records for archive doc to test, aborting test to avoid overwritting any existing data'
            )
            doexit(1)

    logger.info(
        'Creating test document with user_id = {}, bucket = {} and archive_id = {}'
        .format(test_user_id, test_bucket, test_archive_id))
    result = mgr.put(test_user_id,
                     test_bucket,
                     test_archive_id,
                     data=test_data)
    if not result:
        logger.warn(
            'Got empty response form archive PUT operation: {}'.format(result))

    logger.info('Checking document fetch')
    loaded = str(mgr.get(test_user_id, test_bucket, test_archive_id), 'utf-8')
    if not loaded:
        logger.error(
            'Failed retrieving the written document. Got: {}'.format(loaded))
        doexit(ExitCode.obj_store_failed)

    if str(loaded) != test_data:
        logger.error(
            'Failed retrieving the written document. Got something other than expected. Expected: "{}" Got: "{}"'
            .format(test_data, loaded))
        doexit(ExitCode.obj_store_failed)

    logger.info('Removing test object')
    mgr.delete(test_user_id, test_bucket, test_archive_id)

    if mgr.exists(test_user_id, test_bucket, test_archive_id):
        logger.error('Found archive object after it should have been removed')
        doexit(ExitCode.obj_store_failed)

    logger.info('Archive config check completed successfully')