Exemple #1
0
def on_state_change(settings,
                    tabpy_state,
                    python_service,
                    logger=logging.getLogger(__name__)):
    try:
        logger.log(logging.INFO, "Loading state from state file")
        config = util._get_state_from_file(
            settings[SettingsParameters.StateFilePath], logger=logger)
        new_ps_state = TabPyState(config=config, settings=settings)

        (has_changes,
         changes) = _get_latest_service_state(settings, tabpy_state,
                                              new_ps_state, python_service)
        if not has_changes:
            logger.info("Nothing changed, return.")
            return

        new_endpoints = new_ps_state.get_endpoints()
        for object_name in changes['endpoints']:
            (object_type, object_version,
             object_path) = changes['endpoints'][object_name]

            if not object_path and not object_version:  # removal
                logger.info(f'Removing object: URI={object_name}')

                python_service.manage_request(DeleteObjects([object_name]))

                cleanup_endpoint_files(object_name,
                                       settings[SettingsParameters.UploadDir],
                                       logger=logger)

            else:
                endpoint_info = new_endpoints[object_name]
                is_update = object_version > 1
                if object_type == 'alias':
                    msg = LoadObject(object_name, endpoint_info['target'],
                                     object_version, is_update, 'alias')
                else:
                    local_path = object_path
                    msg = LoadObject(object_name, local_path, object_version,
                                     is_update, object_type)

                python_service.manage_request(msg)
                wait_for_endpoint_loaded(python_service, object_name)

                # cleanup old version of endpoint files
                if object_version > 2:
                    cleanup_endpoint_files(
                        object_name,
                        settings[SettingsParameters.UploadDir],
                        logger=logger,
                        retain_versions=[object_version, object_version - 1])

    except Exception as e:
        err_msg = format_exception(e, 'on_state_change')
        logger.log(logging.ERROR,
                   f'Error submitting update model request: error={err_msg}')
Exemple #2
0
def on_state_change(settings):
    try:
        py_handler = settings['py_handler']

        logger.info("Loading state from state file")
        config = util._get_state_from_file(settings['state_file_path'])
        new_ps_state = TabPyState(config=config, settings=settings)

        (has_changes,
         changes) = _get_latest_service_state(settings, new_ps_state)
        if not has_changes:
            logger.info("Nothing changed, return.")
            return

        new_endpoints = new_ps_state.get_endpoints()
        for object_name in changes['endpoints']:
            (object_type, object_version,
             object_path) = changes['endpoints'][object_name]

            if not object_path and not object_version:  # removal
                logger.info("Removing object: URI={}".format(object_name))

                py_handler.manage_request(DeleteObjects([object_name]))

                cleanup_endpoint_files(object_name, settings['upload_dir'])

            else:
                endpoint_info = new_endpoints[object_name]
                is_update = object_version > 1
                if object_type == 'alias':
                    msg = LoadObject(object_name, endpoint_info['target'],
                                     object_version, is_update, 'alias')
                else:
                    local_path = object_path
                    msg = LoadObject(object_name, local_path, object_version,
                                     is_update, object_type)

                py_handler.manage_request(msg)
                wait_for_endpoint_loaded(py_handler, object_name)

                # cleanup old version of endpoint files
                if object_version > 2:
                    cleanup_endpoint_files(
                        object_name, settings['upload_dir'],
                        [object_version, object_version - 1])

    except Exception as e:
        err_msg = format_exception(e, 'on_state_change')
        logger.error(
            "Error submitting update model request: error={}".format(err_msg))
def get_config():
    """Provide consistent mechanism for pulling in configuration.

    Attempt to retain backward compatibility for existing implementations by
    grabbing port setting from CLI first.

    Take settings in the following order:

    1. CLI arguments, if present - port only - may be able to deprecate
    2. common.config file, and
    3. OS environment variables (for ease of setting defaults if not present)
    4. current defaults if a setting is not present in any location

    Additionally provide similar configuration capabilities in between
    common.config and environment variables.
    For consistency use the same variable name in the config file as in the os
    environment.
    For naming standards use all capitals and start with 'TABPY_'
    """

    try:
        import tabpy_server.common.config as config
    except ImportError:
        config = None

    settings = {}

    cli_args = parse_arguments()

    if cli_args.port is not None:
        settings['port'] = cli_args.port
    else:
        try:
            settings['port'] = config.TABPY_PORT
        except AttributeError:
            settings['port'] = os.getenv('TABPY_PORT', 9004)

    try:
        settings['server_version'] = config.TABPY_SERVER_VERSION
    except AttributeError:
        settings['server_version'] = os.getenv('TABPY_SERVER_VERSION', 'Alpha')

    try:
        settings['bind_ip'] = config.TABPY_BIND_IP
    except AttributeError:
        settings['bind_ip'] = os.getenv('TABPY_BIND_IP', '0.0.0.0')

    try:
        settings['upload_dir'] = config.TABPY_QUERY_OBJECT_PATH
    except AttributeError:
        settings['upload_dir'] = os.getenv('TABPY_QUERY_OBJECT_PATH',
                                           '/tmp/query_objects')

    if not os.path.exists(settings['upload_dir']):
        os.makedirs(settings['upload_dir'])

    try:
        _state_file_path = config.TABPY_STATE_PATH
    except AttributeError:
        _state_file_path = os.getenv('TABPY_STATE_PATH', './')
    settings['state_file_path'] = os.path.realpath(
        os.path.normpath(os.path.expanduser(_state_file_path)))

    # if state.ini does not exist try and create it - remove last dependence
    # on batch/shell script
    if not os.path.isfile('{}/state.ini'.format(settings['state_file_path'])):
        shutil.copy('./state.ini.template',
                    '{}/state.ini'.format(settings['state_file_path']))

    log_info("Loading state from state file")
    tabpy_state = _get_state_from_file(settings['state_file_path'])
    settings['tabpy'] = TabPyState(config=tabpy_state)

    settings['py_handler'] = PythonServiceHandler(PythonService())
    settings['compress_response'] = True if TORNADO_MAJOR >= 4 else "gzip"
    settings['static_path'] = os.path.join(os.path.dirname(__file__), "static")

    # Set subdirectory from config if applicable
    subdirectory = ""
    if tabpy_state.has_option("Service Info", "Subdirectory"):
        subdirectory = "/" + tabpy_state.get("Service Info", "Subdirectory")

    return settings, subdirectory
Exemple #4
0
    def _parse_config(self, config_file):
        """Provide consistent mechanism for pulling in configuration.

        Attempt to retain backward compatibility for
        existing implementations by grabbing port
        setting from CLI first.

        Take settings in the following order:

        1. CLI arguments if present
        2. config file
        3. OS environment variables (for ease of
           setting defaults if not present)
        4. current defaults if a setting is not present in any location

        Additionally provide similar configuration capabilities in between
        config file and environment variables.
        For consistency use the same variable name in the config file as
        in the os environment.
        For naming standards use all capitals and start with 'TABPY_'
        """
        self.settings = {}
        self.subdirectory = ""
        self.tabpy_state = None
        self.python_service = None
        self.credentials = {}

        parser = configparser.ConfigParser()

        if os.path.isfile(config_file):
            with open(config_file) as f:
                parser.read_string(f.read())
        else:
            logger.warning(f'Unable to find config file at {config_file}, '
                           'using default settings.')

        def set_parameter(settings_key,
                          config_key,
                          default_val=None,
                          check_env_var=False):
            if config_key is not None and\
               parser.has_section('TabPy') and\
               parser.has_option('TabPy', config_key):
                self.settings[settings_key] = parser.get('TabPy', config_key)
                logger.debug(f'Parameter {settings_key} set to '
                             f'"{self.settings[settings_key]}" '
                             'from config file')
            elif check_env_var:
                self.settings[settings_key] = os.getenv(
                    config_key, default_val)
                logger.debug(f'Parameter {settings_key} set to '
                             f'"{self.settings[settings_key]}" '
                             'from environment variable')
            elif default_val is not None:
                self.settings[settings_key] = default_val
                logger.debug(f'Parameter {settings_key} set to '
                             f'"{self.settings[settings_key]}" '
                             'from default value')
            else:
                logger.debug(f'Parameter {settings_key} is not set')

        set_parameter(SettingsParameters.Port,
                      ConfigParameters.TABPY_PORT,
                      default_val=9004,
                      check_env_var=True)
        set_parameter(SettingsParameters.ServerVersion,
                      None,
                      default_val=__version__)

        set_parameter(SettingsParameters.EvaluateTimeout,
                      ConfigParameters.TABPY_EVALUATE_TIMEOUT,
                      default_val=30)
        try:
            self.settings[SettingsParameters.EvaluateTimeout] = float(
                self.settings[SettingsParameters.EvaluateTimeout])
        except ValueError:
            logger.warning('Evaluate timeout must be a float type. Defaulting '
                           'to evaluate timeout of 30 seconds.')
            self.settings[SettingsParameters.EvaluateTimeout] = 30

        set_parameter(SettingsParameters.UploadDir,
                      ConfigParameters.TABPY_QUERY_OBJECT_PATH,
                      default_val='/tmp/query_objects',
                      check_env_var=True)
        if not os.path.exists(self.settings[SettingsParameters.UploadDir]):
            os.makedirs(self.settings[SettingsParameters.UploadDir])

        # set and validate transfer protocol
        set_parameter(SettingsParameters.TransferProtocol,
                      ConfigParameters.TABPY_TRANSFER_PROTOCOL,
                      default_val='http')
        self.settings[SettingsParameters.TransferProtocol] =\
            self.settings[SettingsParameters.TransferProtocol].lower()

        set_parameter(SettingsParameters.CertificateFile,
                      ConfigParameters.TABPY_CERTIFICATE_FILE)
        set_parameter(SettingsParameters.KeyFile,
                      ConfigParameters.TABPY_KEY_FILE)
        self._validate_transfer_protocol_settings()

        # if state.ini does not exist try and create it - remove
        # last dependence on batch/shell script
        set_parameter(SettingsParameters.StateFilePath,
                      ConfigParameters.TABPY_STATE_PATH,
                      default_val='./tabpy-server/tabpy_server',
                      check_env_var=True)
        self.settings[SettingsParameters.StateFilePath] = os.path.realpath(
            os.path.normpath(
                os.path.expanduser(
                    self.settings[SettingsParameters.StateFilePath])))
        state_file_path = self.settings[SettingsParameters.StateFilePath]
        logger.info('Loading state from state file '
                    f'{os.path.join(state_file_path, "state.ini")}')
        tabpy_state = _get_state_from_file(state_file_path)
        self.tabpy_state = TabPyState(config=tabpy_state,
                                      settings=self.settings)

        self.python_service = PythonServiceHandler(PythonService())
        self.settings['compress_response'] = True
        set_parameter(SettingsParameters.StaticPath,
                      ConfigParameters.TABPY_STATIC_PATH,
                      default_val='./')
        self.settings[SettingsParameters.StaticPath] =\
            os.path.abspath(self.settings[SettingsParameters.StaticPath])
        logger.debug(f'Static pages folder set to '
                     f'"{self.settings[SettingsParameters.StaticPath]}"')

        # Set subdirectory from config if applicable
        if tabpy_state.has_option("Service Info", "Subdirectory"):
            self.subdirectory = "/" + \
                tabpy_state.get("Service Info", "Subdirectory")

        # If passwords file specified load credentials
        set_parameter(ConfigParameters.TABPY_PWD_FILE,
                      ConfigParameters.TABPY_PWD_FILE)
        if ConfigParameters.TABPY_PWD_FILE in self.settings:
            if not self._parse_pwd_file():
                msg = ('Failed to read passwords file '
                       f'{self.settings[ConfigParameters.TABPY_PWD_FILE]}')
                logger.critical(msg)
                raise RuntimeError(msg)
        else:
            logger.info("Password file is not specified: "
                        "Authentication is not enabled")

        features = self._get_features()
        self.settings[SettingsParameters.ApiVersions] =\
            {'v1': {'features': features}}

        set_parameter(SettingsParameters.LogRequestContext,
                      ConfigParameters.TABPY_LOG_DETAILS,
                      default_val='false')
        self.settings[SettingsParameters.LogRequestContext] = (
            self.settings[SettingsParameters.LogRequestContext].lower() !=
            'false')
        call_context_state =\
            'enabled' if self.settings[SettingsParameters.LogRequestContext]\
            else 'disabled'
        logger.info(f'Call context logging is {call_context_state}')
Exemple #5
0
def get_config(config_file):
    """Provide consistent mechanism for pulling in configuration.

    Attempt to retain backward compatibility for existing implementations by
    grabbing port setting from CLI first.

    Take settings in the following order:

    1. CLI arguments, if present - port only - may be able to deprecate
    2. common.config file, and
    3. OS environment variables (for ease of setting defaults if not present)
    4. current defaults if a setting is not present in any location

    Additionally provide similar configuration capabilities in between
    common.config and environment variables.
    For consistency use the same variable name in the config file as in the os
    environment.
    For naming standards use all capitals and start with 'TABPY_'
    """
    parser = configparser.ConfigParser()

    if os.path.isfile(config_file):
        with open(config_file) as f:
            parser.read_string(f.read())
    else:
        logger.warning(
            "Unable to find config file at '{}', using default settings.".
            format(config_file))

    settings = {}
    for section in parser.sections():
        if section == "TabPy":
            for key, val in parser.items(section):
                settings[key] = val
            break

    def set_parameter(settings_key,
                      config_key,
                      default_val=None,
                      check_env_var=False):
        if config_key is not None and parser.has_option('TabPy', config_key):
            settings[settings_key] = parser.get('TabPy', config_key)
        elif check_env_var:
            settings[settings_key] = os.getenv(config_key, default_val)
        elif default_val is not None:
            settings[settings_key] = default_val

    if cli_args is not None and cli_args.port is not None:
        settings['port'] = cli_args.port
    else:
        set_parameter('port',
                      'TABPY_PORT',
                      default_val=9004,
                      check_env_var=True)
        try:
            settings['port'] = int(settings['port'])
        except ValueError:
            logger.warning('Error during config validation, invalid port: {}. '
                           'Using default port 9004'.format(settings['port']))
            settings['port'] = 9004

    set_parameter('server_version', None, default_val=__version__)
    set_parameter('bind_ip',
                  'TABPY_BIND_IP',
                  default_val='0.0.0.0',
                  check_env_var=True)

    set_parameter('upload_dir',
                  'TABPY_QUERY_OBJECT_PATH',
                  default_val='/tmp/query_objects',
                  check_env_var=True)
    if not os.path.exists(settings['upload_dir']):
        os.makedirs(settings['upload_dir'])

    set_parameter('state_file_path',
                  'TABPY_STATE_PATH',
                  default_val='./',
                  check_env_var=True)
    settings['state_file_path'] = os.path.realpath(
        os.path.normpath(os.path.expanduser(settings['state_file_path'])))

    # set and validate transfer protocol
    set_parameter('transfer_protocol',
                  'TABPY_TRANSFER_PROTOCOL',
                  default_val='http')
    settings['transfer_protocol'] = settings['transfer_protocol'].lower()

    set_parameter('certificate_file', 'TABPY_CERTIFICATE_FILE')
    set_parameter('key_file', 'TABPY_KEY_FILE')
    validate_transfer_protocol_settings(settings)

    # if state.ini does not exist try and create it - remove last dependence
    # on batch/shell script
    state_file_path = settings['state_file_path']
    logger.info("Loading state from state file {}".format(
        os.path.join(state_file_path, "state.ini")))
    tabpy_state = _get_state_from_file(state_file_path)
    settings['tabpy'] = TabPyState(config=tabpy_state, settings=settings)

    settings['py_handler'] = PythonServiceHandler(PythonService())
    settings['compress_response'] = True if TORNADO_MAJOR >= 4 else "gzip"
    settings['static_path'] = os.path.join(os.path.dirname(__file__), "static")

    # Set subdirectory from config if applicable
    subdirectory = ""
    if tabpy_state.has_option("Service Info", "Subdirectory"):
        subdirectory = "/" + tabpy_state.get("Service Info", "Subdirectory")

    return settings, subdirectory
Exemple #6
0
def get_config():
    """Provide consistent mechanism for pulling in configuration.

    Attempt to retain backward compatibility for existing implementations by
    grabbing port setting from CLI first.

    Take settings in the following order:

    1. CLI arguments, if present - port only - may be able to deprecate
    2. common.config file, and
    3. OS environment variables (for ease of setting defaults if not present)
    4. current defaults if a setting is not present in any location

    Additionally provide similar configuration capabilities in between
    common.config and environment variables.
    For consistency use the same variable name in the config file as in the os
    environment.
    For naming standards use all capitals and start with 'TABPY_'
    """
    parser = configparser.ConfigParser()

    cli_args = parse_arguments()
    path = cli_args.config if cli_args.config is not None else os.path.join(
        os.path.dirname(__file__), 'common', 'default.conf')
    if os.path.isfile(path):
        with open(path) as f:
            data = '[dummy-header]\n' + f.read()
        parser.read_string(data)
    else:
        logger.warning(
            "Unable to find config file at '{}', using default settings.".
            format(path))

    settings = {}
    for section in parser.sections():
        for key, val in parser.items(section):
            settings[key] = val

    def set_parameter(settings_key,
                      config_key,
                      default_val=None,
                      check_env_var=False):
        if config_key is not None and parser.has_option(
                'dummy-header', config_key):
            settings[settings_key] = parser.get('dummy-header', config_key)
        elif check_env_var:
            settings[settings_key] = os.getenv(config_key, default_val)
        elif default_val is not None:
            settings[settings_key] = default_val

    # convert log level string into int, set log level
    # defaults to INFO if the level is not recognized
    set_parameter('log_level',
                  'TABPY_LOG_LEVEL',
                  default_val='INFO',
                  check_env_var=True)
    settings['log_level'] = settings['log_level'].upper()
    level = settings['log_level']
    if level not in {'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'}:
        logger.warning('The log level indicated is not supported: {}. '
                       'Will be using the default level INFO.'.format(level))
        level = 'INFO'
    settings['log_level'] = level

    # Create application wide logging
    root_logger = logging.getLogger('')
    root_logger.setLevel(level)
    temp_dir = tempfile.gettempdir()
    fh = logging.handlers.RotatingFileHandler(filename=os.path.join(
        temp_dir, "tabpy_log.log"),
                                              maxBytes=10000000,
                                              backupCount=5)
    formatter = logging.Formatter(
        '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    fh.setFormatter(formatter)
    root_logger.addHandler(fh)

    if cli_args.port is not None:
        settings['port'] = cli_args.port
    else:
        set_parameter('port',
                      'TABPY_PORT',
                      default_val=9004,
                      check_env_var=True)
        try:
            settings['port'] = int(settings['port'])
        except ValueError:
            logger.warning('Error during config validation, invalid port: {}. '
                           'Using default port 9004'.format(settings['port']))
            settings['port'] = 9004

    set_parameter('server_version', None, default_val=__version__)
    set_parameter('bind_ip',
                  'TABPY_BIND_IP',
                  default_val='0.0.0.0',
                  check_env_var=True)

    set_parameter('upload_dir',
                  'TABPY_QUERY_OBJECT_PATH',
                  default_val='/tmp/query_objects',
                  check_env_var=True)
    if not os.path.exists(settings['upload_dir']):
        os.makedirs(settings['upload_dir'])

    set_parameter('state_file_path',
                  'TABPY_STATE_PATH',
                  default_val='./',
                  check_env_var=True)
    settings['state_file_path'] = os.path.realpath(
        os.path.normpath(os.path.expanduser(settings['state_file_path'])))

    # set and validate transfer protocol
    set_parameter('transfer_protocol',
                  'TABPY_TRANSFER_PROTOCOL',
                  default_val='http')
    settings['transfer_protocol'] = settings['transfer_protocol'].lower()

    set_parameter('certificate_file', 'TABPY_CERTIFICATE_FILE')
    set_parameter('key_file', 'TABPY_KEY_FILE')
    validate_transfer_protocol_settings(settings)

    # if state.ini does not exist try and create it - remove last dependence
    # on batch/shell script
    state_file_path = settings['state_file_path']
    logger.info(
        "Loading state from state file {}/state.ini".format(state_file_path))
    tabpy_state = _get_state_from_file(state_file_path)
    settings['tabpy'] = TabPyState(config=tabpy_state, settings=settings)

    settings['py_handler'] = PythonServiceHandler(PythonService())
    settings['compress_response'] = True if TORNADO_MAJOR >= 4 else "gzip"
    settings['static_path'] = os.path.join(os.path.dirname(__file__), "static")

    # Set subdirectory from config if applicable
    subdirectory = ""
    if tabpy_state.has_option("Service Info", "Subdirectory"):
        subdirectory = "/" + tabpy_state.get("Service Info", "Subdirectory")

    return settings, subdirectory
Exemple #7
0
    def _parse_config(self, config_file):
        """Provide consistent mechanism for pulling in configuration.

        Attempt to retain backward compatibility for
        existing implementations by grabbing port
        setting from CLI first.

        Take settings in the following order:

        1. CLI arguments if present
        2. config file
        3. OS environment variables (for ease of
           setting defaults if not present)
        4. current defaults if a setting is not present in any location

        Additionally provide similar configuration capabilities in between
        config file and environment variables.
        For consistency use the same variable name in the config file as
        in the os environment.
        For naming standards use all capitals and start with 'TABPY_'
        """
        self.settings = {}
        self.subdirectory = ""
        self.tabpy_state = None
        self.python_service = None
        self.credentials = {}

        parser = configparser.ConfigParser()

        if os.path.isfile(config_file):
            with open(config_file) as f:
                parser.read_string(f.read())
        else:
            logger.warning(
                "Unable to find config file at '{}', "
                "using default settings.".format(config_file))

        def set_parameter(settings_key,
                          config_key,
                          default_val=None,
                          check_env_var=False):
            if config_key is not None and\
               parser.has_option('TabPy', config_key):
                self.settings[settings_key] = parser.get('TabPy', config_key)
            elif check_env_var:
                self.settings[settings_key] = os.getenv(
                    config_key, default_val)
            elif default_val is not None:
                self.settings[settings_key] = default_val

        set_parameter('port', ConfigParameters.TABPY_PORT,
                      default_val=9004, check_env_var=True)
        set_parameter('server_version', None, default_val=__version__)

        set_parameter('upload_dir', ConfigParameters.TABPY_QUERY_OBJECT_PATH,
                      default_val='/tmp/query_objects', check_env_var=True)
        if not os.path.exists(self.settings['upload_dir']):
            os.makedirs(self.settings['upload_dir'])

        # set and validate transfer protocol
        set_parameter('transfer_protocol',
                      ConfigParameters.TABPY_TRANSFER_PROTOCOL,
                      default_val='http')
        self.settings['transfer_protocol'] =\
            self.settings['transfer_protocol'].lower()

        set_parameter('certificate_file',
                      ConfigParameters.TABPY_CERTIFICATE_FILE)
        set_parameter('key_file', ConfigParameters.TABPY_KEY_FILE)
        self._validate_transfer_protocol_settings()

        # if state.ini does not exist try and create it - remove
        # last dependence on batch/shell script
        set_parameter('state_file_path', ConfigParameters.TABPY_STATE_PATH,
                      default_val='./', check_env_var=True)
        self.settings['state_file_path'] = os.path.realpath(
            os.path.normpath(
                os.path.expanduser(self.settings['state_file_path'])))
        state_file_path = self.settings['state_file_path']
        logger.info("Loading state from state file %s" %
                    os.path.join(state_file_path, "state.ini"))
        tabpy_state = _get_state_from_file(state_file_path)
        self.tabpy_state = TabPyState(
            config=tabpy_state, settings=self.settings)

        self.python_service = PythonServiceHandler(PythonService())
        self.settings['compress_response'] = True if TORNADO_MAJOR >= 4\
            else "gzip"
        self.settings['static_path'] = os.path.join(
            os.path.dirname(__file__), "static")

        # Set subdirectory from config if applicable
        if tabpy_state.has_option("Service Info", "Subdirectory"):
            self.subdirectory = "/" + \
                tabpy_state.get("Service Info", "Subdirectory")

        # If passwords file specified load credentials
        set_parameter(ConfigParameters.TABPY_PWD_FILE,
                      ConfigParameters.TABPY_PWD_FILE)
        if ConfigParameters.TABPY_PWD_FILE in self.settings:
            if not self._parse_pwd_file():
                log_and_raise('Failed to read passwords file %s' %
                              self.settings[ConfigParameters.TABPY_PWD_FILE],
                              RuntimeError)
        else:
            logger.info("Password file is not specified")

        features = self._get_features()
        self.settings['versions'] = {'v1': {'features': features}}