def on_state_change(settings,
tabpy_state,
python_service,
logger=logging.getLogger(__name__)):
try:
logger.log(logging.INFO, "Loading state from state file")
config = util._get_state_from_file(
settings[SettingsParameters.StateFilePath], logger=logger)
new_ps_state = TabPyState(config=config, settings=settings)
(has_changes,
changes) = _get_latest_service_state(settings, tabpy_state,
new_ps_state, python_service)
if not has_changes:
logger.info("Nothing changed, return.")
return
new_endpoints = new_ps_state.get_endpoints()
for object_name in changes['endpoints']:
(object_type, object_version,
object_path) = changes['endpoints'][object_name]
if not object_path and not object_version: # removal
logger.info(f'Removing object: URI={object_name}')
python_service.manage_request(DeleteObjects([object_name]))
cleanup_endpoint_files(object_name,
settings[SettingsParameters.UploadDir],
logger=logger)
else:
endpoint_info = new_endpoints[object_name]
is_update = object_version > 1
if object_type == 'alias':
msg = LoadObject(object_name, endpoint_info['target'],
object_version, is_update, 'alias')
else:
local_path = object_path
msg = LoadObject(object_name, local_path, object_version,
is_update, object_type)
python_service.manage_request(msg)
wait_for_endpoint_loaded(python_service, object_name)
# cleanup old version of endpoint files
if object_version > 2:
cleanup_endpoint_files(
object_name,
settings[SettingsParameters.UploadDir],
logger=logger,
retain_versions=[object_version, object_version - 1])
except Exception as e:
err_msg = format_exception(e, 'on_state_change')
logger.log(logging.ERROR,
f'Error submitting update model request: error={err_msg}')
def on_state_change(settings):
try:
py_handler = settings['py_handler']
logger.info("Loading state from state file")
config = util._get_state_from_file(settings['state_file_path'])
new_ps_state = TabPyState(config=config, settings=settings)
(has_changes,
changes) = _get_latest_service_state(settings, new_ps_state)
if not has_changes:
logger.info("Nothing changed, return.")
return
new_endpoints = new_ps_state.get_endpoints()
for object_name in changes['endpoints']:
(object_type, object_version,
object_path) = changes['endpoints'][object_name]
if not object_path and not object_version: # removal
logger.info("Removing object: URI={}".format(object_name))
py_handler.manage_request(DeleteObjects([object_name]))
cleanup_endpoint_files(object_name, settings['upload_dir'])
else:
endpoint_info = new_endpoints[object_name]
is_update = object_version > 1
if object_type == 'alias':
msg = LoadObject(object_name, endpoint_info['target'],
object_version, is_update, 'alias')
else:
local_path = object_path
msg = LoadObject(object_name, local_path, object_version,
is_update, object_type)
py_handler.manage_request(msg)
wait_for_endpoint_loaded(py_handler, object_name)
# cleanup old version of endpoint files
if object_version > 2:
cleanup_endpoint_files(
object_name, settings['upload_dir'],
[object_version, object_version - 1])
except Exception as e:
err_msg = format_exception(e, 'on_state_change')
logger.error(
"Error submitting update model request: error={}".format(err_msg))
def get_config():
"""Provide consistent mechanism for pulling in configuration.
Attempt to retain backward compatibility for existing implementations by
grabbing port setting from CLI first.
Take settings in the following order:
1. CLI arguments, if present - port only - may be able to deprecate
2. common.config file, and
3. OS environment variables (for ease of setting defaults if not present)
4. current defaults if a setting is not present in any location
Additionally provide similar configuration capabilities in between
common.config and environment variables.
For consistency use the same variable name in the config file as in the os
environment.
For naming standards use all capitals and start with 'TABPY_'
"""
try:
import tabpy_server.common.config as config
except ImportError:
config = None
settings = {}
cli_args = parse_arguments()
if cli_args.port is not None:
settings['port'] = cli_args.port
else:
try:
settings['port'] = config.TABPY_PORT
except AttributeError:
settings['port'] = os.getenv('TABPY_PORT', 9004)
try:
settings['server_version'] = config.TABPY_SERVER_VERSION
except AttributeError:
settings['server_version'] = os.getenv('TABPY_SERVER_VERSION', 'Alpha')
try:
settings['bind_ip'] = config.TABPY_BIND_IP
except AttributeError:
settings['bind_ip'] = os.getenv('TABPY_BIND_IP', '0.0.0.0')
try:
settings['upload_dir'] = config.TABPY_QUERY_OBJECT_PATH
except AttributeError:
settings['upload_dir'] = os.getenv('TABPY_QUERY_OBJECT_PATH',
'/tmp/query_objects')
if not os.path.exists(settings['upload_dir']):
os.makedirs(settings['upload_dir'])
try:
_state_file_path = config.TABPY_STATE_PATH
except AttributeError:
_state_file_path = os.getenv('TABPY_STATE_PATH', './')
settings['state_file_path'] = os.path.realpath(
os.path.normpath(os.path.expanduser(_state_file_path)))
# if state.ini does not exist try and create it - remove last dependence
# on batch/shell script
if not os.path.isfile('{}/state.ini'.format(settings['state_file_path'])):
shutil.copy('./state.ini.template',
'{}/state.ini'.format(settings['state_file_path']))
log_info("Loading state from state file")
tabpy_state = _get_state_from_file(settings['state_file_path'])
settings['tabpy'] = TabPyState(config=tabpy_state)
settings['py_handler'] = PythonServiceHandler(PythonService())
settings['compress_response'] = True if TORNADO_MAJOR >= 4 else "gzip"
settings['static_path'] = os.path.join(os.path.dirname(__file__), "static")
# Set subdirectory from config if applicable
subdirectory = ""
if tabpy_state.has_option("Service Info", "Subdirectory"):
subdirectory = "/" + tabpy_state.get("Service Info", "Subdirectory")
return settings, subdirectory
def _parse_config(self, config_file):
"""Provide consistent mechanism for pulling in configuration.
Attempt to retain backward compatibility for
existing implementations by grabbing port
setting from CLI first.
Take settings in the following order:
1. CLI arguments if present
2. config file
3. OS environment variables (for ease of
setting defaults if not present)
4. current defaults if a setting is not present in any location
Additionally provide similar configuration capabilities in between
config file and environment variables.
For consistency use the same variable name in the config file as
in the os environment.
For naming standards use all capitals and start with 'TABPY_'
"""
self.settings = {}
self.subdirectory = ""
self.tabpy_state = None
self.python_service = None
self.credentials = {}
parser = configparser.ConfigParser()
if os.path.isfile(config_file):
with open(config_file) as f:
parser.read_string(f.read())
else:
logger.warning(f'Unable to find config file at {config_file}, '
'using default settings.')
def set_parameter(settings_key,
config_key,
default_val=None,
check_env_var=False):
if config_key is not None and\
parser.has_section('TabPy') and\
parser.has_option('TabPy', config_key):
self.settings[settings_key] = parser.get('TabPy', config_key)
logger.debug(f'Parameter {settings_key} set to '
f'"{self.settings[settings_key]}" '
'from config file')
elif check_env_var:
self.settings[settings_key] = os.getenv(
config_key, default_val)
logger.debug(f'Parameter {settings_key} set to '
f'"{self.settings[settings_key]}" '
'from environment variable')
elif default_val is not None:
self.settings[settings_key] = default_val
logger.debug(f'Parameter {settings_key} set to '
f'"{self.settings[settings_key]}" '
'from default value')
else:
logger.debug(f'Parameter {settings_key} is not set')
set_parameter(SettingsParameters.Port,
ConfigParameters.TABPY_PORT,
default_val=9004,
check_env_var=True)
set_parameter(SettingsParameters.ServerVersion,
None,
default_val=__version__)
set_parameter(SettingsParameters.EvaluateTimeout,
ConfigParameters.TABPY_EVALUATE_TIMEOUT,
default_val=30)
try:
self.settings[SettingsParameters.EvaluateTimeout] = float(
self.settings[SettingsParameters.EvaluateTimeout])
except ValueError:
logger.warning('Evaluate timeout must be a float type. Defaulting '
'to evaluate timeout of 30 seconds.')
self.settings[SettingsParameters.EvaluateTimeout] = 30
set_parameter(SettingsParameters.UploadDir,
ConfigParameters.TABPY_QUERY_OBJECT_PATH,
default_val='/tmp/query_objects',
check_env_var=True)
if not os.path.exists(self.settings[SettingsParameters.UploadDir]):
os.makedirs(self.settings[SettingsParameters.UploadDir])
# set and validate transfer protocol
set_parameter(SettingsParameters.TransferProtocol,
ConfigParameters.TABPY_TRANSFER_PROTOCOL,
default_val='http')
self.settings[SettingsParameters.TransferProtocol] =\
self.settings[SettingsParameters.TransferProtocol].lower()
set_parameter(SettingsParameters.CertificateFile,
ConfigParameters.TABPY_CERTIFICATE_FILE)
set_parameter(SettingsParameters.KeyFile,
ConfigParameters.TABPY_KEY_FILE)
self._validate_transfer_protocol_settings()
# if state.ini does not exist try and create it - remove
# last dependence on batch/shell script
set_parameter(SettingsParameters.StateFilePath,
ConfigParameters.TABPY_STATE_PATH,
default_val='./tabpy-server/tabpy_server',
check_env_var=True)
self.settings[SettingsParameters.StateFilePath] = os.path.realpath(
os.path.normpath(
os.path.expanduser(
self.settings[SettingsParameters.StateFilePath])))
state_file_path = self.settings[SettingsParameters.StateFilePath]
logger.info('Loading state from state file '
f'{os.path.join(state_file_path, "state.ini")}')
tabpy_state = _get_state_from_file(state_file_path)
self.tabpy_state = TabPyState(config=tabpy_state,
settings=self.settings)
self.python_service = PythonServiceHandler(PythonService())
self.settings['compress_response'] = True
set_parameter(SettingsParameters.StaticPath,
ConfigParameters.TABPY_STATIC_PATH,
default_val='./')
self.settings[SettingsParameters.StaticPath] =\
os.path.abspath(self.settings[SettingsParameters.StaticPath])
logger.debug(f'Static pages folder set to '
f'"{self.settings[SettingsParameters.StaticPath]}"')
# Set subdirectory from config if applicable
if tabpy_state.has_option("Service Info", "Subdirectory"):
self.subdirectory = "/" + \
tabpy_state.get("Service Info", "Subdirectory")
# If passwords file specified load credentials
set_parameter(ConfigParameters.TABPY_PWD_FILE,
ConfigParameters.TABPY_PWD_FILE)
if ConfigParameters.TABPY_PWD_FILE in self.settings:
if not self._parse_pwd_file():
msg = ('Failed to read passwords file '
f'{self.settings[ConfigParameters.TABPY_PWD_FILE]}')
logger.critical(msg)
raise RuntimeError(msg)
else:
logger.info("Password file is not specified: "
"Authentication is not enabled")
features = self._get_features()
self.settings[SettingsParameters.ApiVersions] =\
{'v1': {'features': features}}
set_parameter(SettingsParameters.LogRequestContext,
ConfigParameters.TABPY_LOG_DETAILS,
default_val='false')
self.settings[SettingsParameters.LogRequestContext] = (
self.settings[SettingsParameters.LogRequestContext].lower() !=
'false')
call_context_state =\
'enabled' if self.settings[SettingsParameters.LogRequestContext]\
else 'disabled'
logger.info(f'Call context logging is {call_context_state}')
def get_config(config_file):
"""Provide consistent mechanism for pulling in configuration.
Attempt to retain backward compatibility for existing implementations by
grabbing port setting from CLI first.
Take settings in the following order:
1. CLI arguments, if present - port only - may be able to deprecate
2. common.config file, and
3. OS environment variables (for ease of setting defaults if not present)
4. current defaults if a setting is not present in any location
Additionally provide similar configuration capabilities in between
common.config and environment variables.
For consistency use the same variable name in the config file as in the os
environment.
For naming standards use all capitals and start with 'TABPY_'
"""
parser = configparser.ConfigParser()
if os.path.isfile(config_file):
with open(config_file) as f:
parser.read_string(f.read())
else:
logger.warning(
"Unable to find config file at '{}', using default settings.".
format(config_file))
settings = {}
for section in parser.sections():
if section == "TabPy":
for key, val in parser.items(section):
settings[key] = val
break
def set_parameter(settings_key,
config_key,
default_val=None,
check_env_var=False):
if config_key is not None and parser.has_option('TabPy', config_key):
settings[settings_key] = parser.get('TabPy', config_key)
elif check_env_var:
settings[settings_key] = os.getenv(config_key, default_val)
elif default_val is not None:
settings[settings_key] = default_val
if cli_args is not None and cli_args.port is not None:
settings['port'] = cli_args.port
else:
set_parameter('port',
'TABPY_PORT',
default_val=9004,
check_env_var=True)
try:
settings['port'] = int(settings['port'])
except ValueError:
logger.warning('Error during config validation, invalid port: {}. '
'Using default port 9004'.format(settings['port']))
settings['port'] = 9004
set_parameter('server_version', None, default_val=__version__)
set_parameter('bind_ip',
'TABPY_BIND_IP',
default_val='0.0.0.0',
check_env_var=True)
set_parameter('upload_dir',
'TABPY_QUERY_OBJECT_PATH',
default_val='/tmp/query_objects',
check_env_var=True)
if not os.path.exists(settings['upload_dir']):
os.makedirs(settings['upload_dir'])
set_parameter('state_file_path',
'TABPY_STATE_PATH',
default_val='./',
check_env_var=True)
settings['state_file_path'] = os.path.realpath(
os.path.normpath(os.path.expanduser(settings['state_file_path'])))
# set and validate transfer protocol
set_parameter('transfer_protocol',
'TABPY_TRANSFER_PROTOCOL',
default_val='http')
settings['transfer_protocol'] = settings['transfer_protocol'].lower()
set_parameter('certificate_file', 'TABPY_CERTIFICATE_FILE')
set_parameter('key_file', 'TABPY_KEY_FILE')
validate_transfer_protocol_settings(settings)
# if state.ini does not exist try and create it - remove last dependence
# on batch/shell script
state_file_path = settings['state_file_path']
logger.info("Loading state from state file {}".format(
os.path.join(state_file_path, "state.ini")))
tabpy_state = _get_state_from_file(state_file_path)
settings['tabpy'] = TabPyState(config=tabpy_state, settings=settings)
settings['py_handler'] = PythonServiceHandler(PythonService())
settings['compress_response'] = True if TORNADO_MAJOR >= 4 else "gzip"
settings['static_path'] = os.path.join(os.path.dirname(__file__), "static")
# Set subdirectory from config if applicable
subdirectory = ""
if tabpy_state.has_option("Service Info", "Subdirectory"):
subdirectory = "/" + tabpy_state.get("Service Info", "Subdirectory")
return settings, subdirectory
def get_config():
"""Provide consistent mechanism for pulling in configuration.
Attempt to retain backward compatibility for existing implementations by
grabbing port setting from CLI first.
Take settings in the following order:
1. CLI arguments, if present - port only - may be able to deprecate
2. common.config file, and
3. OS environment variables (for ease of setting defaults if not present)
4. current defaults if a setting is not present in any location
Additionally provide similar configuration capabilities in between
common.config and environment variables.
For consistency use the same variable name in the config file as in the os
environment.
For naming standards use all capitals and start with 'TABPY_'
"""
parser = configparser.ConfigParser()
cli_args = parse_arguments()
path = cli_args.config if cli_args.config is not None else os.path.join(
os.path.dirname(__file__), 'common', 'default.conf')
if os.path.isfile(path):
with open(path) as f:
data = '[dummy-header]\n' + f.read()
parser.read_string(data)
else:
logger.warning(
"Unable to find config file at '{}', using default settings.".
format(path))
settings = {}
for section in parser.sections():
for key, val in parser.items(section):
settings[key] = val
def set_parameter(settings_key,
config_key,
default_val=None,
check_env_var=False):
if config_key is not None and parser.has_option(
'dummy-header', config_key):
settings[settings_key] = parser.get('dummy-header', config_key)
elif check_env_var:
settings[settings_key] = os.getenv(config_key, default_val)
elif default_val is not None:
settings[settings_key] = default_val
# convert log level string into int, set log level
# defaults to INFO if the level is not recognized
set_parameter('log_level',
'TABPY_LOG_LEVEL',
default_val='INFO',
check_env_var=True)
settings['log_level'] = settings['log_level'].upper()
level = settings['log_level']
if level not in {'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'}:
logger.warning('The log level indicated is not supported: {}. '
'Will be using the default level INFO.'.format(level))
level = 'INFO'
settings['log_level'] = level
# Create application wide logging
root_logger = logging.getLogger('')
root_logger.setLevel(level)
temp_dir = tempfile.gettempdir()
fh = logging.handlers.RotatingFileHandler(filename=os.path.join(
temp_dir, "tabpy_log.log"),
maxBytes=10000000,
backupCount=5)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
root_logger.addHandler(fh)
if cli_args.port is not None:
settings['port'] = cli_args.port
else:
set_parameter('port',
'TABPY_PORT',
default_val=9004,
check_env_var=True)
try:
settings['port'] = int(settings['port'])
except ValueError:
logger.warning('Error during config validation, invalid port: {}. '
'Using default port 9004'.format(settings['port']))
settings['port'] = 9004
set_parameter('server_version', None, default_val=__version__)
set_parameter('bind_ip',
'TABPY_BIND_IP',
default_val='0.0.0.0',
check_env_var=True)
set_parameter('upload_dir',
'TABPY_QUERY_OBJECT_PATH',
default_val='/tmp/query_objects',
check_env_var=True)
if not os.path.exists(settings['upload_dir']):
os.makedirs(settings['upload_dir'])
set_parameter('state_file_path',
'TABPY_STATE_PATH',
default_val='./',
check_env_var=True)
settings['state_file_path'] = os.path.realpath(
os.path.normpath(os.path.expanduser(settings['state_file_path'])))
# set and validate transfer protocol
set_parameter('transfer_protocol',
'TABPY_TRANSFER_PROTOCOL',
default_val='http')
settings['transfer_protocol'] = settings['transfer_protocol'].lower()
set_parameter('certificate_file', 'TABPY_CERTIFICATE_FILE')
set_parameter('key_file', 'TABPY_KEY_FILE')
validate_transfer_protocol_settings(settings)
# if state.ini does not exist try and create it - remove last dependence
# on batch/shell script
state_file_path = settings['state_file_path']
logger.info(
"Loading state from state file {}/state.ini".format(state_file_path))
tabpy_state = _get_state_from_file(state_file_path)
settings['tabpy'] = TabPyState(config=tabpy_state, settings=settings)
settings['py_handler'] = PythonServiceHandler(PythonService())
settings['compress_response'] = True if TORNADO_MAJOR >= 4 else "gzip"
settings['static_path'] = os.path.join(os.path.dirname(__file__), "static")
# Set subdirectory from config if applicable
subdirectory = ""
if tabpy_state.has_option("Service Info", "Subdirectory"):
subdirectory = "/" + tabpy_state.get("Service Info", "Subdirectory")
return settings, subdirectory
def _parse_config(self, config_file):
"""Provide consistent mechanism for pulling in configuration.
Attempt to retain backward compatibility for
existing implementations by grabbing port
setting from CLI first.
Take settings in the following order:
1. CLI arguments if present
2. config file
3. OS environment variables (for ease of
setting defaults if not present)
4. current defaults if a setting is not present in any location
Additionally provide similar configuration capabilities in between
config file and environment variables.
For consistency use the same variable name in the config file as
in the os environment.
For naming standards use all capitals and start with 'TABPY_'
"""
self.settings = {}
self.subdirectory = ""
self.tabpy_state = None
self.python_service = None
self.credentials = {}
parser = configparser.ConfigParser()
if os.path.isfile(config_file):
with open(config_file) as f:
parser.read_string(f.read())
else:
logger.warning(
"Unable to find config file at '{}', "
"using default settings.".format(config_file))
def set_parameter(settings_key,
config_key,
default_val=None,
check_env_var=False):
if config_key is not None and\
parser.has_option('TabPy', config_key):
self.settings[settings_key] = parser.get('TabPy', config_key)
elif check_env_var:
self.settings[settings_key] = os.getenv(
config_key, default_val)
elif default_val is not None:
self.settings[settings_key] = default_val
set_parameter('port', ConfigParameters.TABPY_PORT,
default_val=9004, check_env_var=True)
set_parameter('server_version', None, default_val=__version__)
set_parameter('upload_dir', ConfigParameters.TABPY_QUERY_OBJECT_PATH,
default_val='/tmp/query_objects', check_env_var=True)
if not os.path.exists(self.settings['upload_dir']):
os.makedirs(self.settings['upload_dir'])
# set and validate transfer protocol
set_parameter('transfer_protocol',
ConfigParameters.TABPY_TRANSFER_PROTOCOL,
default_val='http')
self.settings['transfer_protocol'] =\
self.settings['transfer_protocol'].lower()
set_parameter('certificate_file',
ConfigParameters.TABPY_CERTIFICATE_FILE)
set_parameter('key_file', ConfigParameters.TABPY_KEY_FILE)
self._validate_transfer_protocol_settings()
# if state.ini does not exist try and create it - remove
# last dependence on batch/shell script
set_parameter('state_file_path', ConfigParameters.TABPY_STATE_PATH,
default_val='./', check_env_var=True)
self.settings['state_file_path'] = os.path.realpath(
os.path.normpath(
os.path.expanduser(self.settings['state_file_path'])))
state_file_path = self.settings['state_file_path']
logger.info("Loading state from state file %s" %
os.path.join(state_file_path, "state.ini"))
tabpy_state = _get_state_from_file(state_file_path)
self.tabpy_state = TabPyState(
config=tabpy_state, settings=self.settings)
self.python_service = PythonServiceHandler(PythonService())
self.settings['compress_response'] = True if TORNADO_MAJOR >= 4\
else "gzip"
self.settings['static_path'] = os.path.join(
os.path.dirname(__file__), "static")
# Set subdirectory from config if applicable
if tabpy_state.has_option("Service Info", "Subdirectory"):
self.subdirectory = "/" + \
tabpy_state.get("Service Info", "Subdirectory")
# If passwords file specified load credentials
set_parameter(ConfigParameters.TABPY_PWD_FILE,
ConfigParameters.TABPY_PWD_FILE)
if ConfigParameters.TABPY_PWD_FILE in self.settings:
if not self._parse_pwd_file():
log_and_raise('Failed to read passwords file %s' %
self.settings[ConfigParameters.TABPY_PWD_FILE],
RuntimeError)
else:
logger.info("Password file is not specified")
features = self._get_features()
self.settings['versions'] = {'v1': {'features': features}}