def _setup(self, name=None):
"""
Load the config json file pointed to by the environment variable. This is
used the first time settings are needed, if the user hasn't configured
settings manually.
Arguments
---------
name : :class:`str`, optional
The name used to describe the settings object. Defaults to ``settings``
Raises
------
ImproperlyConfigured
If the settings has already been configured, will throw an error. Under
normal circumstances, :func:`_setup` will not be called a second time.
"""
settings_file = os.environ.get(ENVIRONMENT_VARIABLE)
if not settings_file:
desc = ("setting %s" % name) if name else "settings"
raise ImproperlyConfigured(
"Requested %s, but settings are not configured. "
"You must either define the environment variable %s "
"or call settings.configure() before accessing settings." %
(desc, ENVIRONMENT_VARIABLE))
with open(settings_file) as fid:
self.configure(json.load(fid))
self._wrapped.config_file = os.environ.get(ENVIRONMENT_VARIABLE)
def default(self, obj):
if isinstance(obj, LazySettings):
if obj._wrapped is None:
raise ImproperlyConfigured('Settings not initialized')
return TerraJSONEncoder.serializableSettings(obj._wrapped)
# elif isinstance(obj, datetime):
# return str(obj)
return JSONEncoder.default(self, obj) # pragma: no cover
def configure(self, *args, **kwargs):
"""
Called to manually configure the settings. The 'default_settings'
parameter sets where to retrieve any unspecified values from (its
argument should be a :class:`dict`).
Arguments
---------
*args :
Passed along to :class:`Settings`
**kwargs :
Passed along to :class:`Settings`
Raises
------
ImproperlyConfigured
If settings is already configured, will throw this exception
"""
if self._wrapped is not None:
raise ImproperlyConfigured('Settings already configured.')
logger.debug2('Pre settings configure')
self._wrapped = Settings(*args, **kwargs)
for pattern, settings in global_templates:
if nested_in_dict(pattern, self._wrapped):
# Not the most efficient way to do this, but insignificant "preupdate"
d = {}
nested_update(d, settings)
nested_update(d, self._wrapped)
# Nested update and run patch code
self._wrapped.update(d)
def read_json(json_file):
# In case json_file is an @settings_property function
if getattr(json_file, 'settings_property', None):
json_file = json_file(settings)
return Settings(json_load(json_file))
nested_patch_inplace(
self._wrapped, lambda key, value:
(isinstance(key, str) and (isinstance(value, str) or getattr(
value, 'settings_property', False)) and
any(key.endswith(pattern) for pattern in json_include_suffixes)),
lambda key, value: read_json(value))
# Importing these here is intentional, it guarantees the signals are
# connected so that executor and computes can setup logging if need be
import terra.executor # noqa
import terra.compute # noqa
from terra.core.signals import post_settings_configured
post_settings_configured.send(sender=self)
logger.debug2('Post settings configure')
def configure_logger(self, sender, **kwargs):
'''
Call back function to configure the logger after settings have been
configured
'''
from terra import settings
if self._configured:
self.root_logger.error("Configure logger called twice, this is "
"unexpected")
raise ImproperlyConfigured()
formatter = logging.Formatter(fmt=settings.logging.format,
datefmt=settings.logging.date_format,
style=settings.logging.style)
# Setup log file for use in configure
self.log_file = os.path.join(settings.processing_dir,
self.default_log_prefix)
os.makedirs(settings.processing_dir, exist_ok=True)
self.log_file = open(self.log_file, 'a')
self.file_handler = logging.StreamHandler(stream=self.log_file)
# Configure log level
level = settings.logging.level
if isinstance(level, str):
# make level case insensitive
level = level.upper()
self.stderr_handler.setLevel(level)
self.file_handler.setLevel(level)
# Configure format
self.file_handler.setFormatter(formatter)
self.stderr_handler.setFormatter(formatter)
# Swap some handlers
self.root_logger.addHandler(self.file_handler)
self.root_logger.removeHandler(self.preconfig_stderr_handler)
self.root_logger.removeHandler(self.preconfig_file_handler)
self.root_logger.removeHandler(self.tmp_handler)
# Log the settings only to the file handler
with HandlerLoggingContext(self.root_logger, [self.file_handler]):
self.root_logger.log(DEBUG1,
"Settings:\n" + pprint.pformat(dict(settings)),
extra=extra_logger_variables)
# For some reason python doesn't make the root logger the designated
# class, so much add extra manually here. Not even sure why I chose
# root_logger here...
# filter the stderr buffer
self.preconfig_stderr_handler.buffer = \
[x for x in self.preconfig_stderr_handler.buffer
if (x.levelno >= self.stderr_handler.level)]
# Use this if statement if you want to prevent repeating any critical/error
# level messages. This is probably not necessary because error/critical
# messages before configure should be rare, and are probably worth
# repeating. Repeating is the only way to get them formatted right the
# second time anyways. This applys to stderr only, not the log file
# if (x.levelno >= level)] and
# (x.levelno < default_stderr_handler_level)]
# Filter file buffer. Never remove default_stderr_handler_level message,
# they won't be in the new output file
self.preconfig_file_handler.buffer = \
[x for x in self.preconfig_file_handler.buffer
if (x.levelno >= self.file_handler.level)]
# Flush the buffers
self.preconfig_stderr_handler.setTarget(self.stderr_handler)
self.preconfig_stderr_handler.flush()
self.preconfig_stderr_handler = None
self.preconfig_file_handler.setTarget(self.file_handler)
self.preconfig_file_handler.flush()
self.preconfig_file_handler = None
self.tmp_handler = None
# Remove the temporary file now that you are done with it
self.tmp_file.close()
os.unlink(self.tmp_file.name)
self.tmp_file = None
self._configured = True
def configure_logger(self, sender=None, signal=None, **kwargs):
'''
Call back function to configure the logger after settings have been
configured
'''
from terra import settings
from terra.core.settings import TerraJSONEncoder
if self._configured:
self.root_logger.error("Configure logger called twice, this is "
"unexpected")
raise ImproperlyConfigured()
# This sends a signal to the current Executor type, which has already been
# imported at the end of LazySettings.configure. We don't import Executor
# here to reduce the concerns of this module
import terra.core.signals
terra.core.signals.logger_configure.send(sender=self, **kwargs)
self.set_level_and_formatter()
# Now that the real logger has been set up, swap some handlers
self.root_logger.removeHandler(self.preconfig_stderr_handler)
self.root_logger.removeHandler(self.preconfig_main_log_handler)
self.root_logger.removeHandler(self.tmp_handler)
if os.environ.get('TERRA_DISABLE_SETTINGS_DUMP') != '1':
os.makedirs(settings.settings_dir, exist_ok=True)
settings_dump = os.path.join(
settings.settings_dir,
datetime.now(timezone.utc).strftime(
f'settings_{settings.terra.uuid}_%Y_%m_%d_%H_%M_%S_%f.json'
))
with open(settings_dump, 'w') as fid:
fid.write(TerraJSONEncoder.dumps(settings, indent=2))
# filter the stderr buffer
self.preconfig_stderr_handler.buffer = \
[x for x in self.preconfig_stderr_handler.buffer
if (x.levelno >= self.stderr_handler.level)]
# Use this if statement if you want to prevent repeating any critical/error
# level messages. This is probably not necessary because error/critical
# messages before configure should be rare, and are probably worth
# repeating. Repeating is the only way to get them formatted right the
# second time anyways. This applies to stderr only, not the log file
# if (x.levelno >= level)] and
# (x.levelno < default_stderr_handler_level)]
# Filter file buffer. Never remove default_stderr_handler_level message,
# they won't be in the new output file
self.preconfig_main_log_handler.buffer = \
[x for x in self.preconfig_main_log_handler.buffer
if (x.levelno >= self.main_log_handler.level)]
# Flush the buffers
self.preconfig_stderr_handler.setTarget(self.stderr_handler)
self.preconfig_stderr_handler.flush()
self.preconfig_stderr_handler = None
self.preconfig_main_log_handler.setTarget(self.main_log_handler)
self.preconfig_main_log_handler.flush()
self.preconfig_main_log_handler = None
self.tmp_handler = None
# Remove the temporary file now that you are done with it
self.tmp_file.close()
if os.path.exists(
self.tmp_file.name) and self.tmp_file.name != os.devnull:
os.unlink(self.tmp_file.name)
self.tmp_file = None
self._configured = True