def test_json_serializer(self):
@settings_property
def c(self):
return self.a + self.b
with self.assertRaises(ImproperlyConfigured):
TerraJSONEncoder.dumps(settings)
settings._wrapped = Settings({'a': 11, 'b': 22, 'c': c})
j = json.loads(TerraJSONEncoder.dumps(settings))
self.assertEqual(j['a'], 11)
self.assertEqual(j['b'], 22)
self.assertEqual(j['c'], 33)
def apply_async(self,
args=None,
kwargs=None,
task_id=None,
*args2,
**kwargs2):
current_settings = TerraJSONEncoder.serializableSettings(settings)
return super().apply_async(args=args,
kwargs=kwargs,
headers={'settings': current_settings},
task_id=task_id,
*args2,
**kwargs2)
def translate_paths(self, payload, reverse_compute_volume_map,
executor_volume_map):
if reverse_compute_volume_map or executor_volume_map:
# If either translation is needed, start by applying the ~ home dir
# expansion and settings_property (which wouldn't have made it through
# pure json conversion, but the ~ will)
payload = TerraJSONEncoder.serializableSettings(payload)
# Go from compute runner to master controller
if reverse_compute_volume_map:
payload = terra.compute.utils.translate_settings_paths(
payload, reverse_compute_volume_map)
# Go from master controller to executor
if executor_volume_map:
payload = terra.compute.utils.translate_settings_paths(
payload, executor_volume_map)
return payload
def pre_run(self):
"""
"""
# Create a temp directory, store it in this instance
self.temp_dir = TemporaryDirectory()
# Use a config.json file to store settings within that temp directory
temp_config_file = os.path.join(self.temp_dir.name, 'config.json')
# Serialize config file
docker_config = TerraJSONEncoder.serializableSettings(settings)
# Dump the serialized config to the temp config file
with open(temp_config_file, 'w') as fid:
json.dump(docker_config, fid)
# Set the Terra settings file for this service runner to the temp config
# file
self.env['TERRA_SETTINGS_FILE'] = temp_config_file
def test_nested_json_serializer(self):
@settings_property
def c(self):
return self.a + self.b
settings._wrapped = Settings({
'a': 11,
'b': 22,
'q': {
'x': c,
'y': c,
'foo': {
't': [c]
}
}
})
j = json.loads(TerraJSONEncoder.dumps(settings))
self.assertEqual(j['a'], 11)
self.assertEqual(j['b'], 22)
self.assertEqual(j['q']['x'], 33)
self.assertEqual(j['q']['y'], 33)
self.assertEqual(j['q']['foo']['t'][0], 33)
def pre_run(self):
super().pre_run()
# Create a temp directory, store it in this instance
self.temp_dir = TemporaryDirectory(suffix=f"_{type(self).__name__}")
if self.env.get('TERRA_KEEP_TEMP_DIR', None) == "1":
self.temp_dir._finalizer.detach()
# Use a config.json file to store settings within that temp directory
temp_config_file = os.path.join(self.temp_dir.name, 'config.json')
# Serialize config file
venv_config = TerraJSONEncoder.serializableSettings(settings)
# Dump the serialized config to the temp config file
venv_config['terra']['zone'] = 'runner'
with open(temp_config_file, 'w') as fid:
json.dump(venv_config, fid)
# Set the Terra settings file for this service runner to the temp config
# file
self.env['TERRA_SETTINGS_FILE'] = temp_config_file
def pre_run(self):
self.temp_dir = TemporaryDirectory()
temp_dir = pathlib.Path(self.temp_dir.name)
# Check to see if and are already defined, this will play nicely with
# external influences
env_volume_index = 1
while f'TERRA_VOLUME_{env_volume_index}' in self.env:
env_volume_index += 1
# Setup volumes for docker
self.env[f'TERRA_VOLUME_{env_volume_index}'] = \
f'{str(temp_dir)}:/tmp_settings:rw'
env_volume_index += 1
# Copy self.volumes to the environment variables
for index, ((volume_host, volume_container), volume_flags) in \
enumerate(zip(self.volumes, self.volumes_flags)):
volume_str = f'{volume_host}:{volume_container}'
if volume_flags:
volume_str += f':{volume_flags}'
self.env[f'TERRA_VOLUME_{env_volume_index}'] = volume_str
env_volume_index += 1
# volume_map = compute.configuration_map(self, [str(temp_compose_file)])
volume_map = compute.configuration_map(self)
logger.debug3("Volume map: %s", volume_map)
# Setup config file for docker
docker_config = TerraJSONEncoder.serializableSettings(settings)
self.env['TERRA_SETTINGS_FILE'] = '/tmp_settings/config.json'
if os.name == "nt":
logger.warning("Windows volume mapping is experimental.")
# Prevent the setting file name from being expanded.
self.env['TERRA_AUTO_ESCAPE'] = self.env['TERRA_AUTO_ESCAPE'] \
+ '|TERRA_SETTINGS_FILE'
def patch_volume(value, volume_map):
value_path = pathlib.PureWindowsPath(ntpath.normpath(value))
for vol_from, vol_to in volume_map:
vol_from = pathlib.PureWindowsPath(
ntpath.normpath(vol_from))
if isinstance(value, str):
try:
remainder = value_path.relative_to(vol_from)
except ValueError:
continue
if self.container_platform == "windows":
value = pathlib.PureWindowsPath(vol_to)
else:
value = pathlib.PurePosixPath(vol_to)
value /= remainder
return str(value)
return value
else:
def patch_volume(value, volume_map):
for vol_from, vol_to in volume_map:
if isinstance(value, str) and value.startswith(vol_from):
return value.replace(vol_from, vol_to, 1)
return value
# Apply map translation to settings configuration
docker_config = nested_patch(
docker_config, lambda key, value: (isinstance(key, str) and any(
key.endswith(pattern) for pattern in filename_suffixes)),
lambda key, value: patch_volume(value, reversed(volume_map)))
# Dump the settings
with open(temp_dir / 'config.json', 'w') as fid:
json.dump(docker_config, fid)
def pre_run(self):
# Need to run Base's pre_run first, so it has a chance to update settings
# for special executors, etc...
super().pre_run()
self.temp_dir = TemporaryDirectory(suffix=f"_{type(self).__name__}")
if self.env.get('TERRA_KEEP_TEMP_DIR', None) == "1":
self.temp_dir._finalizer.detach()
temp_dir = pathlib.Path(self.temp_dir.name)
# Check to see if and are already defined, this will play nicely with
# external influences
env_volume_index = 1
while f'{self.env["JUST_PROJECT_PREFIX"]}_VOLUME_{env_volume_index}' in \
self.env:
env_volume_index += 1
# Setup volumes for container
self.env[f'{self.env["JUST_PROJECT_PREFIX"]}_'
f'VOLUME_{env_volume_index}'] = \
f'{str(temp_dir)}:/tmp_settings'
env_volume_index += 1
if os.environ.get('TERRA_DISABLE_SETTINGS_DUMP') != '1':
os.makedirs(settings.settings_dir, exist_ok=True)
self.env[f'{self.env["JUST_PROJECT_PREFIX"]}_'
f'VOLUME_{env_volume_index}'] = \
f'{settings.settings_dir}:/settings'
env_volume_index += 1
# Copy self.volumes to the environment variables
for _, ((volume_host, volume_container), volume_flags) in \
enumerate(zip(self.volumes, self.volumes_flags)):
volume_str = f'{volume_host}:{volume_container}'
if volume_flags:
volume_str += f':{volume_flags}'
self.env[f'{self.env["JUST_PROJECT_PREFIX"]}_'
f'VOLUME_{env_volume_index}'] = \
volume_str
env_volume_index += 1
settings.compute.volume_map = compute.configuration_map(self)
logger.debug4("Compute Volume map: %s", settings.compute.volume_map)
# Setup config file for container
self.env['TERRA_SETTINGS_FILE'] = '/tmp_settings/config.json'
container_config = translate_settings_paths(
TerraJSONEncoder.serializableSettings(settings),
settings.compute.volume_map,
self.container_platform)
if os.name == "nt": # pragma: no linux cover
# logger.warning("Windows volume mapping is experimental.")
# Prevent the setting file name from being expanded.
self.env['TERRA_AUTO_ESCAPE'] = self.env['TERRA_AUTO_ESCAPE'] \
+ '|TERRA_SETTINGS_FILE'
# Dump the settings
container_config['terra']['zone'] = 'runner'
with open(temp_dir / 'config.json', 'w') as fid:
json.dump(container_config, fid)
def configure_logger(self, sender=None, signal=None, **kwargs):
'''
Call back function to configure the logger after settings have been
configured
'''
from terra import settings
from terra.core.settings import TerraJSONEncoder
if self._configured:
self.root_logger.error("Configure logger called twice, this is "
"unexpected")
raise ImproperlyConfigured()
# This sends a signal to the current Executor type, which has already been
# imported at the end of LazySettings.configure. We don't import Executor
# here to reduce the concerns of this module
import terra.core.signals
terra.core.signals.logger_configure.send(sender=self, **kwargs)
self.set_level_and_formatter()
# Now that the real logger has been set up, swap some handlers
self.root_logger.removeHandler(self.preconfig_stderr_handler)
self.root_logger.removeHandler(self.preconfig_main_log_handler)
self.root_logger.removeHandler(self.tmp_handler)
if os.environ.get('TERRA_DISABLE_SETTINGS_DUMP') != '1':
os.makedirs(settings.settings_dir, exist_ok=True)
settings_dump = os.path.join(
settings.settings_dir,
datetime.now(timezone.utc).strftime(
f'settings_{settings.terra.uuid}_%Y_%m_%d_%H_%M_%S_%f.json'
))
with open(settings_dump, 'w') as fid:
fid.write(TerraJSONEncoder.dumps(settings, indent=2))
# filter the stderr buffer
self.preconfig_stderr_handler.buffer = \
[x for x in self.preconfig_stderr_handler.buffer
if (x.levelno >= self.stderr_handler.level)]
# Use this if statement if you want to prevent repeating any critical/error
# level messages. This is probably not necessary because error/critical
# messages before configure should be rare, and are probably worth
# repeating. Repeating is the only way to get them formatted right the
# second time anyways. This applies to stderr only, not the log file
# if (x.levelno >= level)] and
# (x.levelno < default_stderr_handler_level)]
# Filter file buffer. Never remove default_stderr_handler_level message,
# they won't be in the new output file
self.preconfig_main_log_handler.buffer = \
[x for x in self.preconfig_main_log_handler.buffer
if (x.levelno >= self.main_log_handler.level)]
# Flush the buffers
self.preconfig_stderr_handler.setTarget(self.stderr_handler)
self.preconfig_stderr_handler.flush()
self.preconfig_stderr_handler = None
self.preconfig_main_log_handler.setTarget(self.main_log_handler)
self.preconfig_main_log_handler.flush()
self.preconfig_main_log_handler = None
self.tmp_handler = None
# Remove the temporary file now that you are done with it
self.tmp_file.close()
if os.path.exists(
self.tmp_file.name) and self.tmp_file.name != os.devnull:
os.unlink(self.tmp_file.name)
self.tmp_file = None
self._configured = True