def test_service_simple(self):
# Must not be a decorator, because at decorator time (before setUp is run),
# settings._wrapped is still None. Mock the version from setUpModule so I
# change the values without affecting any other test
with mock.patch.dict(settings._wrapped, {}):
compute = docker.Compute()
compute.configuration_map(SomeService())
# Test setting for translation
settings.foo_dir = "/foo"
settings.bar_dir = "/not_foo"
service = SomeService()
# Simple case
self.common(compute, service)
def test_service_other_dir_methods(self):
compute = docker.Compute()
compute.configuration_map(SomeService())
# Test setting for translation
settings.foo_dir = "/foo"
settings.bar_dir = "/not_foo"
# Run same tests with a TERRA_VOLUME externally set
service = SomeService()
service.add_volume('/test1', '/test2', 'z')
service.env['TERRA_VOLUME_1'] = "/Foo:/Bar"
self.common(compute, service)
# Make sure this is still set correctly
self.assertEqual(service.env['TERRA_VOLUME_1'], "/Foo:/Bar")
self.assertIn('/test1:/test2:z',
(v for k, v in service.env.items()
if k.startswith('TERRA_VOLUME_')),
'Added volume failed to be bound')
def test_config_non_existing_service(self):
compute = docker.Compute()
service = TestDockerMap.Service()
with warnings.catch_warnings():
warnings.simplefilter('ignore')
# USe the default name, foo, which doesn't even exist
volume_map = compute.configuration_map(service)
# Should be empty
self.assertEqual(volume_map, [])
def test_config_test_service(self):
compute = docker.Compute()
service = TestDockerMap.Service()
service.compose_service_name = "test"
volume_map = compute.configuration_map(service)
ans = [('/tmp', '/bar'),
('/opt/projects/terra/terra_dsm/external/terra', '/terra'),
('/tmp/.X11-unix', '/tmp/.X11-unix'),
('/opt/projects/terra/terra_dsm/external/terra/external/vsi_common',
'/vsi')]
self.assertEqual(volume_map, ans)
def pre_run(self):
self.temp_dir = TemporaryDirectory()
temp_dir = pathlib.Path(self.temp_dir.name)
# Check to see if and are already defined, this will play nicely with
# external influences
env_volume_index = 1
while f'TERRA_VOLUME_{env_volume_index}' in self.env:
env_volume_index += 1
# Setup volumes for docker
self.env[f'TERRA_VOLUME_{env_volume_index}'] = \
f'{str(temp_dir)}:/tmp_settings:rw'
env_volume_index += 1
# Copy self.volumes to the environment variables
for index, ((volume_host, volume_container), volume_flags) in \
enumerate(zip(self.volumes, self.volumes_flags)):
volume_str = f'{volume_host}:{volume_container}'
if volume_flags:
volume_str += f':{volume_flags}'
self.env[f'TERRA_VOLUME_{env_volume_index}'] = volume_str
env_volume_index += 1
# volume_map = compute.configuration_map(self, [str(temp_compose_file)])
volume_map = compute.configuration_map(self)
logger.debug3("Volume map: %s", volume_map)
# Setup config file for docker
docker_config = TerraJSONEncoder.serializableSettings(settings)
self.env['TERRA_SETTINGS_FILE'] = '/tmp_settings/config.json'
if os.name == "nt":
logger.warning("Windows volume mapping is experimental.")
# Prevent the setting file name from being expanded.
self.env['TERRA_AUTO_ESCAPE'] = self.env['TERRA_AUTO_ESCAPE'] \
+ '|TERRA_SETTINGS_FILE'
def patch_volume(value, volume_map):
value_path = pathlib.PureWindowsPath(ntpath.normpath(value))
for vol_from, vol_to in volume_map:
vol_from = pathlib.PureWindowsPath(
ntpath.normpath(vol_from))
if isinstance(value, str):
try:
remainder = value_path.relative_to(vol_from)
except ValueError:
continue
if self.container_platform == "windows":
value = pathlib.PureWindowsPath(vol_to)
else:
value = pathlib.PurePosixPath(vol_to)
value /= remainder
return str(value)
return value
else:
def patch_volume(value, volume_map):
for vol_from, vol_to in volume_map:
if isinstance(value, str) and value.startswith(vol_from):
return value.replace(vol_from, vol_to, 1)
return value
# Apply map translation to settings configuration
docker_config = nested_patch(
docker_config, lambda key, value: (isinstance(key, str) and any(
key.endswith(pattern) for pattern in filename_suffixes)),
lambda key, value: patch_volume(value, reversed(volume_map)))
# Dump the settings
with open(temp_dir / 'config.json', 'w') as fid:
json.dump(docker_config, fid)
def pre_run(self):
# Need to run Base's pre_run first, so it has a chance to update settings
# for special executors, etc...
super().pre_run()
self.temp_dir = TemporaryDirectory(suffix=f"_{type(self).__name__}")
if self.env.get('TERRA_KEEP_TEMP_DIR', None) == "1":
self.temp_dir._finalizer.detach()
temp_dir = pathlib.Path(self.temp_dir.name)
# Check to see if and are already defined, this will play nicely with
# external influences
env_volume_index = 1
while f'{self.env["JUST_PROJECT_PREFIX"]}_VOLUME_{env_volume_index}' in \
self.env:
env_volume_index += 1
# Setup volumes for container
self.env[f'{self.env["JUST_PROJECT_PREFIX"]}_'
f'VOLUME_{env_volume_index}'] = \
f'{str(temp_dir)}:/tmp_settings'
env_volume_index += 1
if os.environ.get('TERRA_DISABLE_SETTINGS_DUMP') != '1':
os.makedirs(settings.settings_dir, exist_ok=True)
self.env[f'{self.env["JUST_PROJECT_PREFIX"]}_'
f'VOLUME_{env_volume_index}'] = \
f'{settings.settings_dir}:/settings'
env_volume_index += 1
# Copy self.volumes to the environment variables
for _, ((volume_host, volume_container), volume_flags) in \
enumerate(zip(self.volumes, self.volumes_flags)):
volume_str = f'{volume_host}:{volume_container}'
if volume_flags:
volume_str += f':{volume_flags}'
self.env[f'{self.env["JUST_PROJECT_PREFIX"]}_'
f'VOLUME_{env_volume_index}'] = \
volume_str
env_volume_index += 1
settings.compute.volume_map = compute.configuration_map(self)
logger.debug4("Compute Volume map: %s", settings.compute.volume_map)
# Setup config file for container
self.env['TERRA_SETTINGS_FILE'] = '/tmp_settings/config.json'
container_config = translate_settings_paths(
TerraJSONEncoder.serializableSettings(settings),
settings.compute.volume_map,
self.container_platform)
if os.name == "nt": # pragma: no linux cover
# logger.warning("Windows volume mapping is experimental.")
# Prevent the setting file name from being expanded.
self.env['TERRA_AUTO_ESCAPE'] = self.env['TERRA_AUTO_ESCAPE'] \
+ '|TERRA_SETTINGS_FILE'
# Dump the settings
container_config['terra']['zone'] = 'runner'
with open(temp_dir / 'config.json', 'w') as fid:
json.dump(container_config, fid)