def setUp(self):
# Create a patch so that you can configure settings without
self.patches.append(mock.patch.object(settings, '_wrapped', None))
super().setUp()
# Configure all your tests with this settings
settings.configure({})
def test_json(self):
with NamedTemporaryFile(mode='w', dir=self.temp_dir.name,
delete=False) as fid:
fid.write('{"a": 15, "b":"22", "c": true}')
@settings_property
def c(self):
return self['a']
settings.add_templates([(
{},
{
'a': fid.name,
'b_json': fid.name,
# Both json AND settings_property
'c_json': c
})])
settings.configure({})
self.assertEqual(settings.a, fid.name)
self.assertEqual(settings.b_json.a, 15)
self.assertEqual(settings.b_json.b, "22")
self.assertEqual(settings.b_json.c, True)
self.assertEqual(settings.c_json.a, 15)
self.assertEqual(settings.c_json.b, "22")
self.assertEqual(settings.c_json.c, True)
def test_replay(self):
# Swap out the stderr stream handler for this test
test_handler = logging.handlers.MemoryHandler(capacity=1000)
test_handler.setLevel(
logger._SetupTerraLogger.default_stderr_handler_level)
self._logs.root_logger.handlers = [
test_handler if h is self._logs.stderr_handler else h
for h in self._logs.root_logger.handlers
]
self._logs.stderr_handler = test_handler
test_logger = logger.getLogger(f'{__name__}.test_replay')
message1 = str(uuid.uuid4())
message2 = str(uuid.uuid4())
message3 = str(uuid.uuid4())
test_logger.error(message1)
test_logger.debug1(message2)
test_logger.debug2(message3)
self.assertEqual(str(test_handler.buffer).count(message1), 1)
self.assertEqual(str(test_handler.buffer).count(message2), 0)
self.assertEqual(str(test_handler.buffer).count(message3), 0)
settings.configure({
'processing_dir': self.temp_dir.name,
'logging': {
'level': 'debug1'
}
})
self.assertEqual(str(test_handler.buffer).count(message1), 2)
self.assertEqual(str(test_handler.buffer).count(message2), 1)
self.assertEqual(str(test_handler.buffer).count(message3), 0)
def test_executor_name_by_name(self):
settings.configure(
{'executor': {
'type': 'concurrent.futures.ProcessPoolExecutor'
}})
self.assertIsInstance(Executor._connection(),
concurrent.futures.ProcessPoolExecutor)
def test_executor_handler(self):
settings.configure({'executor': {'type': 'DummyExecutor'}})
test_executor = ExecutorHandler()
self.assertIsNotNone(test_executor._connection)
self.assertIsInstance(test_executor._connection(), DummyExecutor)
def test_undefined_key(self):
settings.configure()
with self.assertRaises(AttributeError):
settings.q
with self.assertRaises(KeyError):
settings['q']
self.assertTrue(settings.configured)
def test_executor(self):
settings.configure({'executor': {'type': 'DummyExecutor'}})
executee = Executor()
self.assertIsNotNone(Executor._connection)
self.assertIsInstance(Executor._connection(), DummyExecutor)
self.assertIsInstance(executee, DummyExecutor)
def test_properties_processing_dir_default_nonwritable(
self, mock_access, mock_mkdtemp):
mock_access.return_value = False # mock cwd as never writable
mock_dir = "/tmp/mock"
mock_mkdtemp.return_value = mock_dir
settings.configure({})
with self.assertLogs(), settings:
self.assertEqual(settings.processing_dir, mock_dir)
def test_add_templates_domino(self):
import terra.core.settings as s
self.assertEqual(s.global_templates, [({}, {'a': 11, 'b': 22})])
settings.add_templates([({'a': 11}, {'c': 33})])
settings.configure()
self.assertEqual(settings.a, 11)
self.assertEqual(settings.b, 22)
self.assertNotIn("c", settings)
def test_executor_name_celery(self):
try:
import terra.executor.celery
except ImportError:
raise SkipTest('Celery does not appear to be installed')
settings.configure({'executor': {'type': 'CeleryExecutor'}})
self.assertIsInstance(Executor._connection(),
terra.executor.celery.CeleryExecutor)
def test_expandvars(self):
with EnvironmentContext(FOO="NOTBAR"):
settings.configure({
'test1': 'this${FOO}that',
'test2': 'a${GKLDGSJLGKJSGURNAONV}b'
})
with EnvironmentContext(FOO="BAR"):
self.assertEqual(settings.test1, 'thisBARthat')
self.assertEqual(settings.test2, 'a${GKLDGSJLGKJSGURNAONV}b')
def test_volumes_and_configuration_map(self):
# Add a volumes
service = base.BaseService()
service.add_volume("/local", "/remote")
# Test configuration_map
settings.configure({})
# Make sure the volume is in the map
self.assertEqual([("/local", "/remote")],
base.BaseCompute().configuration_map(service))
def test_properties_processing_dir_nonexisting_config_file(self):
settings.configure({})
def mock_mkdtemp(prefix):
return f'"{prefix}"'
with mock.patch.object(tempfile, 'mkdtemp', mock_mkdtemp), \
self.assertLogs(), settings:
settings.config_file = '/land/of/foo.bar'
self.assertEqual(settings.processing_dir, '"terra_"')
def test_compute_handler_short_name(self):
# Since the "docker" library is installed, this test actually imports
# docker before importing the real docker Compute, hence the warning.
settings.configure({'compute': {'arch': 'docker'}})
# Suppress imp warnings caused by docker
with warnings.catch_warnings():
warnings.simplefilter('ignore')
# Make sure it worked
self.assertIsInstance(utils.ComputeHandler()._connection,
terra.compute.docker.Compute)
def test_properties_status_file(self):
settings.configure({})
with settings:
if os.name == "nt":
settings.processing_dir = 'C:\\foobar'
ans = 'C:\\foobar\\status.json'
else:
settings.processing_dir = '/foobar'
ans = '/foobar/status.json'
self.assertEqual(settings.status_file, ans)
def test_configure(self):
self.assertFalse(settings.configured)
settings.configure(b="333", c=444)
with self.assertRaises(ImproperlyConfigured):
settings.configure()
self.assertTrue(settings.configured)
self.assertEqual(settings.a, 11)
self.assertEqual(settings.b, "333")
self.assertEqual(settings.c, 444)
def test_expandvars_once(self):
settings.configure({'test2': 'a${GKLDGSJLGKJSGURNAONV}b'})
# Evaluate it here once
self.assertNotIsInstance(settings._wrapped['test2'], ExpandedString)
self.assertEqual(settings.test2, 'a${GKLDGSJLGKJSGURNAONV}b')
self.assertIsInstance(settings._wrapped['test2'], ExpandedString)
with EnvironmentContext(GKLDGSJLGKJSGURNAONV="FOO"):
# Show it is not evaluated again here
self.assertEqual(settings.test2, 'a${GKLDGSJLGKJSGURNAONV}b')
def test_level(self):
settings.configure({
'processing_dir': self.temp_dir.name,
'logging': {
'level': 'DEBUG1'
}
})
self.assertEqual(settings.logging.level, "DEBUG1")
self.assertEqual(self._logs.root_logger.level, logger.NOTSET)
self.assertEqual(self._logs.stderr_handler.level, logger.DEBUG1)
def test_expanduser_once(self):
settings.configure({
'test_dir': ExpandedString('~/foo'),
'test_file': '~/bar'
})
self.assertNotIsInstance(settings._wrapped['test_file'],
ExpandedString)
self.assertEqual(settings.test_file, os.path.expanduser('~/bar'))
self.assertIsInstance(settings._wrapped['test_file'], ExpandedString)
self.assertEqual(settings.test_dir, '~/foo')
def test_level_case_insensitive(self):
with self.assertLogs(level=logger.DEBUG2):
settings.configure({
'processing_dir': self.temp_dir.name,
'logging': {
'level': 'debug2'
}
})
self.assertEqual(settings.logging.level, "debug2")
self.assertEqual(self._logs.root_logger.level, logger.NOTSET)
self.assertEqual(self._logs.stderr_handler.level, logger.DEBUG2)
def setUp(self):
# Use setting
self.patches.append(mock.patch.object(settings, '_wrapped', None))
# For registering a service
self.patches.append(
mock.patch.dict(terra.compute.base.services, clear=True))
super().setUp()
# Register this Compute
settings.configure({'compute': {'arch': Compute.__module__}})
# Manually register without using decorator
Compute.register(Service)(Service_test)
# Manually register against base compute. Normally this is never done, but
# done for some core case tests below
terra.compute.base.BaseCompute.register(Service2)(Service2_test)
def test_properties_unittest(self):
settings.configure({})
with settings, EnvironmentContext(TERRA_UNITTEST="1"):
self.assertTrue(settings.unittest)
with settings, EnvironmentContext(TERRA_UNITTEST="0"):
self.assertFalse(settings.unittest)
# Test when unset
with settings, EnvironmentContext(TERRA_UNITTEST='1'):
os.environ.pop('TERRA_UNITTEST')
self.assertFalse(settings.unittest)
# Make sure I didn't break anything
self.assertEqual(os.environ['TERRA_UNITTEST'], '1')
def main():
if env.get('TERRA_SETTINGS_FILE', '') == '':
settings.configure({
'executor': {
'type': 'CeleryExecutor'
},
'terra': {
'zone': 'task_controller'
},
'logging': {
'level': 'NOTSET'
},
'processing_dir':
tempfile.mkdtemp(prefix="terra_celery_"),
})
app.start()
def test_formatter(self):
settings.configure({
'processing_dir': self.temp_dir.name,
'logging': {
'format': 'foo {asctime} {msg}',
'date_format': 'bar',
'style': '{'
}
})
# This doesn't get formatted
# with self.assertLogs(__name__, logger.ERROR) as cm:
# logger.getLogger(__name__).error('Hi')
record = logging.LogRecord(__name__, logger.ERROR, __file__, 0, "Hiya",
(), None)
self.assertEqual(self._logs.stderr_handler.format(record),
"foo bar Hiya")
def setUp(self):
# Use settings
self.patches.append(mock.patch.object(settings, '_wrapped', None))
# This will resets the _connection to an uninitialized state
self.patches.append(
mock.patch.object(terra.compute.utils.ComputeHandler,
'_connection',
mock.PropertyMock(return_value=docker.Compute())))
# patches.append(mock.patch.dict(base.services, clear=True))
super().setUp()
# Configure for docker
settings.configure({
'compute': {'arch': 'docker'},
'processing_dir': self.temp_dir.name,
'test_dir': '/opt/projects/terra/terra_dsm/external/terra/foo'})
def test_run(self):
settings.configure({})
with self.assertLogs(dummy.__name__, level="INFO") as cm:
self.dummyCompute.run(self.test_service_name)
# Find which log messages state the specific steps of processing
run = ['INFO:terra.compute.dummy:Run:' in o for o in cm.output].index(True)
pre_run = ['INFO:terra.compute.dummy:Pre run:' in o
for o in cm.output].index(True)
create = ['INFO:terra.compute.dummy:Create:' in o
for o in cm.output].index(True)
start = ['INFO:terra.compute.dummy:Start:' in o
for o in cm.output].index(True)
post_run = ['INFO:terra.compute.dummy:Post run:' in o
for o in cm.output].index(True)
# Make sure everything ran in order
self.assertLess(pre_run, run)
self.assertLess(run, create)
self.assertLess(create, start)
self.assertLess(start, post_run)
def test_add_templates_order(self):
import terra.core.settings as s
settings.add_templates([({'a': 11}, {'c': 33})])
# Demonstrate one of the possible complications/solutions.
settings.add_templates([({}, {'a': 11})])
self.assertEqual(s.global_templates, [({}, {
'a': 11
}), ({
'a': 11
}, {
'c': 33
}), ({}, {
'a': 11,
'b': 22
})])
settings.configure()
self.assertEqual(settings.a, 11)
self.assertEqual(settings.b, 22)
self.assertIn("c", settings)
self.assertEqual(settings.c, 33)
def test_settings_property(self):
import terra.core.settings
@settings_property
def a(self):
return self.c.e + 1
def ce(self):
return self.c.b - 2
def d(self):
return 3
terra.core.settings.global_templates[0][1].update({
'a': a,
'c': {
'e': settings_property(ce),
'b': settings_property(lambda self: 13.1),
},
'd': d
})
self.assertFalse(settings.configured)
settings.configure()
# Not cached
self.assertNotEqual(settings._wrapped['a'], 12.1)
self.assertEqual(settings.a, 12.1)
# Verify cached
self.assertEqual(settings._wrapped['a'], 12.1)
self.assertEqual(settings.c.e, 11.1)
self.assertEqual(settings.c.b, 13.1)
# A non settings_property function is just a function
self.assertEqual(settings.d, d)
self.assertEqual(settings.d(None), 3)
def test_nested_in(self):
settings.configure({
'a': 11,
'b': 22,
'q': {
'x': 33,
'y': 44,
'foo': {
't': 15
}
}
})
self.assertIn('a', settings)
self.assertIn('b', settings)
self.assertIn('q', settings)
self.assertNotIn('x', settings)
self.assertNotIn('y', settings)
self.assertNotIn('foo', settings)
self.assertNotIn('t', settings)
self.assertIn('q.x', settings)
self.assertIn('q.y', settings)
self.assertIn('q.foo', settings)
self.assertNotIn('q.t', settings)
self.assertIn('q.foo.t', settings)
def test_executor_name_thread(self):
settings.configure({'executor': {'type': 'ThreadPoolExecutor'}})
self.assertIsInstance(Executor._connection(),
concurrent.futures.ThreadPoolExecutor)
