def test_timeout_in_running_job(self):
"""Test timeout in running job."""
def wait(job):
del job
time.sleep(0.1)
self.fake_plugin.side_effect = wait
self.expand.return_value = {
"workers": [{
"fun": self.fake_plugin,
"timeout": 0.05
}]
}
with pytest.raises(TimeoutError,
match="Timeout for .* expired "
"after 0.1 seconds"):
process("msg", "prod_list", self.queue)
# wait a little to ensure alarm is not raised later
time.sleep(0.11)
def test_crash_handler_call(self):
"""Test crash hander call.
This will raise KeyError as there are no configured workers in the config returned by expand().
"""
self.traceback.format_exc.return_value = 'baz'
crash_handlers = {
"crash_handlers": {
"config": {
"foo": "bar"
},
"handlers": [{
"fun": self.sendmail
}]
}
}
self.expand.return_value = crash_handlers
with pytest.raises(KeyError):
process("msg", "prod_list", self.queue)
config = crash_handlers['crash_handlers']['config']
self.sendmail.assert_called_once_with(config, 'baz')
def test_workers_initialized():
"""Test that the config loading works when workers are defined."""
from tempfile import NamedTemporaryFile
import os
queue = mock.MagicMock()
with NamedTemporaryFile(mode='w+t', delete=False) as tmp_file:
fname = tmp_file.name
tmp_file.write(yaml_test_minimal)
tmp_file.close()
try:
with mock.patch("trollflow2.launcher.get_dask_client") as gdc:
# `get_dask_client()` is called just after config reading, so if we get there loading worked
gdc.side_effect = StopIteration
try:
process("msg", fname, queue)
except StopIteration:
pass
finally:
os.remove(fname)
def test_open_bad_yaml(self):
"""Test failure in yaml.load(), e.g. bad formatting."""
self.open.side_effect = YAMLError
with pytest.raises(YAMLError):
process("msg", "prod_list", self.queue)
def test_open_missing_file(self):
"""Test failure in open() due to a missing config file."""
self.open.side_effect = IOError
with pytest.raises(IOError):
process("msg", "prod_list", self.queue)
def test_plugin_with_no_stop_work(self):
"""Test that plugins with no `stop` method (like regular functions) can be used."""
self.fake_plugin.stop = mock.MagicMock(
side_effect=AttributeError('boo'))
process("msg", "prod_list", self.queue)
def test_error_propagation(self):
"""Test that errors are propagated."""
self.fake_plugin.side_effect = KeyboardInterrupt
with pytest.raises(KeyboardInterrupt):
process("msg", "prod_list", self.queue)
def test_dask_client_is_closed(self):
"""Test that the dask client is closed."""
process("msg", "prod_list", self.queue)
self.client.close.assert_called_once()
def test_yaml_config_is_read_only_once(self):
"""Test that the yaml config is read only once."""
process("msg", "prod_list", self.queue)
self.yaml.load.assert_called_once()
def test_product_list_is_opened(self):
"""Test product list is opened."""
process("msg", "prod_list", self.queue)
self.open.assert_called_with("prod_list")
def test_plugin_is_stopped_after_processing(self):
"""Test plugin is stopped after processing."""
self.fake_plugin.stop.assert_not_called()
process("msg", "prod_list", self.queue)
self.fake_plugin.stop.assert_called_once()
def test_process(self):
"""Test subprocessing."""
from trollflow2.launcher import process
with mock.patch('trollflow2.launcher.traceback') as traceback,\
mock.patch('trollflow2.launcher.sendmail') as sendmail,\
mock.patch('trollflow2.launcher.expand') as expand,\
mock.patch('trollflow2.launcher.yaml') as yaml_,\
mock.patch('trollflow2.launcher.message_to_jobs') as message_to_jobs,\
mock.patch('trollflow2.launcher.open') as open_:
fid = mock.MagicMock()
fid.read.return_value = yaml_test1
open_.return_value.__enter__.return_value = fid
mock_config = mock.MagicMock()
yaml_.load.return_value = mock_config
yaml_.YAMLError = yaml.YAMLError
fun1 = mock.MagicMock()
# Return something resembling a config
expand.return_value = {"workers": [{"fun": fun1}]}
message_to_jobs.return_value = {1: {"job1": dict([])}}
the_queue = mock.MagicMock()
process("msg", "prod_list", the_queue)
open_.assert_called_with("prod_list")
yaml_.load.assert_called_once()
message_to_jobs.assert_called_with("msg",
{"workers": [{
"fun": fun1
}]})
fun1.assert_called_with({
'job1': {},
'processing_priority': 1,
'produced_files': the_queue
})
# Test that errors are propagated
fun1.side_effect = KeyboardInterrupt
with self.assertRaises(KeyboardInterrupt):
process("msg", "prod_list", the_queue)
# Test crash hander call. This will raise KeyError as there
# are no configured workers in the config returned by expand()
traceback.format_exc.return_value = 'baz'
crash_handlers = {
"crash_handlers": {
"config": {
"foo": "bar"
},
"handlers": [{
"fun": sendmail
}]
}
}
expand.return_value = crash_handlers
with self.assertRaises(KeyError):
process("msg", "prod_list", the_queue)
config = crash_handlers['crash_handlers']['config']
sendmail.assert_called_once_with(config, 'baz')
# Test failure in open(), e.g. a missing file
open_.side_effect = IOError
with self.assertRaises(IOError):
process("msg", "prod_list", the_queue)
# Test failure in yaml.load(), e.g. bad formatting
open_.side_effect = yaml.YAMLError
with self.assertRaises(yaml.YAMLError):
process("msg", "prod_list", the_queue)