def test_AMQPException(self):
ping = self.inspect().ping
ping.side_effect = Exception(0, 'fake error', '')
mon = CeleryNodeMonitor(no_distribute=False, interval=1)
with mock.patch('openquake.engine.logs.LOG') as log:
mon.ping(timeout=0.1)
self.assertTrue(log.warn.called)
def test_AMQPException(self):
ping = self.inspect().ping
ping.side_effect = Exception(0, 'fake error', '')
mon = CeleryNodeMonitor(no_distribute=False, interval=1)
with mock.patch('openquake.engine.logs.LOG') as log:
mon.ping(timeout=0.1)
self.assertTrue(log.warn.called)
def test_all_nodes_were_down(self):
ping = self.inspect().ping
ping.return_value = {}
mon = CeleryNodeMonitor(no_distribute=False, interval=1)
with self.assertRaises(SystemExit), mock.patch('sys.stderr') as stderr:
mon.__enter__()
self.assertEqual(ping.call_count, 1) # called only once
self.assertTrue(stderr.write.called) # an error message was printed
def test_all_nodes_were_down(self):
ping = self.inspect().ping
ping.return_value = {}
mon = CeleryNodeMonitor(no_distribute=False, interval=1)
with self.assertRaises(SystemExit), mock.patch('sys.stderr') as stderr:
mon.__enter__()
self.assertEqual(ping.call_count, 1) # called only once
self.assertTrue(stderr.write.called) # an error message was printed
def run_job_lite(cfg_files, log_level, log_file, exports=''):
"""
Run a job using the specified config file and other options.
:param str cfg_files:
Path to calculation config (INI-style) files.
:param str log_level:
'debug', 'info', 'warn', 'error', or 'critical'
:param str log_file:
Path to log file.
:param exports:
A comma-separated string of export types requested by the user.
Currently only 'xml' is supported.
"""
# first of all check the database version and exit if the db is outdated
upgrader.check_versions(django_db.connections['admin'])
with CeleryNodeMonitor(openquake.engine.no_distribute(), interval=3):
job = job_from_files(cfg_files, getpass.getuser(), log_level, exports)
t0 = time.time()
run_calc(job, log_level, log_file, exports, lite=True)
duration = time.time() - t0
if job.status == 'complete':
print_results(job.id, duration, list_outputs)
else:
sys.exit('Calculation %s failed' % job.id)
return job
def test_all_nodes_are_up(self):
ping = self.inspect().ping
ping.return_value = {'node1': []}
mon = CeleryNodeMonitor(no_distribute=False, interval=1)
with mon:
time.sleep(1.1)
# one ping was done in the thread, plus one at the beginning
self.assertEqual(ping.call_count, 2)
def test_one_node_went_down(self):
ping = self.inspect().ping
ping.return_value = {'node1': []}
mon = CeleryNodeMonitor(no_distribute=False, interval=1)
with mon, mock.patch('os.kill') as kill, \
mock.patch('openquake.engine.logs.LOG') as log:
time.sleep(1.1)
ping.return_value = {}
time.sleep(1)
# two pings was done in the thread, plus 1 at the beginning
self.assertEqual(ping.call_count, 3)
# check that kill was called with a SIGABRT
pid, signum = kill.call_args[0]
self.assertEqual(pid, os.getpid())
self.assertEqual(signum, signal.SIGABRT)
self.assertTrue(log.critical.called)
def test_no_distribute(self):
with CeleryNodeMonitor(no_distribute=True, interval=0.1):
time.sleep(0.5)
self.assertIsNone(self.inspect.call_args)