def test_exchange_and_conn_args(self):
expected_conn_args = {
'password': '******', 'hostname': 'localhost', 'userid': 'guest',
'virtual_host': '/',
}
exchange, conn_args = base.exchange_and_conn_args()
self.assertEqual('oq.tasks', exchange.name)
self.assertEqual('direct', exchange.type)
self.assertEqual(expected_conn_args, conn_args)
def test_exchange_and_conn_args(self):
expected_conn_args = {
'password': '******',
'hostname': 'localhost',
'userid': 'guest',
'virtual_host': '/',
}
exchange, conn_args = base.exchange_and_conn_args()
self.assertEqual('oq.tasks', exchange.name)
self.assertEqual('direct', exchange.type)
self.assertEqual(expected_conn_args, conn_args)
def test_hazard_curves_task(self):
# Test the `hazard_curves` task, but execute it as a normal function
# (for purposes of test coverage).
hc = self.job.hazard_calculation
self.calc.pre_execute()
# Update job status to move on to the execution phase.
self.job.is_running = True
self.job.status = 'executing'
self.job.save()
src_prog = models.SourceProgress.objects.filter(
is_complete=False,
lt_realization__hazard_calculation=hc).latest('id')
src_id = src_prog.parsed_source.id
lt_rlz = src_prog.lt_realization
exchange, conn_args = base.exchange_and_conn_args()
routing_key = base.ROUTING_KEY_FMT % dict(job_id=self.job.id)
task_signal_queue = kombu.Queue('tasks.job.%s' % self.job.id,
exchange=exchange,
routing_key=routing_key,
durable=False,
auto_delete=True)
def test_callback(body, message):
self.assertEqual(dict(job_id=self.job.id, num_items=1), body)
message.ack()
with kombu.BrokerConnection(**conn_args) as conn:
task_signal_queue(conn.channel()).declare()
with conn.Consumer(task_signal_queue, callbacks=[test_callback]):
# call the task as a normal function
core.hazard_curves(
self.job.id, [src_id], lt_rlz.id,
logictree.LogicTreeProcessor.from_hc(
self.job.hazard_calculation))
# wait for the completion signal
conn.drain_events()
# refresh the source_progress record and make sure it is marked as
# complete
src_prog = models.SourceProgress.objects.get(id=src_prog.id)
self.assertTrue(src_prog.is_complete)
def test_hazard_curves_task(self):
# Test the `hazard_curves` task, but execute it as a normal function
# (for purposes of test coverage).
hc = self.job.hazard_calculation
self.calc.pre_execute()
# Update job status to move on to the execution phase.
self.job.is_running = True
self.job.status = 'executing'
self.job.save()
src_prog = models.SourceProgress.objects.filter(
is_complete=False,
lt_realization__hazard_calculation=hc).latest('id')
src_id = src_prog.parsed_source.id
lt_rlz = src_prog.lt_realization
exchange, conn_args = base.exchange_and_conn_args()
routing_key = base.ROUTING_KEY_FMT % dict(job_id=self.job.id)
task_signal_queue = kombu.Queue(
'tasks.job.%s' % self.job.id, exchange=exchange,
routing_key=routing_key, durable=False, auto_delete=True)
def test_callback(body, message):
self.assertEqual(dict(job_id=self.job.id, num_items=1),
body)
message.ack()
with kombu.BrokerConnection(**conn_args) as conn:
task_signal_queue(conn.channel()).declare()
with conn.Consumer(task_signal_queue, callbacks=[test_callback]):
# call the task as a normal function
core.hazard_curves(
self.job.id, [src_id], lt_rlz.id,
logictree.LogicTreeProcessor.from_hc(
self.job.hazard_calculation))
# wait for the completion signal
conn.drain_events()
# refresh the source_progress record and make sure it is marked as
# complete
src_prog = models.SourceProgress.objects.get(id=src_prog.id)
self.assertTrue(src_prog.is_complete)
def test_signal_task_complete(self):
job_id = 7
num_sources = 10
def test_callback(body, message):
self.assertEqual(dict(job_id=job_id, num_items=num_sources), body)
message.ack()
exchange, conn_args = base.exchange_and_conn_args()
routing_key = base.ROUTING_KEY_FMT % dict(job_id=job_id)
task_signal_queue = kombu.Queue('tasks.job.%s' % job_id,
exchange=exchange,
routing_key=routing_key,
durable=False,
auto_delete=True)
with kombu.BrokerConnection(**conn_args) as conn:
task_signal_queue(conn.channel()).declare()
with conn.Consumer(task_signal_queue, callbacks=[test_callback]):
# send the signal:
base.signal_task_complete(job_id=job_id, num_items=num_sources)
conn.drain_events()
def test_signal_task_complete(self):
job_id = 7
num_sources = 10
def test_callback(body, message):
self.assertEqual(dict(job_id=job_id, num_items=num_sources),
body)
message.ack()
exchange, conn_args = base.exchange_and_conn_args()
routing_key = base.ROUTING_KEY_FMT % dict(job_id=job_id)
task_signal_queue = kombu.Queue(
'tasks.job.%s' % job_id, exchange=exchange,
routing_key=routing_key, durable=False, auto_delete=True)
with kombu.BrokerConnection(**conn_args) as conn:
task_signal_queue(conn.channel()).declare()
with conn.Consumer(task_signal_queue,
callbacks=[test_callback]):
# send the signal:
base.signal_task_complete(
job_id=job_id, num_items=num_sources)
conn.drain_events()
def test_complete_event_based_calculation_cycle(self):
# * Run `pre_execute()`.
# * Execute the `stochastic_event_sets` task as a normal function.
# * Check that the proper results (GMF, SES) were computed.
# * Finally, call `post_execute()` and verify that `complete logic
# tree` artifacts were created.
# There 4 sources in the test input model; we can test them all with 1
# task.
sources_per_task = 4
self.calc.pre_execute()
# Test the job stats:
job_stats = models.JobStats.objects.get(oq_job=self.job.id)
# num sources * num lt samples / block size (items per task):
self.assertEqual(8, job_stats.num_tasks)
self.assertEqual(121, job_stats.num_sites)
self.assertEqual(2, job_stats.num_realizations)
self.job.is_running = True
self.job.status = 'executing'
self.job.save()
hc = self.job.hazard_calculation
rlz1, rlz2 = models.LtRealization.objects.filter(
hazard_calculation=hc.id).order_by('ordinal')
task_arg_gen = self.calc.task_arg_gen(sources_per_task)
task_arg_list = list(task_arg_gen)
self.assertEqual(2, len(task_arg_list))
# Now test the completion signal messaging of the task:
def test_callback(body, message):
self.assertEqual(
dict(job_id=self.job.id, num_items=sources_per_task), body)
message.ack()
exchange, conn_args = base.exchange_and_conn_args()
routing_key = base.ROUTING_KEY_FMT % dict(job_id=self.job.id)
task_signal_queue = kombu.Queue(
'htasks.job.%s' % self.job.id, exchange=exchange,
routing_key=routing_key, durable=False, auto_delete=True)
with kombu.BrokerConnection(**conn_args) as conn:
task_signal_queue(conn.channel()).declare()
with conn.Consumer(task_signal_queue, callbacks=[test_callback]):
# call the task as a normal function
for args in task_arg_list:
core.ses_and_gmfs(*args)
# wait for the completion signal
conn.drain_events()
# Check the 'total' counter (computed by the task arg generator):
# 2 realizations * 4 sources = 8 total
self.assertEqual(8, self.calc.progress['total'])
# Now check that we saved the right number of ruptures to the DB.
ruptures1 = models.SESRupture.objects.filter(
ses__ses_collection__lt_realization=rlz1)
self.assertEqual(118, ruptures1.count())
ruptures2 = models.SESRupture.objects.filter(
ses__ses_collection__lt_realization=rlz2)
self.assertEqual(92, ruptures2.count())
# Check that we have the right number of gmf_sets.
# The correct number is (num_realizations * ses_per_logic_tree_path).
gmf_sets = models.GmfSet.objects.filter(
gmf_collection__output__oq_job=self.job.id,
complete_logic_tree_gmf=False)
# 2 realizations, 5 ses_per_logic_tree_path
self.assertEqual(10, gmf_sets.count())
for imt in hc.intensity_measure_types:
imt, sa_period, sa_damping = models.parse_imt(imt)
# Now check that we have the right number of GMFs in the DB.
for gmf_set in gmf_sets:
# For each gmf_set, we should have a number of GMF records
# equal to the numbers of sites in the calculation, _per_ IMT.
# In this case, that's 121.
gmfs = models.Gmf.objects.filter(
gmf_set=gmf_set, imt=imt, sa_period=sa_period,
sa_damping=sa_damping)
# Sanity check: make sure they all came from the same task:
task_ord = gmfs[0].result_grp_ordinal
self.assertTrue(
all(x.result_grp_ordinal == task_ord for x in gmfs))
# Expected number of ruptures:
exp_n_rups = models.SESRupture.objects.filter(
ses__ses_collection__output__oq_job=self.job.id,
ses__ordinal=gmf_set.ses_ordinal,
result_grp_ordinal=task_ord).count()
self.assertEqual(121, gmfs.count())
self.assertTrue(all(len(x.gmvs) == exp_n_rups for x in gmfs))
# TODO: At some point, we'll need to test the actual values of these
# ruptures. We'll need to collect QA test data for this.
# Check the complete logic tree SES and make sure it contains
# all of the ruptures.
complete_lt_ses = models.SES.objects.get(
ses_collection__output__oq_job=self.job.id,
ses_collection__output__output_type='complete_lt_ses',
complete_logic_tree_ses=True)
clt_ses_ruptures = models.SESRupture.objects.filter(
ses=complete_lt_ses.id)
self.assertEqual(210, clt_ses_ruptures.count())
# Test the computed `investigation_time`
# 2 lt realizations * 5 ses_per_logic_tree_path * 50.0 years
self.assertEqual(500.0, complete_lt_ses.investigation_time)
self.assertIsNone(complete_lt_ses.ordinal)
