def store_hazard_logic_trees(a_job):
"""Helper function to store the source model and GMPE logic trees in the
KVS so that it can be read by the Java code. This is basically what the
@preload decorator does.
:param a_job:
:class:`openquake.job.Job` instance.
"""
lt_proc = LogicTreeProcessor(
a_job['BASE_PATH'],
a_job['SOURCE_MODEL_LOGIC_TREE_FILE_PATH'],
a_job['GMPE_LOGIC_TREE_FILE_PATH'])
src_model_seed = a_job['SOURCE_MODEL_LT_RANDOM_SEED']
gmpe_seed = a_job['GMPE_LT_RANDOM_SEED']
src_model_rnd = random.Random()
src_model_rnd.seed(src_model_seed)
gmpe_rnd = random.Random()
gmpe_rnd.seed(gmpe_seed)
store_source_model(a_job.job_id, src_model_rnd.getrandbits(32),
a_job.params, lt_proc)
store_gmpe_map(a_job.job_id, gmpe_rnd.getrandbits(32), lt_proc)
def distribute_disagg(the_job, sites, realizations, poes, result_dir):
"""Compute disaggregation by splitting up the calculation over sites,
realizations, and PoE values.
:param the_job:
Job definition
:type the_job:
:class:`openquake.job.Job` instance
:param sites:
List of :class:`openquake.shapes.Site` objects
:param poes:
Probability of Exceedence levels for the calculation
:type poes:
List of floats
:param result_dir:
Path where full disaggregation results should be stored
:returns:
Result data in the following form::
[(realization_1, poe_1,
[(site_1, gmv_1, matrix_path_1),
(site_2, gmv_2, matrix_path_2)]
),
(realization_1, poe_2,
[(site_1, gmv_1, matrix_path_3),
(site_2, gmv_2, matrix_path_4)]
),
...
(realization_N, poe_N,
[(site_1, gmv_1, matrix_path_N-1),
(site_2, gmv_2, matrix_path_N)]
),
]
A single matrix result in this form looks like this::
[(1, 0.1,
[(Site(0.0, 0.0), 0.2257,
'/var/lib/openquake/disagg-results/job-372/some_guid.h5'),]
),
]
"""
# accumulates the final results of this method
full_da_results = []
# accumulates task data across the realization and poe loops
task_data = []
src_model_rnd = random.Random()
src_model_rnd.seed(the_job['SOURCE_MODEL_LT_RANDOM_SEED'])
gmpe_rnd = random.Random()
gmpe_rnd.seed(the_job['GMPE_LT_RANDOM_SEED'])
for rlz in xrange(1, realizations + 1): # 1 to N, inclusive
# cache the source model and gmpe model in the KVS
# so the Java code can access it
store_source_model(the_job.job_id, src_model_rnd.getrandbits(32),
the_job.params, the_job.calc)
store_gmpe_map(the_job.job_id, gmpe_rnd.getrandbits(32),
the_job.calc)
for poe in poes:
task_site_pairs = []
for site in sites:
a_task = compute_disagg_matrix_task.delay(
the_job.job_id, site, rlz, poe, result_dir)
task_site_pairs.append((a_task, site))
task_data.append((rlz, poe, task_site_pairs))
for rlz, poe, task_site_pairs in task_data:
# accumulates all data for a given (realization, poe) pair
rlz_poe_data = []
for a_task, site in task_site_pairs:
a_task.wait()
if not a_task.successful():
msg = (
"Full Disaggregation matrix computation task"
" for job %s with task_id=%s, realization=%s, PoE=%s,"
" site=%s has failed with the following error: %s")
msg %= (
the_job.job_id, a_task.task_id, rlz, poe, site,
a_task.result)
LOG.critical(msg)
raise RuntimeError(msg)
else:
gmv, matrix_path = a_task.result
rlz_poe_data.append((site, gmv, matrix_path))
full_da_results.append((rlz, poe, rlz_poe_data))
return full_da_results