def execute(self):
"""Loop over realizations (logic tree samples), split the geometry of
interest into blocks of sites, and distribute Celery tasks to carry out
the UHS computation.
"""
job_ctxt = self.job_ctxt
all_sites = job_ctxt.sites_to_compute()
site_block_size = config.hazard_block_size()
job_profile = job_ctxt.oq_job_profile
src_model_rnd = random.Random(job_profile.source_model_lt_random_seed)
gmpe_rnd = random.Random(job_profile.gmpe_lt_random_seed)
for rlz in xrange(job_ctxt.oq_job_profile.realizations):
# Sample the gmpe and source models:
general.store_source_model(
job_ctxt.job_id, src_model_rnd.getrandbits(32),
job_ctxt.params, self.lt_processor)
general.store_gmpe_map(
job_ctxt.job_id, gmpe_rnd.getrandbits(32), self.lt_processor)
for site_block in block_splitter(all_sites, site_block_size):
tf_args = dict(job_id=job_ctxt.job_id, realization=rlz)
num_tasks_completed = completed_task_count(job_ctxt.job_id)
ath_args = dict(job_id=job_ctxt.job_id,
num_tasks=len(site_block),
start_count=num_tasks_completed)
utils_tasks.distribute(
compute_uhs_task, ('site', site_block), tf_args=tf_args,
ath=uhs_task_handler, ath_args=ath_args)
def execute(self):
"""Loop over realizations (logic tree samples), split the geometry of
interest into blocks of sites, and distribute Celery tasks to carry out
the UHS computation.
"""
job_ctxt = self.job_ctxt
all_sites = job_ctxt.sites_to_compute()
site_block_size = config.hazard_block_size()
job_profile = job_ctxt.oq_job_profile
self.initialize_pr_data(
sites=all_sites, realizations=job_ctxt.oq_job_profile.realizations)
src_model_rnd = random.Random(job_profile.source_model_lt_random_seed)
gmpe_rnd = random.Random(job_profile.gmpe_lt_random_seed)
for rlz in xrange(job_ctxt.oq_job_profile.realizations):
# Sample the gmpe and source models:
general.store_source_model(
job_ctxt.job_id, src_model_rnd.getrandbits(32),
job_ctxt.params, self.lt_processor)
general.store_gmpe_map(
job_ctxt.job_id, gmpe_rnd.getrandbits(32), self.lt_processor)
for site_block in block_splitter(all_sites, site_block_size):
tf_args = dict(job_id=job_ctxt.job_id, realization=rlz)
num_tasks_completed = completed_task_count(job_ctxt.job_id)
ath_args = dict(job_id=job_ctxt.job_id,
num_tasks=len(site_block),
start_count=num_tasks_completed)
utils_tasks.distribute(
compute_uhs_task, ('site', site_block), tf_args=tf_args,
ath=uhs_task_handler, ath_args=ath_args)
def store_hazard_logic_trees(a_job):
"""Helper function to store the source model and GMPE logic trees in the
KVS so that it can be read by the Java code. This is basically what the
@preload decorator does.
:param a_job:
:class:`openquake.engine.JobContext` instance.
"""
lt_proc = LogicTreeProcessor(a_job['BASE_PATH'],
a_job['SOURCE_MODEL_LOGIC_TREE_FILE_PATH'],
a_job['GMPE_LOGIC_TREE_FILE_PATH'])
src_model_seed = a_job['SOURCE_MODEL_LT_RANDOM_SEED']
gmpe_seed = a_job['GMPE_LT_RANDOM_SEED']
src_model_rnd = random.Random()
src_model_rnd.seed(src_model_seed)
gmpe_rnd = random.Random()
gmpe_rnd.seed(gmpe_seed)
store_source_model(a_job.job_id, src_model_rnd.getrandbits(32),
a_job.params, lt_proc)
store_gmpe_map(a_job.job_id, gmpe_rnd.getrandbits(32), lt_proc)
def store_hazard_logic_trees(a_job):
"""Helper function to store the source model and GMPE logic trees in the
KVS so that it can be read by the Java code. This is basically what the
@preload decorator does.
:param a_job:
:class:`openquake.engine.JobContext` instance.
"""
lt_proc = LogicTreeProcessor(
a_job['BASE_PATH'],
a_job['SOURCE_MODEL_LOGIC_TREE_FILE_PATH'],
a_job['GMPE_LOGIC_TREE_FILE_PATH'])
src_model_seed = a_job['SOURCE_MODEL_LT_RANDOM_SEED']
gmpe_seed = a_job['GMPE_LT_RANDOM_SEED']
src_model_rnd = random.Random()
src_model_rnd.seed(src_model_seed)
gmpe_rnd = random.Random()
gmpe_rnd.seed(gmpe_seed)
store_source_model(a_job.job_id, src_model_rnd.getrandbits(32),
a_job.params, lt_proc)
store_gmpe_map(a_job.job_id, gmpe_rnd.getrandbits(32), lt_proc)
def distribute_disagg(self, sites, realizations, poes, result_dir):
"""Compute disaggregation by splitting up the calculation over sites,
realizations, and PoE values.
:param the_job:
JobContext definition
:type the_job:
:class:`openquake.engine.JobContext` instance
:param sites:
List of :class:`openquake.shapes.Site` objects
:param poes:
Probability of Exceedence levels for the calculation
:type poes:
List of floats
:param result_dir:
Path where full disaggregation results should be stored
:returns:
Result data in the following form::
[(realization_1, poe_1,
[(site_1, gmv_1, matrix_path_1),
(site_2, gmv_2, matrix_path_2)]
),
(realization_1, poe_2,
[(site_1, gmv_1, matrix_path_3),
(site_2, gmv_2, matrix_path_4)]
),
...
(realization_N, poe_N,
[(site_1, gmv_1, matrix_path_N-1),
(site_2, gmv_2, matrix_path_N)]
),
]
A single matrix result in this form looks like this::
[(1, 0.1,
[(Site(0.0, 0.0), 0.2257,
'/var/lib/openquake/disagg-results/job-372/some_guid.h5'),]
),
]
"""
# accumulates the final results of this method
full_da_results = []
# accumulates task data across the realization and poe loops
task_data = []
src_model_rnd = random.Random()
src_model_rnd.seed(self.job_ctxt['SOURCE_MODEL_LT_RANDOM_SEED'])
gmpe_rnd = random.Random()
gmpe_rnd.seed(self.job_ctxt['GMPE_LT_RANDOM_SEED'])
for rlz in xrange(1, realizations + 1): # 1 to N, inclusive
# cache the source model and gmpe model in the KVS
# so the Java code can access it
general.store_source_model(self.job_ctxt.job_id,
src_model_rnd.getrandbits(32),
self.job_ctxt.params, self.calc)
general.store_gmpe_map(self.job_ctxt.job_id,
gmpe_rnd.getrandbits(32), self.calc)
for poe in poes:
task_site_pairs = []
for site in sites:
a_task = compute_disagg_matrix_task.delay(
self.job_ctxt.job_id, site, rlz, poe, result_dir)
task_site_pairs.append((a_task, site))
task_data.append((rlz, poe, task_site_pairs))
for rlz, poe, task_site_pairs in task_data:
# accumulates all data for a given (realization, poe) pair
rlz_poe_data = []
for a_task, site in task_site_pairs:
a_task.wait()
if not a_task.successful():
msg = (
"Full Disaggregation matrix computation task"
" for job %s with task_id=%s, realization=%s, PoE=%s,"
" site=%s has failed with the following error: %s")
msg %= (self.job_ctxt.job_id, a_task.task_id, rlz, poe,
site, a_task.result)
LOG.critical(msg)
raise RuntimeError(msg)
else:
gmv, matrix_path = a_task.result
rlz_poe_data.append((site, gmv, matrix_path))
full_da_results.append((rlz, poe, rlz_poe_data))
return full_da_results
def distribute_disagg(self, sites, realizations, poes, result_dir):
"""Compute disaggregation by splitting up the calculation over sites,
realizations, and PoE values.
:param the_job:
CalculationProxy definition
:type the_job:
:class:`openquake.engine.CalculationProxy` instance
:param sites:
List of :class:`openquake.shapes.Site` objects
:param poes:
Probability of Exceedence levels for the calculation
:type poes:
List of floats
:param result_dir:
Path where full disaggregation results should be stored
:returns:
Result data in the following form::
[(realization_1, poe_1,
[(site_1, gmv_1, matrix_path_1),
(site_2, gmv_2, matrix_path_2)]
),
(realization_1, poe_2,
[(site_1, gmv_1, matrix_path_3),
(site_2, gmv_2, matrix_path_4)]
),
...
(realization_N, poe_N,
[(site_1, gmv_1, matrix_path_N-1),
(site_2, gmv_2, matrix_path_N)]
),
]
A single matrix result in this form looks like this::
[(1, 0.1,
[(Site(0.0, 0.0), 0.2257,
'/var/lib/openquake/disagg-results/job-372/some_guid.h5'),]
),
]
"""
# accumulates the final results of this method
full_da_results = []
# accumulates task data across the realization and poe loops
task_data = []
src_model_rnd = random.Random()
src_model_rnd.seed(self.calc_proxy['SOURCE_MODEL_LT_RANDOM_SEED'])
gmpe_rnd = random.Random()
gmpe_rnd.seed(self.calc_proxy['GMPE_LT_RANDOM_SEED'])
for rlz in xrange(1, realizations + 1): # 1 to N, inclusive
# cache the source model and gmpe model in the KVS
# so the Java code can access it
store_source_model(self.calc_proxy.job_id,
src_model_rnd.getrandbits(32),
self.calc_proxy.params, self.calc)
store_gmpe_map(self.calc_proxy.job_id, gmpe_rnd.getrandbits(32),
self.calc)
for poe in poes:
task_site_pairs = []
for site in sites:
a_task = compute_disagg_matrix_task.delay(
self.calc_proxy.job_id, site, rlz, poe, result_dir)
task_site_pairs.append((a_task, site))
task_data.append((rlz, poe, task_site_pairs))
for rlz, poe, task_site_pairs in task_data:
# accumulates all data for a given (realization, poe) pair
rlz_poe_data = []
for a_task, site in task_site_pairs:
a_task.wait()
if not a_task.successful():
msg = (
"Full Disaggregation matrix computation task"
" for job %s with task_id=%s, realization=%s, PoE=%s,"
" site=%s has failed with the following error: %s")
msg %= (
self.calc_proxy.job_id, a_task.task_id, rlz, poe,
site, a_task.result)
LOG.critical(msg)
raise RuntimeError(msg)
else:
gmv, matrix_path = a_task.result
rlz_poe_data.append((site, gmv, matrix_path))
full_da_results.append((rlz, poe, rlz_poe_data))
return full_da_results