def touch_result_file(job_id, path, sites, n_samples, n_periods):
"""Given a path (including the file name), create an empty HDF5 result file
containing 1 empty data set for each site. Each dataset will be a matrix
with the number of rows = number of samples and number of cols = number of
UHS periods.
:param int job_id:
ID of the job record in the DB/KVS.
:param str path:
Location (including a file name) on an NFS where the empty
result file should be created.
:param sites:
List of :class:`openquake.shapes.Site` objects.
:param int n_samples:
Number of logic tree samples (the y-dimension of each dataset).
:param int n_periods:
Number of UHS periods (the x-dimension of each dataset).
"""
utils_tasks.check_job_status(job_id)
# TODO: Generate the sites, instead of pumping them through rabbit?
with h5py.File(path, 'w') as h5_file:
for site in sites:
ds_name = 'lon:%s-lat:%s' % (site.longitude, site.latitude)
ds_shape = (n_samples, n_periods)
h5_file.create_dataset(ds_name, dtype=numpy.float64,
shape=ds_shape)
def compute_disagg_matrix_task(job_id, site, realization, poe, result_dir):
""" Compute a complete 5D Disaggregation matrix. This task leans heavily
on the DisaggregationCalculator (in the OpenQuake Java lib) to handle this
computation.
:param job_id: id of the job record in the KVS
:type job_id: `str`
:param site: a single site of interest
:type site: :class:`openquake.shapes.Site` instance`
:param int realization: logic tree sample iteration number
:param poe: Probability of Exceedence
:type poe: `float`
:param result_dir: location for the Java code to write the matrix in an
HDF5 file (in a distributed environment, this should be the path of a
mounted NFS)
:returns: 2-tuple of (ground_motion_value, path_to_h5_matrix_file)
"""
# check and see if the job is still valid (i.e., not complete or failed)
check_job_status(job_id)
log_msg = (
"Computing full disaggregation matrix for job_id=%s, site=%s, "
"realization=%s, PoE=%s. Matrix results will be serialized to `%s`.")
log_msg %= (job_id, site, realization, poe, result_dir)
LOG.info(log_msg)
return compute_disagg_matrix(job_id, site, poe, result_dir)
def compute_hazard_curve(job_id, sites, realization):
""" Generate hazard curve for a given site list. """
check_job_status(job_id)
hazengine = job.Job.from_kvs(job_id)
with mixins.Mixin(hazengine, hazjob.HazJobMixin):
keys = hazengine.compute_hazard_curve(sites, realization)
return keys
def compute_ground_motion_fields(job_id, sites, history, realization, seed):
""" Generate ground motion fields """
# TODO(JMC): Use a block_id instead of a sites list
check_job_status(job_id)
hazengine = job.Job.from_kvs(job_id)
with mixins.Mixin(hazengine, hazjob.HazJobMixin):
hazengine.compute_ground_motion_fields(sites, history, realization,
seed)
def compute_quantile_curves(job_id, sites, realizations, quantiles):
"""Compute the quantile hazard curve for each site given."""
check_job_status(job_id)
HAZARD_LOG.info("Computing QUANTILE curves for %s sites (job_id %s)"
% (len(sites), job_id))
return classical_psha.compute_quantile_hazard_curves(job_id, sites,
realizations, quantiles)
def compute_mean_curves(job_id, sites, realizations):
"""Compute the mean hazard curve for each site given."""
check_job_status(job_id)
HAZARD_LOG.info("Computing MEAN curves for %s sites (job_id %s)"
% (len(sites), job_id))
return classical_psha.compute_mean_hazard_curves(job_id, sites,
realizations)
def test_not_completed_with_true(self):
with patch('openquake.job.Job.is_job_completed') as mock:
mock.return_value = True
try:
tasks.check_job_status(31)
except tasks.JobCompletedError as exc:
self.assertEqual(exc.message, 31)
else:
self.fail("JobCompletedError wasn't raised")
self.assertEqual(mock.call_args_list, [((31, ), {})])
def compute_hazard_curve(job_id, site_list, realization, callback=None):
""" Generate hazard curve for a given site list. """
check_job_status(job_id)
hazengine = job.Job.from_kvs(job_id)
with mixins.Mixin(hazengine, hazjob.HazJobMixin):
keys = hazengine.compute_hazard_curve(site_list, realization)
if callback:
subtask(callback).delay(job_id, site_list)
return keys
def generate_erf(job_id):
"""
Stubbed ERF generator
Takes a job_id, returns a job_id.
Connects to the Java HazardEngine using hazardwrapper, waits for an ERF to
be generated, and then writes it to KVS.
"""
# TODO(JM): implement real ERF computation
check_job_status(job_id)
kvs.get_client().set(kvs.tokens.erf_key(job_id),
json.JSONEncoder().encode([job_id]))
return job_id
def compute_mgm_intensity(job_id, block_id, site_id):
"""
Compute mean ground intensity for a specific site.
"""
check_job_status(job_id)
kvs_client = kvs.get_client()
mgm_key = kvs.tokens.mgm_key(job_id, block_id, site_id)
mgm = kvs_client.get(mgm_key)
if not mgm:
# TODO(jm): implement hazardwrapper and make this work.
# TODO(chris): uncomment below when hazardwapper is done
# Synchronous execution.
#result = hazardwrapper.apply(args=[job_id, block_id, site_id])
#mgm = kvs_client.get(mgm_key)
pass
return json.JSONDecoder().decode(mgm)
def compute_uhs_task(job_id, realization, site, result_dir):
"""Compute Uniform Hazard Spectra for a given site of interest and 1 or
more Probability of Exceedance values. The bulk of the computation will
be done by utilizing the `UHSCalculator` class in the Java code.
UHS results (for each poe) will be written as a 1D array into temporary
HDF5 files. (The files will later be collected and 'reduced' into final
result files.)
:param int job_id:
ID of the job record in the DB/KVS.
:param realization:
Logic tree sample number (from 1 to N, where N is the
NUMBER_OF_LOGIC_TREE_SAMPLES param defined in the job config.
:param site:
The site of interest (a :class:`openquake.shapes.Site` object).
:param result_dir:
NFS result directory path. For each poe, a subfolder will be created to
contain intermediate calculation results. (Each call to this task will
generate 1 result file per poe.)
:returns:
A list of the resulting file names (1 per poe).
"""
utils_tasks.check_job_status(job_id)
the_job = Job.from_kvs(job_id)
log_msg = (
"Computing UHS for job_id=%s, site=%s, realization=%s."
" UHS results will be serialized to `%s`.")
log_msg %= (the_job.job_id, site, realization, result_dir)
LOG.info(log_msg)
uhs_results = compute_uhs(the_job, site)
return write_uhs_results(result_dir, realization, site, uhs_results)
def compute_risk(job_id, block_id, **kwargs):
""" A task for computing risk, calls the mixed in compute_risk method """
check_job_status(job_id)
engine = job.Job.from_kvs(job_id)
with mixins.Mixin(engine, RiskJobMixin) as mixed:
return mixed.compute_risk(block_id, **kwargs)
def test_not_completed(self):
with patch('openquake.job.Job.is_job_completed') as mock:
mock.return_value = False
tasks.check_job_status(42)
self.assertEqual(mock.call_args_list, [((42, ), {})])