def jvm(max_mem=None):
"""Return the jpype module, after guaranteeing the JVM is running and
the classpath has been loaded properly."""
jarpaths = (os.path.abspath(
os.path.join(os.path.dirname(__file__), "../lib")),
os.path.abspath(
os.path.join(os.path.dirname(__file__), "../dist")))
# TODO(JMC): Make sure these directories exist
# LOG.debug("Jarpath is %s", jarpaths)
if not jpype.isJVMStarted():
max_mem = get_jvm_max_mem(max_mem)
LOG.debug("Default JVM path is %s" % jpype.getDefaultJVMPath())
jpype.startJVM(
jpype.getDefaultJVMPath(),
"-Djava.ext.dirs=%s:%s" % jarpaths,
#"-Dnet.spy.log.LoggerImpl=net.spy.memcached.compat.log.Log4JLogger",
# "-Dlog4j.debug",
"-Dlog4j.configuration=log4j.properties",
"-Dlog4j.rootLogger=%s, A1" % (FLAGS.debug.upper()),
# "-Dlog4j.rootLogger=DEBUG, A1",
"-Xmx%sM" % max_mem)
if FLAGS.capture_java_debug:
mystream = jpype.JProxy("org.gem.IPythonPipe", inst=sys.stdout)
errstream = jpype.JProxy("org.gem.IPythonPipe", inst=sys.stderr)
outputstream = jpype.JClass("org.gem.PythonOutputStream")()
err_stream = jpype.JClass("org.gem.PythonOutputStream")()
outputstream.setPythonStdout(mystream)
err_stream.setPythonStdout(errstream)
ps = jpype.JClass("java.io.PrintStream")
jpype.java.lang.System.setOut(ps(outputstream))
jpype.java.lang.System.setErr(ps(err_stream))
return jpype
def jvm(max_mem=None):
"""Return the jpype module, after guaranteeing the JVM is running and
the classpath has been loaded properly."""
jarpaths = (os.path.abspath(
os.path.join(os.path.dirname(__file__), "../lib")),
os.path.abspath(
os.path.join(os.path.dirname(__file__), "../dist")))
log4j_properties_path = os.path.abspath(
os.path.join(os.path.dirname(__file__),
"../log4j.properties"))
if not jpype.isJVMStarted():
max_mem = get_jvm_max_mem(max_mem)
LOG.debug("Default JVM path is %s" % jpype.getDefaultJVMPath())
jpype.startJVM(jpype.getDefaultJVMPath(),
"-Djava.ext.dirs=%s:%s" % jarpaths,
# "-Dlog4j.debug", # turn on log4j internal debugging
"-Dlog4j.configuration=file://%s" % log4j_properties_path,
"-Xmx%sM" % max_mem)
# override the log level set in log4j configuration file this can't be
# done on the JVM command line (i.e. -Dlog4j.rootLogger= is not
# supported by log4j)
_set_java_log_level(FLAGS.debug.upper())
if FLAGS.capture_java_debug:
_setup_java_capture(sys.stdout, sys.stderr)
return jpype
def jvm(max_mem=4000):
"""Return the jpype module, after guaranteeing the JVM is running and
the classpath has been loaded properly."""
jarpaths = (os.path.abspath(
os.path.join(os.path.dirname(__file__), "../lib")),
os.path.abspath(
os.path.join(os.path.dirname(__file__), "../dist")))
# TODO(JMC): Make sure these directories exist
# LOG.debug("Jarpath is %s", jarpaths)
if not jpype.isJVMStarted():
LOG.debug("Default JVM path is %s" % jpype.getDefaultJVMPath())
jpype.startJVM(jpype.getDefaultJVMPath(),
"-Djava.ext.dirs=%s:%s" % jarpaths,
#"-Dnet.spy.log.LoggerImpl=net.spy.memcached.compat.log.Log4JLogger",
# "-Dlog4j.debug",
"-Dlog4j.configuration=log4j.properties",
"-Dlog4j.rootLogger=%s, A1" % (FLAGS.debug.upper()),
# "-Dlog4j.rootLogger=DEBUG, A1",
"-Xmx%sM" % max_mem)
if FLAGS.capture_java_debug:
mystream = jpype.JProxy("org.gem.IPythonPipe", inst=sys.stdout)
errstream = jpype.JProxy("org.gem.IPythonPipe", inst=sys.stderr)
outputstream = jpype.JClass("org.gem.PythonOutputStream")()
err_stream = jpype.JClass("org.gem.PythonOutputStream")()
outputstream.setPythonStdout(mystream)
err_stream.setPythonStdout(errstream)
ps = jpype.JClass("java.io.PrintStream")
jpype.java.lang.System.setOut(ps(outputstream))
jpype.java.lang.System.setErr(ps(err_stream))
return jpype
def _partition(self):
"""Split the set of sites to compute in blocks and store
the in the underlying kvs system.
"""
sites = []
self.blocks_keys = []
region_constraint = self.region
# we use the exposure, if specified,
# otherwise we use the input region
if self.has(EXPOSURE):
sites = self._read_sites_from_exposure()
LOG.debug("Loaded %s sites from exposure portfolio." % len(sites))
elif self.region:
sites = self.region.sites
else:
raise Exception("I don't know how to get the sites!")
if self.partition:
block_count = 0
for block in BlockSplitter(sites, constraint=region_constraint):
self.blocks_keys.append(block.id)
block.to_kvs()
block_count += 1
LOG.debug("Job has partitioned %s sites into %s blocks" % (
len(sites), block_count))
else:
block = Block(sites)
self.blocks_keys.append(block.id)
block.to_kvs()
def plot_aggregate_curve(job, aggregate_curve):
"""Plot an aggreate loss curve.
This function is triggered only if the AGGREGATE_LOSS_CURVE
parameter is specified in the configuration file.
:param job: the job the engine is currently processing.
:type job:
:py:class:`openquake.risk.job.probabilistic.ProbabilisticEventMixin`
:param aggregate_curve: the aggregate curve to plot.
:type aggregate_curve: :py:class:`openquake.shapes.Curve`
"""
if not job.has("AGGREGATE_LOSS_CURVE"):
LOG.debug("AGGREGATE_LOSS_CURVE parameter not specified, " \
"skipping aggregate loss curve computation...")
return
path = os.path.join(job.params["BASE_PATH"],
job.params["OUTPUT_DIR"], _filename(job.job_id))
plotter = curve.CurvePlot(path)
plotter.write(_for_plotting(aggregate_curve,
job.params["INVESTIGATION_TIME"]), autoscale_y=False)
plotter.close()
LOG.debug("Aggregate loss curve stored at %s" % path)
def compute_quantile_hazard_curves(job, sites):
"""Compute a quantile hazard curve for each site in the list
using as input all the pre-computed curves for different realizations.
The QUANTILE_LEVELS parameter in the configuration file specifies
all the values used in the computation.
"""
keys = []
quantiles = _extract_quantiles_from_config(job)
LOG.debug("List of QUANTILES is %s" % quantiles)
for site in sites:
for quantile in quantiles:
quantile_curve = {
"site_lat": site.latitude,
"site_lon": site.longitude,
"curve": _reconstruct_curve_list_from(compute_quantile_curve(curves_at(job.id, site), quantile)),
}
key = kvs.tokens.quantile_hazard_curve_key(job.id, site, quantile)
keys.append(key)
LOG.debug("QUANTILE curve at %s is %s" % (key, quantile_curve))
kvs.set_value_json_encoded(key, quantile_curve)
return keys
def compute_aggregate_curve(job):
"""Compute and plot an aggreate loss curve.
This function expects to find in kvs a set of pre computed
GMFs and assets.
This function is triggered only if the AGGREGATE_LOSS_CURVE
parameter is specified in the configuration file.
:param job: the job the engine is currently processing.
:type job: openquake.risk.job.probabilistic.ProbabilisticEventMixin
"""
if not job.has("AGGREGATE_LOSS_CURVE"):
LOG.debug("AGGREGATE_LOSS_CURVE parameter not specified, " \
"skipping aggregate loss curve computation...")
return
epsilon_provider = risk_job.EpsilonProvider(job.params)
aggregate_loss_curve = \
prob.AggregateLossCurve.from_kvs(job.id, epsilon_provider)
path = os.path.join(job.params["BASE_PATH"],
job.params["OUTPUT_DIR"], _filename(job.id))
plotter = curve.CurvePlot(path)
plotter.write(_for_plotting(
aggregate_loss_curve.compute(),
job.params["INVESTIGATION_TIME"]), autoscale_y=False)
plotter.close()
LOG.debug("Aggregate loss curve stored at %s" % path)
def compute_aggregate_curve(job):
"""Compute and plot an aggreate loss curve.
This function expects to find in kvs a set of pre computed
GMFs and assets.
This function is triggered only if the AGGREGATE_LOSS_CURVE
parameter is specified in the configuration file.
:param job: the job the engine is currently processing.
:type job: openquake.risk.job.probabilistic.ProbabilisticEventMixin
"""
if not job.has("AGGREGATE_LOSS_CURVE"):
LOG.debug("AGGREGATE_LOSS_CURVE parameter not specified, " \
"skipping aggregate loss curve computation...")
return
epsilon_provider = risk_job.EpsilonProvider(job.params)
aggregate_loss_curve = \
prob.AggregateLossCurve.from_kvs(job.id, epsilon_provider)
path = os.path.join(job.params["BASE_PATH"], job.params["OUTPUT_DIR"],
_filename(job.id))
plotter = curve.CurvePlot(path)
plotter.write(_for_plotting(aggregate_loss_curve.compute(),
job.params["INVESTIGATION_TIME"]),
autoscale_y=False)
plotter.close()
LOG.debug("Aggregate loss curve stored at %s" % path)
def _get_iml_from(curve, job, poe):
"""Return the interpolated IML using the values defined in
the INTENSITY_MEASURE_LEVELS parameter as the reference grid to
interpolate in.
IML from config is in ascending order (abscissa of hazard curve)
PoE from curve is in descending order (ordinate of hazard curve)
In our interpolation, PoE becomes the x axis, IML the y axis, therefore
the arrays have to be reversed (x axis has to be monotonically
increasing).
"""
# reverse arrays
poes = numpy.array(_extract_y_values_from(curve["curve"]))[::-1]
imls = numpy.log(numpy.array(_extract_imls_from_config(job))[::-1])
site = shapes.Site(curve["site_lon"], curve["site_lat"])
if poe > poes[-1]:
LOG.debug("[HAZARD_MAP] Interpolation out of bounds for PoE %s, "\
"using maximum PoE value pair, PoE: %s, IML: %s, at site %s" % (
poe, poes[-1], math.exp(imls[-1]), site))
return math.exp(imls[-1])
if poe < poes[0]:
LOG.debug("[HAZARD_MAP] Interpolation out of bounds for PoE %s, "\
"using minimum PoE value pair, PoE: %s, IML: %s, at site %s" % (
poe, poes[0], math.exp(imls[0]), site))
return math.exp(imls[0])
return math.exp(interp1d(poes, imls, kind='linear')(poe))
def compute_quantile_hazard_curves(job, sites):
"""Compute a quantile hazard curve for each site in the list
using as input all the pre-computed curves for different realizations.
The QUANTILE_LEVELS parameter in the configuration file specifies
all the values used in the computation.
"""
keys = []
quantiles = _extract_values_from_config(job, QUANTILE_PARAM_NAME)
LOG.debug("[QUANTILE_HAZARD_CURVES] List of quantiles is %s" % quantiles)
for site in sites:
for quantile in quantiles:
hazard_curves = curves_at(job.id, site)
poes = [_extract_y_values_from(curve) for curve in hazard_curves]
quantile_poes = compute_quantile_curve(poes, quantile)
quantile_curve = {"site_lat": site.latitude,
"site_lon": site.longitude,
"curve": _reconstruct_curve_list_from(quantile_poes)}
key = kvs.tokens.quantile_hazard_curve_key(
job.id, site, quantile)
keys.append(key)
kvs.set_value_json_encoded(key, quantile_curve)
return keys
def compute_mean_hazard_maps(job):
"""Compute mean hazard maps using as input all the
pre computed mean hazard curves.
The POES_HAZARD_MAPS parameter in the configuration file specifies
all the values used in the computation.
"""
poes = _extract_values_from_config(job, POES_PARAM_NAME)
LOG.debug("[MEAN_HAZARD_MAPS] List of POEs is %s" % poes)
# get all the pre computed mean curves
pattern = "%s*%s*" % (kvs.tokens.MEAN_HAZARD_CURVE_KEY_TOKEN, job.id)
mean_curves = kvs.mget_decoded(pattern)
LOG.debug("[MEAN_HAZARD_MAPS] Found %s pre computed mean curves"
% len(mean_curves))
keys = []
for poe in poes:
for mean_curve in mean_curves:
site = shapes.Site(mean_curve["site_lon"],
mean_curve["site_lat"])
key = kvs.tokens.mean_hazard_map_key(
job.id, site, poe)
keys.append(key)
_store_iml_for(mean_curve, key, job, poe)
return keys
def read_sites_from_exposure(a_job):
"""
Given the exposure model specified in the job config, read all sites which
are located within the region of interest.
:param a_job: a Job object with an EXPOSURE parameter defined
:type a_job: :py:class:`openquake.job.Job`
:returns: a list of :py:class:`openquake.shapes.Site` objects
"""
sites = []
path = os.path.join(a_job.base_path, a_job.params[conf.EXPOSURE])
reader = exposure.ExposurePortfolioFile(path)
constraint = a_job.region
LOG.debug(
"Constraining exposure parsing to %s" % constraint)
for site, _asset_data in reader.filter(constraint):
# we don't want duplicates (bug 812395):
if not site in sites:
sites.append(site)
return sites
def _partition(self):
"""Split the set of sites to compute in blocks and store
the in the underlying kvs system.
"""
sites = []
self.blocks_keys = []
region_constraint = self.region
# we use the exposure, if specified,
# otherwise we use the input region
if self.has(EXPOSURE):
sites = self._read_sites_from_exposure()
LOG.debug("Loaded %s sites from exposure portfolio." % len(sites))
elif self.region:
sites = self.region.sites
else:
raise Exception("I don't know how to get the sites!")
if self.partition:
block_count = 0
for block in BlockSplitter(sites, constraint=region_constraint):
self.blocks_keys.append(block.id)
block.to_kvs()
block_count += 1
LOG.debug("Job has partitioned %s sites into %s blocks" %
(len(sites), block_count))
else:
block = Block(sites)
self.blocks_keys.append(block.id)
block.to_kvs()
def from_kvs(job_id, epsilon_provider):
"""Return an aggregate curve using the GMFs and assets
stored in the underlying kvs system."""
vuln_model = vulnerability.load_vuln_model_from_kvs(job_id)
aggregate_curve = AggregateLossCurve(vuln_model, epsilon_provider)
gmfs_keys = kvs.get_keys("%s*%s*" % (
job_id, kvs.tokens.GMF_KEY_TOKEN))
LOG.debug("Found %s stored GMFs..." % len(gmfs_keys))
asset_counter = 0
for gmfs_key in gmfs_keys:
assets = _assets_keys_for_gmfs(job_id, gmfs_key)
for asset in assets:
asset_counter += 1
gmfs = kvs.get_value_json_decoded(gmfs_key)
aggregate_curve.append(gmfs,
json.JSONDecoder().decode(asset))
LOG.debug("Found %s stored assets..." % asset_counter)
return aggregate_curve
def from_file(config_file, output_type):
"""
Create a job from external configuration files.
:param config_file: the external configuration file path
:param output_type: where to store results:
* 'db' database
* 'xml' XML files *plus* database
:param params: optional dictionary of default parameters, overridden by
the ones read from the config file
:type params: :py:class:`dict`
"""
# output_type can be set, in addition to 'db' and 'xml', also to
# 'xml_without_db', which has the effect of serializing only to xml
# without requiring a database at all.
# This allows to run tests without requiring a database.
# This is not documented in the public interface because it is
# essentially a detail of our current tests and ci infrastructure.
assert output_type in ('db', 'xml', 'xml_without_db')
config_file = os.path.abspath(config_file)
LOG.debug("Loading Job from %s" % (config_file))
base_path = os.path.abspath(os.path.dirname(config_file))
params = {}
sections = []
for each_config_file in Job.default_configs() + [config_file]:
new_sections, new_params = parse_config_file(each_config_file)
sections.extend(new_sections)
params.update(new_params)
params['BASE_PATH'] = base_path
if output_type == 'xml_without_db':
# we are running a test
job_id = 0
serialize_results_to = ['xml']
else:
# openquake-server creates the job record in advance and stores the
# job id in the config file
job_id = params.get('OPENQUAKE_JOB_ID')
if not job_id:
# create the database record for this job
job_id = prepare_job(params).id
if output_type == 'db':
serialize_results_to = ['db']
else:
serialize_results_to = ['db', 'xml']
job = Job(params, job_id, sections=sections, base_path=base_path)
job.serialize_results_to = serialize_results_to
job.config_file = config_file # pylint: disable=W0201
return job
def cleanup(self):
"""
Perform any necessary cleanup steps after the job completes.
Currently, this method only clears KVS cache data for the job.
"""
LOG.debug("Running KVS garbage collection for job %s" % self.job_id)
gc_cmd = ['python', 'bin/cache_gc.py', '--job=%s' % self.job_id]
# run KVS garbage collection aynchronously
# stdout goes to /dev/null to silence any output from the GC
subprocess.Popen(gc_cmd, env=os.environ, stdout=open('/dev/null', 'w'))
def serialize_mfd(mfd, parent_node):
if mfd.__class__.__name__ == 'org.opensha.sha.magdist.SummedMagFreqDist':
LOG.debug("Serializing a SummedMFD")
mfd_list = mfd.getMagFreqDists()
if mfd_list is None:
mfd_list = [mfd]
for sub_mfd in mfd_list:
serialize_evenly_discretized_mfd(sub_mfd, parent_node)
elif mfd.__class__.__name__ == 'org.opensha.sha.magdist.IncrementalMagFreqDist':
LOG.debug("Serializing an IncrementalMFD")
serialize_evenly_discretized_mfd(mfd, parent_node)
else:
raise Exception("Unhandled mfd class: %s" % mfd.__class__.__name__)
def from_kvs(job_id):
"""Return an aggregate curve using the computed
loss curves in the kvs system."""
client = kvs.get_client(binary=False)
keys = client.keys("%s*%s*" % (job_id,
kvs.tokens.LOSS_CURVE_KEY_TOKEN))
LOG.debug("Found %s stored loss curves..." % len(keys))
aggregate_curve = AggregateLossCurve()
for key in keys:
aggregate_curve.append(shapes.Curve.from_json(kvs.get(key)))
return aggregate_curve
def _read_sites_from_exposure(self):
"""Read the set of sites to compute from the exposure file specified
in the job definition."""
sites = []
path = os.path.join(self.base_path, self[EXPOSURE])
reader = exposure.ExposurePortfolioFile(path)
constraint = self.region
if not constraint:
constraint = AlwaysTrueConstraint()
else:
LOG.debug("Constraining exposure parsing to %s" % constraint.polygon)
for asset_data in reader.filter(constraint):
sites.append(asset_data[0])
return sites
def _slurp_files(self):
"""Read referenced files and write them into kvs, keyed on their
sha1s."""
kvs_client = kvs.get_client(binary=False)
if self.base_path is None:
LOG.debug("Can't slurp files without a base path, homie...")
return
for key, val in self.params.items():
if key[-5:] == '_FILE':
path = os.path.join(self.base_path, val)
with open(path) as data_file:
LOG.debug("Slurping %s" % path)
sha1 = hashlib.sha1(data_file.read()).hexdigest()
data_file.seek(0)
kvs_client.set(sha1, data_file.read())
self.params[key] = sha1
def _slurp_files(self):
"""Read referenced files and write them into kvs, keyed on their
sha1s."""
kvs_client = kvs.get_client(binary=False)
if self.base_path is None:
LOG.debug("Can't slurp files without a base path, homie...")
return
for key, val in self.params.items():
if key[-5:] == '_FILE':
path = os.path.join(self.base_path, val)
with open(path) as data_file:
LOG.debug("Slurping %s" % path)
sha1 = hashlib.sha1(data_file.read()).hexdigest()
data_file.seek(0)
kvs_client.set(sha1, data_file.read())
self.params[key] = sha1
def from_file(config_file):
""" Create a job from external configuration files. """
config_file = os.path.abspath(config_file)
LOG.debug("Loading Job from %s" % (config_file))
base_path = os.path.abspath(os.path.dirname(config_file))
params = {}
sections = []
for each_config_file in Job.default_configs() + [config_file]:
new_sections, new_params = parse_config_file(each_config_file)
sections.extend(new_sections)
params.update(new_params)
params['BASE_PATH'] = base_path
job = Job(params, sections=sections, base_path=base_path)
job.config_file = config_file # pylint: disable=W0201
return job
def from_file(config_file):
""" Create a job from external configuration files. """
config_file = os.path.abspath(config_file)
LOG.debug("Loading Job from %s" % (config_file))
base_path = os.path.abspath(os.path.dirname(config_file))
params = {}
sections = []
for each_config_file in Job.default_configs() + [config_file]:
new_sections, new_params = parse_config_file(each_config_file)
sections.extend(new_sections)
params.update(new_params)
params['BASE_PATH'] = base_path
job = Job(params, sections=sections, base_path=base_path)
job.config_file = config_file # pylint: disable=W0201
return job
def _slurp_files(self):
"""Read referenced files and write them into kvs, keyed on their
sha1s."""
kvs_client = kvs.get_client()
if self.base_path is None:
LOG.debug("Can't slurp files without a base path, homie...")
return
for key, val in self.params.items():
if key[-5:] == '_FILE':
path = os.path.join(self.base_path, val)
with open(path) as data_file:
LOG.debug("Slurping %s" % path)
blob = data_file.read()
file_key = kvs.tokens.generate_blob_key(self.job_id, blob)
kvs_client.set(file_key, blob)
self.params[key] = file_key
self.params[key + "_PATH"] = path
def _read_sites_from_exposure(self):
"""Read the set of sites to compute from the exposure file specified
in the job definition."""
sites = []
path = os.path.join(self.base_path, self[EXPOSURE])
reader = exposure.ExposurePortfolioFile(path)
constraint = self.region
if not constraint:
constraint = AlwaysTrueConstraint()
else:
LOG.debug("Constraining exposure parsing to %s" %
constraint.polygon)
for asset_data in reader.filter(constraint):
sites.append(asset_data[0])
return sites
def from_kvs(job_id):
"""Return an aggregate curve using the GMFs and assets
stored in the underlying kvs system."""
vuln_model = vulnerability.load_vuln_model_from_kvs(job_id)
aggregate_curve = AggregateLossCurve(vuln_model)
client = kvs.get_client(binary=False)
gmfs_keys = client.keys("%s*%s*" % (job_id, kvs.tokens.GMF_KEY_TOKEN))
LOG.debug("Found %s stored GMFs..." % len(gmfs_keys))
for gmfs_key in gmfs_keys: # O(2*n)
asset = _asset_for_gmfs(job_id, gmfs_key)
gmfs = kvs.get_value_json_decoded(gmfs_key)
aggregate_curve.append(gmfs, asset)
return aggregate_curve
def safe_interpolator(poe):
"""
Return the interpolated IML, limiting the value between the minimum and
maximum IMLs of the original points describing the curve.
"""
if poe > poes[-1]:
LOG.debug("[HAZARD_MAP] Interpolation out of bounds for PoE %s, "\
"using maximum PoE value pair, PoE: %s, IML: %s, at site %s"
% (poe, poes[-1], imls[-1], site))
return imls[-1]
if poe < poes[0]:
LOG.debug("[HAZARD_MAP] Interpolation out of bounds for PoE %s, "\
"using minimum PoE value pair, PoE: %s, IML: %s, at site %s"
% (poe, poes[0], imls[0], site))
return imls[0]
return math.exp(interpolator(poe))
def safe_interpolator(poe):
"""
Return the interpolated IML, limiting the value between the minimum and
maximum IMLs of the original points describing the curve.
"""
if poe > poes[-1]:
LOG.debug("[HAZARD_MAP] Interpolation out of bounds for PoE %s, "\
"using maximum PoE value pair, PoE: %s, IML: %s, at site %s"
% (poe, poes[-1], imls[-1], site))
return imls[-1]
if poe < poes[0]:
LOG.debug("[HAZARD_MAP] Interpolation out of bounds for PoE %s, "\
"using minimum PoE value pair, PoE: %s, IML: %s, at site %s"
% (poe, poes[0], imls[0], site))
return imls[0]
return math.exp(interpolator(poe))
def compute_mean_hazard_maps(job_id, sites, imls, poes):
"""Compute mean hazard maps using as input all the
pre computed mean hazard curves.
"""
LOG.debug("[MEAN_HAZARD_MAPS] List of POEs is %s" % poes)
keys = []
for site in sites:
mean_poes = kvs.get_value_json_decoded(kvs.tokens.mean_hazard_curve_key(job_id, site))
interpolate = build_interpolator(mean_poes, imls, site)
for poe in poes:
key = kvs.tokens.mean_hazard_map_key(job_id, site, poe)
keys.append(key)
kvs.set_value_json_encoded(key, interpolate(poe))
return keys
def launch(self):
""" Based on the behaviour specified in the configuration, mix in the
correct behaviour for the tasks and then execute them.
"""
output_dir = os.path.join(self.base_path, self['OUTPUT_DIR'])
if not os.path.exists(output_dir):
os.makedirs(output_dir)
for (key, mixin) in Mixin.ordered_mixins():
if key.upper() not in self.sections:
continue
with Mixin(self, mixin):
# The mixin defines a preload decorator to handle the needed
# data for the tasks and decorates _execute(). the mixin's
# _execute() method calls the expected tasks.
LOG.debug(
"Job %s Launching %s for %s" % (self.job_id, mixin, key))
self.execute()
def compute_quantile_hazard_curves(job_id, sites, realizations, quantiles):
"""Compute a quantile hazard curve for each site in the list
using as input all the pre-computed curves for different realizations.
"""
LOG.debug("[QUANTILE_HAZARD_CURVES] List of quantiles is %s" % quantiles)
keys = []
for site in sites:
poes = poes_at(job_id, site, realizations)
for quantile in quantiles:
quantile_poes = compute_quantile_curve(poes, quantile)
key = kvs.tokens.quantile_hazard_curve_key(job_id, site, quantile)
keys.append(key)
kvs.set_value_json_encoded(key, quantile_poes)
return keys
def compute_mean_hazard_maps(job_id, sites, imls, poes):
"""Compute mean hazard maps using as input all the
pre computed mean hazard curves.
"""
LOG.debug("[MEAN_HAZARD_MAPS] List of POEs is %s" % poes)
keys = []
for site in sites:
mean_poes = kvs.get_value_json_decoded(
kvs.tokens.mean_hazard_curve_key(job_id, site))
interpolate = build_interpolator(mean_poes, imls, site)
for poe in poes:
key = kvs.tokens.mean_hazard_map_key(job_id, site, poe)
keys.append(key)
kvs.set_value_json_encoded(key, interpolate(poe))
return keys
def compute_mean_hazard_curves(job_id, sites):
"""Compute a mean hazard curve for each site in the list
using as input all the pre-computed curves for different realizations."""
keys = []
for site in sites:
mean_curve = {
"site_lon": site.longitude,
"site_lat": site.latitude,
"curve": _reconstruct_curve_list_from(compute_mean_curve(curves_at(job_id, site))),
}
key = kvs.tokens.mean_hazard_curve_key(job_id, site)
keys.append(key)
LOG.debug("MEAN curve at %s is %s" % (key, mean_curve))
kvs.set_value_json_encoded(key, mean_curve)
return keys
def compute_quantile_hazard_curves(job_id, sites, realizations, quantiles):
"""Compute a quantile hazard curve for each site in the list
using as input all the pre-computed curves for different realizations.
"""
LOG.debug("[QUANTILE_HAZARD_CURVES] List of quantiles is %s" % quantiles)
keys = []
for site in sites:
poes = poes_at(job_id, site, realizations)
for quantile in quantiles:
quantile_poes = compute_quantile_curve(poes, quantile)
key = kvs.tokens.quantile_hazard_curve_key(
job_id, site, quantile)
keys.append(key)
kvs.set_value_json_encoded(key, quantile_poes)
return keys
def launch(self):
""" Based on the behaviour specified in the configuration, mix in the
correct behaviour for the tasks and then execute them.
"""
output_dir = os.path.join(self.base_path, self['OUTPUT_DIR'])
if not os.path.exists(output_dir):
os.makedirs(output_dir)
results = []
self._partition()
for (key, mixin) in Mixin.ordered_mixins():
if key.upper() not in self.sections:
continue
with Mixin(self, mixin, key=key):
# The mixin defines a preload decorator to handle the needed
# data for the tasks and decorates _execute(). the mixin's
# _execute() method calls the expected tasks.
LOG.debug("Job %s Launching %s for %s" % (self.id, mixin, key))
results.extend(self.execute())
return results
def compute_quantile_hazard_maps(job_id, sites, quantiles, imls, poes):
"""Compute quantile hazard maps using as input all the
pre computed quantile hazard curves.
"""
LOG.debug("[QUANTILE_HAZARD_MAPS] List of POEs is %s" % poes)
LOG.debug("[QUANTILE_HAZARD_MAPS] List of quantiles is %s" % quantiles)
keys = []
for quantile in quantiles:
for site in sites:
quantile_poes = kvs.get_value_json_decoded(kvs.tokens.quantile_hazard_curve_key(job_id, site, quantile))
interpolate = build_interpolator(quantile_poes, imls, site)
for poe in poes:
key = kvs.tokens.quantile_hazard_map_key(job_id, site, poe, quantile)
keys.append(key)
kvs.set_value_json_encoded(key, interpolate(poe))
return keys
def execute(self):
"""Execute the logic of this mixin."""
if not self.has("AGGREGATE_LOSS_CURVE"):
LOG.debug("AGGREGATE_LOSS_CURVE parameter not specified, " \
"skipping aggregate loss curve computation...")
return []
aggregate_loss_curve = prob.AggregateLossCurve.from_kvs(self.id)
path = os.path.join(self.base_path,
self.params["OUTPUT_DIR"], filename(self.id))
plotter = curve.CurvePlot(path)
plotter.write(for_plotting(aggregate_loss_curve.compute(
self.tses(), self.time_span())), autoscale_y=False)
plotter.close()
LOG.debug("Aggregate loss curve stored at %s" % path)
return [path] # why?
def append(self, gmfs, asset):
"""Add the losses distribution identified by the given GMFs
and asset to the set used to compute the aggregate curve."""
if self.empty:
self._initialize_parameters(gmfs)
assert gmfs["TimeSpan"] == self._time_span
assert gmfs["TSES"] == self._tses
assert len(gmfs["IMLs"]) == self._gmfs_length
if asset["vulnerabilityFunctionReference"] in self.vuln_model:
loss_ratios = compute_loss_ratios(self.vuln_model[
asset["vulnerabilityFunctionReference"]], gmfs,
self.epsilon_provider, asset)
self.distribution.append(loss_ratios * asset["assetValue"])
else:
LOG.debug("Unknown vulnerability function %s, asset %s will " \
"not be included in the aggregate computation"
% (asset["vulnerabilityFunctionReference"],
asset["assetID"]))
def compute_quantile_hazard_maps(job_id, sites, quantiles, imls, poes):
"""Compute quantile hazard maps using as input all the
pre computed quantile hazard curves.
"""
LOG.debug("[QUANTILE_HAZARD_MAPS] List of POEs is %s" % poes)
LOG.debug("[QUANTILE_HAZARD_MAPS] List of quantiles is %s" % quantiles)
keys = []
for quantile in quantiles:
for site in sites:
quantile_poes = kvs.get_value_json_decoded(
kvs.tokens.quantile_hazard_curve_key(job_id, site, quantile))
interpolate = build_interpolator(quantile_poes, imls, site)
for poe in poes:
key = kvs.tokens.quantile_hazard_map_key(
job_id, site, poe, quantile)
keys.append(key)
kvs.set_value_json_encoded(key, interpolate(poe))
return keys
def add(self, ground_motion_field_set, asset):
"""Compute the losses for the given ground motion field set, and
sum those to the current sum of the losses.
If the asset refers to a vulnerability function that is not
supported by the vulnerability model, the distribution
of the losses is discarded.
:param ground_motion_field_set: the set of ground motion
fields used to compute the loss ratios.
:type ground_motion_field_set: :py:class:`dict` with the following
keys:
**IMLs** - tuple of ground motion fields (float)
:param asset: the asset used to compute the loss ratios and losses.
:type asset: :py:class:`dict` as provided by
:py:class:`openquake.parser.exposure.ExposurePortfolioFile`
"""
if asset["vulnerabilityFunctionReference"] not in self.vuln_model:
LOG.debug("Unknown vulnerability function %s, asset %s will " \
"not be included in the aggregate computation"
% (asset["vulnerabilityFunctionReference"],
asset["assetID"]))
return
vuln_function = self.vuln_model[
asset["vulnerabilityFunctionReference"]]
loss_ratios = self.lr_calculator(
vuln_function, ground_motion_field_set,
self.epsilon_provider, asset)
losses = numpy.array(loss_ratios) * asset["assetValue"]
self.sum_losses(losses)
def convert(input_path, input_module, output_path, output_module):
"""Main conversion method. Currently tooled to run GEM1 parsers
via jpype, which involves setting static properties on the classes
for the input directories. The parsing itself is done in the class
constructor, and output is derived from a writeSources method."""
LOG.info("Starting conversion run...")
jarpath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../lib")
LOG.debug("Jarpath is %s", jarpath)
max_mem = 4000
jpype.startJVM(jpype.getDefaultJVMPath(), "-Djava.ext.dirs=%s" % jarpath, "-Xmx%sM" % max_mem)
input_module.init_paths(input_path, jpype)
root_node = etree.Element(NRML + "SeismicSourceList", nsmap=NSMAP)
# All the GEM1 parsers take a bounding box for the ctor
(latmin, latmax, lonmin, lonmax) = input_module.BOUNDING_BOX
# TODO(JMC): Make this support non-Java input parsers, too
for model, _subdir in input_module.JAVA_MODELS:
outfile = os.path.join(output_path, model+"-foo.xml")
if os.path.exists(outfile):
LOG.info("Output exists, skipping generation of %s", outfile)
# continue
java_class = jpype.JClass(model)
input_parser = java_class(latmin, latmax, lonmin, lonmax)
LOG.debug("Loaded a %s parser with %s sources",
model, input_parser.getNumSources())
#print(dir(input_parser))
#print dir(input_parser.srcDataList[0])
for source in input_parser.srcDataList:
source_node = serialize_source(source, root_node)
LOG.debug("Writing output to %s", outfile)
#file_writer_class = jpype.JClass("java.io.FileWriter")
#input_parser.writeSources2KMLfile(
# file_writer_class(outfile))
et = etree.ElementTree(root_node)
et.write(outfile, pretty_print=True)
LOG.info("Finished conversion run.")
def compute_quantile_hazard_maps(job):
"""Compute quantile hazard maps using as input all the
pre computed quantile hazard curves.
The POES_HAZARD_MAPS parameter in the configuration file specifies
all the values used in the computation.
"""
quantiles = _extract_values_from_config(job, QUANTILE_PARAM_NAME)
poes = _extract_values_from_config(job, POES_PARAM_NAME)
LOG.debug("[QUANTILE_HAZARD_MAPS] List of POEs is %s" % poes)
LOG.debug("[QUANTILE_HAZARD_MAPS] List of quantiles is %s" % quantiles)
keys = []
for quantile in quantiles:
# get all the pre computed quantile curves
pattern = "%s*%s*%s" % (kvs.tokens.QUANTILE_HAZARD_CURVE_KEY_TOKEN,
job.id, quantile)
quantile_curves = kvs.mget_decoded(pattern)
LOG.debug("[QUANTILE_HAZARD_MAPS] Found %s pre computed " \
"quantile curves for quantile %s"
% (len(quantile_curves), quantile))
for poe in poes:
for quantile_curve in quantile_curves:
site = shapes.Site(quantile_curve["site_lon"],
quantile_curve["site_lat"])
key = kvs.tokens.quantile_hazard_map_key(
job.id, site, poe, quantile)
keys.append(key)
_store_iml_for(quantile_curve, key, job, poe)
return keys
