class BaseProbabilisticRupture(with_metaclass(abc.ABCMeta, Rupture)):
"""
Base class for a probabilistic rupture, that is a :class:`Rupture`
associated with a temporal occurrence model defining probability of
rupture occurrence in a certain time span.
"""
@abc.abstractmethod
def get_probability_no_exceedance(self, poes):
"""
Compute and return the probability that in the time span for which the
rupture is defined, the rupture itself never generates a ground motion
value higher than a given level at a given site.
Such calculation is performed starting from the conditional probability
that an occurrence of the current rupture is producing a ground motion
value higher than the level of interest at the site of interest.
The actual formula used for such calculation depends on the temporal
occurrence model the rupture is associated with.
The calculation can be performed for multiple intensity measure levels
and multiple sites in a vectorized fashion.
:param poes:
2D numpy array containing conditional probabilities the the a
rupture occurrence causes a ground shaking value exceeding a
ground motion level at a site. First dimension represent sites,
second dimension intensity measure levels. ``poes`` can be obtained
calling the :meth:`method
<openquake.hazardlib.gsim.base.GroundShakingIntensityModel.get_poes>`.
"""
@abc.abstractmethod
def sample_number_of_occurrences(self):
"""
class _IMT(with_metaclass(IMTMeta, tuple)):
"""
Base class for intensity measure type.
Subclasses may define class attribute ``_fields`` as a tuple with names
of parameters the specific intensity measure type requires (if there
are any).
"""
_fields = ()
def __new__(cls, sa_period=None, sa_damping=None):
return tuple.__new__(cls, (cls.__name__, sa_period, sa_damping))
def __getnewargs__(self):
return tuple(getattr(self, field) for field in self._fields)
def __str__(self):
if self[0] == 'SA':
return 'SA(%s)' % self[1]
return self[0]
def __lt__(self, other):
return str(self) < str(other)
def __repr__(self):
return '%s(%s)' % (type(self).__name__, ', '.join(
'%s=%s' % (field, getattr(self, field))
for field in type(self)._fields))
class BaseMSR(with_metaclass(abc.ABCMeta)):
"""
A base class for Magnitude-Area Scaling Relationship.
Allows calculation of rupture area from magnitude.
"""
@abc.abstractmethod
def get_median_area(self, mag, rake):
"""
Return median area (in square km) from magnitude ``mag`` and ``rake``.
To be overridden by subclasses.
:param mag:
Moment magnitude (Mw).
:param rake:
Rake angle (the rupture propagation direction) in degrees,
from -180 to 180.
"""
def __eq__(self, other):
"""
Two instances of the same class are considered equal
"""
return self.__class__ is other.__class__
def __repr__(self):
"""
Returns the name of the class
"""
return "<%s>" % self.__class__.__name__
class SMTimeSeriesReader(with_metaclass(abc.ABCMeta)):
"""
Abstract base class for a reader of a ground motion time series
"""
def __init__(self, input_files, folder_name=None, units="cm/s/s"):
"""
Instantiate and conduct folder checks
"""
self.input_files = []
for fname in input_files:
if folder_name:
filename = os.path.join(folder_name, fname)
if os.path.exists(filename):
self.input_files.append(filename)
else:
if os.path.exists(fname):
self.input_files.append(fname)
self.time_step = None
self.number_steps = None
self.units = units
self.metadata = None
@abc.abstractmethod
def parse_records(self, record=None):
"""
class LiteralNode(with_metaclass(MetaLiteralNode, Node)):
"""
Subclasses should define a non-empty dictionary of validators.
"""
validators = {} # to be overridden in subclasses
def __init__(self,
fulltag,
attrib=None,
text=None,
nodes=None,
lineno=None):
validators = self.__class__.validators
tag = striptag(fulltag)
if tag in validators:
# try to cast the node, if the tag is known
assert not nodes, 'You cannot cast a composite node: %s' % nodes
try:
text = validators[tag](text, **attrib)
assert text is not None
except Exception as exc:
raise ValueError('Could not convert %s->%s: %s, line %s' %
(tag, validators[tag].__name__, exc, lineno))
elif attrib:
# cast the attributes
for n, v in attrib.items():
if n in validators:
try:
attrib[n] = validators[n](v)
except Exception as exc:
raise ValueError(
'Could not convert %s->%s: %s, line %s' %
(n, validators[n].__name__, exc, lineno))
super(LiteralNode, self).__init__(fulltag, attrib, text, nodes, lineno)
class BaseMSRSigma(with_metaclass(abc.ABCMeta, BaseMSR)):
"""
Extends :class:`BaseMSR` and allows to include uncertainties (sigma) in
rupture area estimation.
"""
@abc.abstractmethod
def get_std_dev_area(self, mag, rake):
"""
class BaseCorrelationModel(with_metaclass(abc.ABCMeta)):
"""
Base class for correlation models for spatially-distributed ground-shaking
intensities.
"""
@abc.abstractmethod
def get_lower_triangle_correlation_matrix(self, sites, imt):
"""
Get lower-triangle matrix as a result of Cholesky-decomposition
of correlation matrix.
The resulting matrix should have zeros on values above
the main diagonal.
The actual implementations of :class:`BaseCorrelationModel` interface
might calculate the matrix considering site collection and IMT (like
:class:`JB2009CorrelationModel` does) or might have it pre-constructed
for a specific site collection and IMT, in which case they will need
to make sure that parameters to this function match parameters that
were used to pre-calculate decomposed correlation matrix.
:param sites:
:class:`~openquake.hazardlib.site.SiteCollection` to create
correlation matrix for.
:param imt:
Intensity measure type object, see :mod:`openquake.hazardlib.imt`.
"""
def apply_correlation(self, sites, imt, residuals):
"""
Apply correlation to randomly sampled residuals.
:param sites:
:class:`~openquake.hazardlib.site.SiteCollection` residuals were
sampled for.
:param imt:
Intensity measure type object, see :mod:`openquake.hazardlib.imt`.
:param residuals:
2d numpy array of sampled residuals, where first dimension
represents sites (the length as ``sites`` parameter) and
second one represents different realizations (samples).
:returns:
Array of the same structure and semantics as ``residuals``
but with correlations applied.
"""
# intra-event residual for a single relization is a product
# of lower-triangle decomposed correlation matrix and vector
# of N random numbers (where N is equal to number of sites).
# we need to do that multiplication once per realization
# with the same matrix and different vectors.
corma = self.get_lower_triangle_correlation_matrix(sites, imt)
return numpy.dot(corma, residuals)
class BaseContext(with_metaclass(abc.ABCMeta)):
"""
Base class for context object.
"""
def __eq__(self, other):
"""
Return True if ``other`` has same attributes with same values.
"""
if isinstance(other, self.__class__):
if self._slots_ == other._slots_:
self_other = [
numpy.all(
getattr(self, s, None) == getattr(other, s, None))
for s in self._slots_
]
return numpy.all(self_other)
return False
class SMDatabaseReader(with_metaclass(abc.ABCMeta)):
"""
Abstract base class for strong motion database parser
"""
def __init__(self, db_id, db_name, filename, record_folder=None):
"""
Instantiate and conduct folder checks
"""
self.id = db_id
self.name = db_name
self.filename = filename
self.database = None
if record_folder:
self.record_folder = record_folder
else:
self.record_folder = self.filename
@abc.abstractmethod
def parse(self):
"""
class SMSpectraReader(with_metaclass(abc.ABCMeta)):
"""
Abstract Base Class for a reader of a ground motion spectra record
"""
def __init__(self, input_files, folder_name=None):
"""
Intantiate with basic file checks
"""
self.input_files = []
for fname in input_files:
if folder_name:
filename = os.path.join(folder_name, fname)
if os.path.exists(filename):
self.input_files.append(filename)
else:
if os.path.exists(fname):
self.input_files.append(fname)
@abc.abstractmethod
def parse_spectra(self):
"""
class BaseASR(with_metaclass(abc.ABCMeta)):
"""
A base class for Area-Magnitude Scaling Relationship.
Allows calculation of rupture magnitude from area.
"""
@abc.abstractmethod
def get_median_mag(self, area, rake):
"""
Return median magnitude (Mw) given the area and rake.
:param area:
Area in square km.
:param rake:
Rake angle (the rupture propagation direction) in degrees,
from -180 to 180.
"""
def __repr__(self):
"""
Returns the name of the class
"""
return "<%s>" % self.__class__.__name__
class ParametricSeismicSource(with_metaclass(abc.ABCMeta, BaseSeismicSource)):
"""
Parametric Seismic Source generates earthquake ruptures from source
parameters, and associated probabilities of occurrence are defined through
a magnitude frequency distribution and a temporal occurrence model.
:param mfd:
Magnitude-Frequency distribution for the source.
See :mod:`openquake.hazardlib.mfd`.
:param rupture_mesh_spacing:
The desired distance between two adjacent points in source's
ruptures' mesh, in km. Mainly this parameter allows to balance
the trade-off between time needed to compute the :meth:`distance
<openquake.hazardlib.geo.surface.base.BaseSurface.get_min_distance>`
between the rupture surface and a site and the precision of that
computation.
:param magnitude_scaling_relationship:
Instance of subclass of
:class:`openquake.hazardlib.scalerel.base.BaseMSR` to
describe how does the area of the rupture depend on magnitude and rake.
:param rupture_aspect_ratio:
Float number representing how much source's ruptures are more wide
than tall. Aspect ratio of 1 means ruptures have square shape,
value below 1 means ruptures stretch vertically more than horizontally
and vice versa.
:param temporal_occurrence_model:
Instance of
:class:`openquake.hazardlib.tom.PoissonTOM` defining temporal
occurrence model for calculating rupture occurrence probabilities
:raises ValueError:
If either rupture aspect ratio or rupture mesh spacing is not positive
(if not None).
"""
_slots_ = BaseSeismicSource._slots_ + '''mfd rupture_mesh_spacing
magnitude_scaling_relationship rupture_aspect_ratio
temporal_occurrence_model'''.split()
def __init__(self, source_id, name, tectonic_region_type, mfd,
rupture_mesh_spacing, magnitude_scaling_relationship,
rupture_aspect_ratio, temporal_occurrence_model):
super().__init__(source_id, name, tectonic_region_type)
if rupture_mesh_spacing is not None and not rupture_mesh_spacing > 0:
raise ValueError('rupture mesh spacing must be positive')
if rupture_aspect_ratio is not None and not rupture_aspect_ratio > 0:
raise ValueError('rupture aspect ratio must be positive')
self.mfd = mfd
self.rupture_mesh_spacing = rupture_mesh_spacing
self.magnitude_scaling_relationship = magnitude_scaling_relationship
self.rupture_aspect_ratio = rupture_aspect_ratio
self.temporal_occurrence_model = temporal_occurrence_model
def get_annual_occurrence_rates(self, min_rate=0):
"""
Get a list of pairs "magnitude -- annual occurrence rate".
The list is taken from assigned MFD object
(see :meth:`openquake.hazardlib.mfd.base.BaseMFD.get_annual_occurrence_rates`)
with simple filtering by rate applied.
:param min_rate:
A non-negative value to filter magnitudes by minimum annual
occurrence rate. Only magnitudes with rates greater than that
are included in the result list.
:returns:
A list of two-item tuples -- magnitudes and occurrence rates.
"""
return [(mag, occ_rate)
for (mag, occ_rate) in self.mfd.get_annual_occurrence_rates()
if min_rate is None or occ_rate > min_rate]
def get_min_max_mag(self):
"""
Get the minimum and maximum magnitudes of the ruptures generated
by the source from the underlying MFD.
"""
return self.mfd.get_min_max_mag()
def __repr__(self):
"""
String representation of a source, displaying the source class name
and the source id.
"""
return '<%s %s>' % (self.__class__.__name__, self.source_id)
class BaseLogicTree(with_metaclass(abc.ABCMeta)):
"""
Common code for logic tree readers, parsers and verifiers --
:class:`GMPELogicTree` and :class:`SourceModelLogicTree`.
:param filename:
Full pathname of logic tree file
:param validate:
Boolean indicating whether or not the tree should be validated
while parsed. This should be set to ``True`` on initial load
of the logic tree (before importing it to the database) and
to ``False`` on workers side (when loaded from the database).
:raises ParsingError:
If logic tree file or any of the referenced files is unable to read
or parse.
:raises ValidationError:
If logic tree file has a logic error, which can not be prevented
by xml schema rules (like referencing sources with missing id).
"""
NRML = nrml.NAMESPACE
FILTERS = ('applyToTectonicRegionType', 'applyToSources',
'applyToSourceType')
_xmlschema = None
def __init__(self, filename, validate=True, seed=0, num_samples=0):
self.filename = filename
self.basepath = os.path.dirname(filename)
self.seed = seed
self.num_samples = num_samples
self.branches = {}
self.open_ends = set()
try:
tree = parse(filename)
except etree.ParseError as exc:
# Wrap etree parsing exception to :exc:`ParsingError`.
raise ParsingError(self.filename, str(exc))
[tree] = tree.findall('{%s}logicTree' % self.NRML)
self.root_branchset = None
self.parse_tree(tree, validate)
def skip_branchset_condition(self, attrs):
"""
Override in subclasses to skip a branchset depending on a
condition on its attributes.
:param attrs: a dictionary with the attributes of the branchset
"""
return False
def parse_tree(self, tree_node, validate):
"""
Parse the whole tree and point ``root_branchset`` attribute
to the tree's root. If ``validate`` is set to ``True``, calls
:meth:`validate_tree` when done. Also passes that value
to :meth:`parse_branchinglevel`.
"""
levels = tree_node.findall('{%s}logicTreeBranchingLevel' % self.NRML)
for depth, branchinglevel_node in enumerate(levels):
self.parse_branchinglevel(branchinglevel_node, depth, validate)
if validate:
self.validate_tree(tree_node, self.root_branchset)
def parse_branchinglevel(self, branchinglevel_node, depth, validate):
"""
Parse one branching level.
:param branchinglevel_node:
``etree.Element`` object with tag "logicTreeBranchingLevel".
:param depth:
The sequential number of this branching level, based on 0.
:param validate:
Whether or not the branching level, its branchsets and their
branches should be validated.
Enumerates children branchsets and call :meth:`parse_branchset`,
:meth:`validate_branchset`, :meth:`parse_branches` and finally
:meth:`apply_branchset` for each.
Keeps track of "open ends" -- the set of branches that don't have
any child branchset on this step of execution. After processing
of every branching level only those branches that are listed in it
can have child branchsets (if there is one on the next level).
"""
new_open_ends = set()
branchsets = branchinglevel_node.findall('{%s}logicTreeBranchSet' %
self.NRML)
for number, branchset_node in enumerate(branchsets):
if self.skip_branchset_condition(branchset_node.attrib):
continue
branchset = self.parse_branchset(branchset_node, depth, number,
validate)
self.parse_branches(branchset_node, branchset, validate)
if self.root_branchset is None: # not set yet
self.root_branchset = branchset
else:
self.apply_branchset(branchset_node, branchset)
for branch in branchset.branches:
new_open_ends.add(branch)
self.open_ends.clear()
self.open_ends.update(new_open_ends)
def parse_branchset(self, branchset_node, depth, number, validate):
"""
Create :class:`BranchSet` object using data in ``branchset_node``.
:param branchset_node:
``etree.Element`` object with tag "logicTreeBranchSet".
:param depth:
The sequential number of branchset's branching level, based on 0.
:param number:
Index number of this branchset inside branching level, based on 0.
:param validate:
Whether or not filters defined in branchset and the branchset
itself should be validated.
:returns:
An instance of :class:`BranchSet` with filters applied but with
no branches (they're attached in :meth:`parse_branches`).
"""
uncertainty_type = branchset_node.get('uncertaintyType')
filters = dict((filtername, branchset_node.get(filtername))
for filtername in self.FILTERS
if filtername in branchset_node.attrib)
if validate:
self.validate_filters(branchset_node, uncertainty_type, filters)
filters = self.parse_filters(branchset_node, uncertainty_type, filters)
branchset = BranchSet(uncertainty_type, filters)
if validate:
self.validate_branchset(branchset_node, depth, number, branchset)
return branchset
def parse_branches(self, branchset_node, branchset, validate):
"""
Create and attach branches at ``branchset_node`` to ``branchset``.
:param branchset_node:
Same as for :meth:`parse_branchset`.
:param branchset:
An instance of :class:`BranchSet`.
:param validate:
Whether or not branches' uncertainty values should be validated.
Checks that each branch has :meth:`valid <validate_uncertainty_value>`
value, unique id and that all branches have total weight of 1.0.
:return:
``None``, all branches are attached to provided branchset.
"""
weight_sum = 0
branches = branchset_node.findall('{%s}logicTreeBranch' % self.NRML)
for branchnode in branches:
weight = branchnode.find('{%s}uncertaintyWeight' % self.NRML).text
weight = Decimal(weight.strip())
weight_sum += weight
value_node = branchnode.find('{%s}uncertaintyModel' % self.NRML)
if validate:
self.validate_uncertainty_value(value_node, branchset,
value_node.text.strip())
value = self.parse_uncertainty_value(value_node, branchset,
value_node.text.strip())
branch_id = branchnode.get('branchID')
branch = Branch(branch_id, weight, value)
if branch_id in self.branches:
raise ValidationError(branchnode, self.filename,
"branchID %r is not unique" % branch_id)
self.branches[branch_id] = branch
branchset.branches.append(branch)
if weight_sum != 1.0:
raise ValidationError(branchset_node, self.filename,
"branchset weights don't sum up to 1.0")
def apply_branchset(self, branchset_node, branchset):
# pylint: disable=W0613
"""
Apply ``branchset`` to all "open end" branches.
See :meth:`parse_branchinglevel`.
:param branchset_node:
Same as for :meth:`parse_branchset`.
:param branchset:
An instance of :class:`BranchSet` to make it child
for "open-end" branches.
Can be overridden by subclasses if they want to apply branchests
to branches selectively.
"""
for branch in self.open_ends:
branch.child_branchset = branchset
def validate_tree(self, tree_node, root_branchset):
"""
Check the whole parsed tree for consistency and sanity.
Can be overriden by subclasses. Base class implementation does nothing.
:param tree_node:
``etree.Element`` object with tag "logicTree".
:param root_branchset:
An instance of :class:`BranchSet` which is about to become
the root branchset for this tree.
"""
def sample_path(self, rnd):
"""
Return the model name and a list of branch ids.
:param int random_seed: the seed used for the sampling
"""
branchset = self.root_branchset
branch_ids = []
while branchset is not None:
[branch] = sample(branchset.branches, 1, rnd)
branch_ids.append(branch.branch_id)
branchset = branch.child_branchset
modelname = self.root_branchset.get_branch_by_id(branch_ids[0]).value
return modelname, branch_ids
def __iter__(self):
"""
Yield Realization tuples. Notice that
weight is not None only when the number_of_logic_tree_samples
is 0. In that case a full enumeration is performed, otherwise
a random sampling is performed.
"""
if self.num_samples:
# random sampling of the logic tree
rnd = random.Random(self.seed)
weight = 1. / self.num_samples
for _ in range(self.num_samples):
name, sm_lt_path = self.sample_path(rnd)
yield Realization(name, weight, tuple(sm_lt_path), None,
tuple(sm_lt_path))
else: # full enumeration
for weight, smlt_path in self.root_branchset.enumerate_paths():
name = smlt_path[0].value
smlt_branch_ids = [branch.branch_id for branch in smlt_path]
yield Realization(name, weight, tuple(smlt_branch_ids), None,
tuple(smlt_branch_ids))
@abc.abstractmethod
def parse_uncertainty_value(self, node, branchset, value):
"""
Do any kind of type conversion or adaptation on the uncertainty value.
Abstract method, must be overridden by subclasses.
Parameters are the same as for :meth:`validate_uncertainty_value`.
:return:
Something to replace ``value`` as the uncertainty value.
"""
@abc.abstractmethod
def validate_uncertainty_value(self, node, branchset, value):
"""
Check the value ``value`` for correctness to be set for one
of branchset's branches.
Abstract method, must be overridden by subclasses.
:param node:
``etree.Element`` object with tag "uncertaintyModel" (the one
that contains the subject value).
:param branchset:
An instance of :class:`BranchSet` which will have the branch
with provided value attached once it's validated.
:param value:
The actual value to be checked. Type depends on branchset's
uncertainty type.
"""
@abc.abstractmethod
def parse_filters(self, branchset_node, uncertainty_type, filters):
"""
Do any kind of type conversion or adaptation on the filters.
Abstract method, must be overriden by subclasses.
Parameters are the same as for :meth:`validate_filters`.
:return:
The filters dictionary to replace the original.
"""
@abc.abstractmethod
def validate_filters(self, node, uncertainty_type, filters):
"""
Check that filters ``filters`` are valid for given uncertainty type.
Abstract method, must be overriden by subclasses.
:param node:
``etree.Element`` object with tag "logicTreeBranchSet".
:param uncertainty_type:
String specifying the uncertainty type.
See the list in :class:`BranchSet`.
:param filters:
Filters dictionary.
"""
@abc.abstractmethod
def validate_branchset(self, branchset_node, depth, number, branchset):
"""
class BaseCalculator(with_metaclass(abc.ABCMeta)):
"""
Abstract base class for all calculators.
:param oqparam: OqParam object
:param monitor: monitor object
:param calc_id: numeric calculation ID
"""
sitemesh = datastore.persistent_attribute('sitemesh')
sitecol = datastore.persistent_attribute('sitecol')
rlzs_assoc = datastore.persistent_attribute('rlzs_assoc')
realizations = datastore.persistent_attribute('realizations')
assets_by_site = datastore.persistent_attribute('assets_by_site')
assetcol = datastore.persistent_attribute('assetcol')
cost_types = datastore.persistent_attribute('cost_types')
taxonomies = datastore.persistent_attribute('taxonomies')
job_info = datastore.persistent_attribute('job_info')
source_chunks = datastore.persistent_attribute('source_chunks')
source_pre_info = datastore.persistent_attribute('source_pre_info')
performance = datastore.persistent_attribute('performance')
csm = datastore.persistent_attribute('composite_source_model')
pre_calculator = None # to be overridden
is_stochastic = False # True for scenario and event based calculators
def __init__(self, oqparam, monitor=DummyMonitor(), calc_id=None,
persistent=True):
self.monitor = monitor
if persistent:
self.datastore = datastore.DataStore(calc_id)
else:
self.datastore = general.AccumDict()
self.datastore.hdf5 = {}
self.datastore.attrs = {}
self.datastore.export_dir = oqparam.export_dir
self.oqparam = oqparam
self.persistent = persistent
def save_params(self, **kw):
"""
Update the current calculation parameters
"""
vars(self.oqparam).update(kw)
for name, val in self.oqparam.to_params():
self.datastore.attrs[name] = val
self.datastore.attrs['oqlite_version'] = repr(__version__)
self.datastore.hdf5.flush()
def run(self, pre_execute=True, clean_up=True, concurrent_tasks=None,
**kw):
"""
Run the calculation and return the exported outputs.
"""
if concurrent_tasks is not None:
self.oqparam.concurrent_tasks = concurrent_tasks
self.save_params(**kw)
exported = {}
try:
if pre_execute:
with self.monitor('pre_execute', autoflush=True):
self.pre_execute()
with self.monitor('execute', autoflush=True):
result = self.execute()
with self.monitor('post_execute', autoflush=True):
self.post_execute(result)
with self.monitor('export', autoflush=True):
exported = self.export()
except:
if kw.get('pdb'): # post-mortem debug
tb = sys.exc_info()[2]
traceback.print_exc(tb)
pdb.post_mortem(tb)
else:
logging.critical('', exc_info=True)
raise
# don't cleanup if there is a critical error, otherwise
# there will likely be a cleanup error covering the real one
if clean_up:
self.clean_up()
return exported
def core_func(*args):
"""
Core routine running on the workers.
"""
raise NotImplementedError
@abc.abstractmethod
def pre_execute(self):
"""
Initialization phase.
"""
@abc.abstractmethod
def execute(self):
"""
Execution phase. Usually will run in parallel the core
function and return a dictionary with the results.
"""
@abc.abstractmethod
def post_execute(self, result):
"""
Post-processing phase of the aggregated output. It must be
overridden with the export code. It will return a dictionary
of output files.
"""
def export(self, exports=None):
"""
Export all the outputs in the datastore in the given export formats.
:returns: dictionary output_key -> sorted list of exported paths
"""
exported = {}
individual_curves = self.oqparam.individual_curves
fmts = exports.split(',') if exports else self.oqparam.exports
for fmt in fmts:
if not fmt:
continue
for key in self.datastore: # top level keys
if 'rlzs' in key and not individual_curves:
continue # skip individual curves
ekey = (key, fmt)
if ekey not in export.export: # non-exportable output
continue
exported[ekey] = export.export(ekey, self.datastore)
logging.info('exported %s: %s', key, exported[ekey])
return exported
def clean_up(self):
"""
Collect the realizations and the monitoring information,
then close the datastore.
"""
if 'rlzs_assoc' in self.datastore:
self.realizations = numpy.array(
[(r.uid, r.weight) for r in self.rlzs_assoc.realizations],
rlz_dt)
performance = self.monitor.collect_performance()
if performance is not None:
self.performance = performance
class BaseCalculator(with_metaclass(abc.ABCMeta)):
"""
Abstract base class for all calculators.
:param oqparam: OqParam object
:param monitor: monitor object
:param calc_id: numeric calculation ID
"""
sitemesh = datastore.persistent_attribute('sitemesh')
sitecol = datastore.persistent_attribute('sitecol')
etags = datastore.persistent_attribute('etags')
assetcol = datastore.persistent_attribute('assetcol')
cost_types = datastore.persistent_attribute('cost_types')
job_info = datastore.persistent_attribute('job_info')
performance = datastore.persistent_attribute('performance')
csm = datastore.persistent_attribute('composite_source_model')
pre_calculator = None # to be overridden
is_stochastic = False # True for scenario and event based calculators
@property
def taxonomies(self):
return self.datastore['assetcol/taxonomies'].value
def __init__(self, oqparam, monitor=Monitor(), calc_id=None):
self.monitor = monitor
self.datastore = datastore.DataStore(calc_id)
self.monitor.calc_id = self.datastore.calc_id
self.monitor.hdf5path = self.datastore.hdf5path
self.datastore.export_dir = oqparam.export_dir
self.oqparam = oqparam
def save_params(self, **kw):
"""
Update the current calculation parameters and save engine_version
"""
vars(self.oqparam).update(engine_version=__version__, **kw)
self.datastore['oqparam'] = self.oqparam # save the updated oqparam
self.datastore.flush()
def set_log_format(self):
"""Set the format of the root logger"""
fmt = '[%(asctime)s #{} %(levelname)s] %(message)s'.format(
self.datastore.calc_id)
for handler in logging.root.handlers:
handler.setFormatter(logging.Formatter(fmt))
def run(self, pre_execute=True, concurrent_tasks=None, close=True, **kw):
"""
Run the calculation and return the exported outputs.
"""
self.close = close
self.set_log_format()
if logversion: # make sure this is logged only once
logging.info('Using engine version %s', __version__)
logversion.pop()
if (concurrent_tasks is not None
and concurrent_tasks != OqParam.concurrent_tasks.default):
self.oqparam.concurrent_tasks = concurrent_tasks
self.save_params(**kw)
exported = {}
try:
if pre_execute:
self.pre_execute()
result = self.execute()
self.post_execute(result)
exported = self.export(kw.get('exports', ''))
except KeyboardInterrupt:
pids = ' '.join(str(p.pid) for p in executor._processes)
sys.stderr.write(
'You can manually kill the workers with kill %s\n' % pids)
raise
except:
if kw.get('pdb'): # post-mortem debug
tb = sys.exc_info()[2]
traceback.print_exc(tb)
pdb.post_mortem(tb)
else:
logging.critical('', exc_info=True)
raise
self.clean_up()
return exported
def core_task(*args):
"""
Core routine running on the workers.
"""
raise NotImplementedError
@abc.abstractmethod
def pre_execute(self):
"""
Initialization phase.
"""
@abc.abstractmethod
def execute(self):
"""
Execution phase. Usually will run in parallel the core
function and return a dictionary with the results.
"""
@abc.abstractmethod
def post_execute(self, result):
"""
Post-processing phase of the aggregated output. It must be
overridden with the export code. It will return a dictionary
of output files.
"""
def export(self, exports=None):
"""
Export all the outputs in the datastore in the given export formats.
:returns: dictionary output_key -> sorted list of exported paths
"""
# avoid circular imports
from openquake.commonlib.export import export as exp
exported = {}
individual_curves = self.oqparam.individual_curves
if exports and isinstance(exports, tuple):
fmts = exports
elif exports: # is a string
fmts = exports.split(',')
else: # use passed values
fmts = self.oqparam.exports
for fmt in fmts:
if not fmt:
continue
keys = set(self.datastore)
if (self.oqparam.uniform_hazard_spectra
and not self.oqparam.hazard_maps):
# do not export the hazard maps, even if they are there
keys.remove('hmaps')
for key in sorted(keys): # top level keys
if 'rlzs' in key and not individual_curves:
continue # skip individual curves
ekey = (key, fmt)
if ekey not in exp: # non-exportable output
continue
with self.monitor('export'):
exported[ekey] = exp(ekey, self.datastore)
logging.info('exported %s: %s', key, exported[ekey])
# special case for uhs which is a view
if (self.oqparam.uniform_hazard_spectra
and 'hmaps' in self.datastore):
ekey = ('uhs', fmt)
exported[ekey] = exp(ekey, self.datastore)
logging.info('exported %s: %s', key, exported[ekey])
return exported
def clean_up(self):
"""
Collect the realizations and the monitoring information,
then close the datastore.
"""
if 'hcurves' in self.datastore:
self.datastore.set_nbytes('hcurves')
if 'hmaps' in self.datastore:
self.datastore.set_nbytes('hmaps')
self.datastore.flush()
if self.close: # in the engine we close later
try:
self.datastore.close()
except RuntimeError: # there could be a mysterious HDF5 error
logging.warn('', exc_info=True)
class BaseCalculator(with_metaclass(abc.ABCMeta)):
"""
Abstract base class for all calculators.
:param oqparam: OqParam object
:param monitor: monitor object
:param calc_id: numeric calculation ID
"""
sitemesh = datastore.persistent_attribute('sitemesh')
sitecol = datastore.persistent_attribute('sitecol')
rlzs_assoc = datastore.persistent_attribute('rlzs_assoc')
realizations = datastore.persistent_attribute('realizations')
assets_by_site = datastore.persistent_attribute('assets_by_site')
assetcol = datastore.persistent_attribute('assetcol')
cost_types = datastore.persistent_attribute('cost_types')
taxonomies = datastore.persistent_attribute('taxonomies')
job_info = datastore.persistent_attribute('job_info')
source_chunks = datastore.persistent_attribute('source_chunks')
performance = datastore.persistent_attribute('performance')
csm = datastore.persistent_attribute('composite_source_model')
pre_calculator = None # to be overridden
is_stochastic = False # True for scenario and event based calculators
def __init__(self, oqparam, monitor=DummyMonitor(), calc_id=None):
self.monitor = monitor
self.datastore = datastore.DataStore(calc_id)
self.monitor.hdf5path = self.datastore.hdf5path
self.datastore.export_dir = oqparam.export_dir
self.oqparam = oqparam
def save_params(self, **kw):
"""
Update the current calculation parameters
"""
vars(self.oqparam).update(kw)
for name, val in self.oqparam.to_params():
self.datastore.attrs[name] = val
self.datastore.attrs['oqlite_version'] = repr(__version__)
self.datastore.hdf5.flush()
def set_log_format(self):
"""Set the format of the root logger"""
fmt = '[%(asctime)s #{} %(levelname)s] %(message)s'.format(
self.datastore.calc_id)
for handler in logging.root.handlers:
handler.setFormatter(logging.Formatter(fmt))
def run(self, pre_execute=True, concurrent_tasks=None, **kw):
"""
Run the calculation and return the exported outputs.
"""
self.set_log_format()
if (concurrent_tasks is not None
and concurrent_tasks != OqParam.concurrent_tasks.default):
self.oqparam.concurrent_tasks = concurrent_tasks
self.save_params(**kw)
exported = {}
try:
if pre_execute:
self.pre_execute()
result = self.execute()
self.post_execute(result)
exported = self.export(kw.get('exports', ''))
except KeyboardInterrupt:
pids = ' '.join(str(p.pid) for p in executor._processes)
sys.stderr.write(
'You can manually kill the workers with kill %s\n' % pids)
raise
except:
if kw.get('pdb'): # post-mortem debug
tb = sys.exc_info()[2]
traceback.print_exc(tb)
pdb.post_mortem(tb)
else:
logging.critical('', exc_info=True)
raise
self.clean_up()
return exported
def core_func(*args):
"""
Core routine running on the workers.
"""
raise NotImplementedError
@abc.abstractmethod
def pre_execute(self):
"""
Initialization phase.
"""
@abc.abstractmethod
def execute(self):
"""
Execution phase. Usually will run in parallel the core
function and return a dictionary with the results.
"""
@abc.abstractmethod
def post_execute(self, result):
"""
Post-processing phase of the aggregated output. It must be
overridden with the export code. It will return a dictionary
of output files.
"""
def export(self, exports=None):
"""
Export all the outputs in the datastore in the given export formats.
:returns: dictionary output_key -> sorted list of exported paths
"""
# avoid circular imports
from openquake.commonlib.export import export as exp
exported = {}
individual_curves = self.oqparam.individual_curves
if exports and isinstance(exports, tuple):
fmts = exports
elif exports: # is a string
fmts = exports.split(',')
else: # use passed values
fmts = self.oqparam.exports
for fmt in fmts:
if not fmt:
continue
for key in self.datastore: # top level keys
if 'rlzs' in key and not individual_curves:
continue # skip individual curves
ekey = (key, fmt)
if ekey not in exp: # non-exportable output
continue
exported[ekey] = exp(ekey, self.datastore)
logging.info('exported %s: %s', key, exported[ekey])
return exported
def clean_up(self):
"""
Collect the realizations and the monitoring information,
then close the datastore.
"""
if 'hcurves' in self.datastore:
_set_nbytes('hcurves', self.datastore)
if 'hmaps' in self.datastore:
_set_nbytes('hmaps', self.datastore)
if 'rlzs_assoc' in self.datastore:
rlzs = self.rlzs_assoc.realizations
self.realizations = numpy.array([(r.uid, r.weight) for r in rlzs],
rlz_dt)
class GroundShakingIntensityModel(with_metaclass(MetaGSIM)):
"""
Base class for all the ground shaking intensity models.
A Ground Shaking Intensity Model (GSIM) defines a set of equations
for computing mean and standard deviation of a Normal distribution
representing the variability of an intensity measure (or of its logarithm)
at a site given an earthquake rupture.
This class is not intended to be subclassed directly, instead
the actual GSIMs should subclass either :class:`GMPE` or :class:`IPE`.
Subclasses of both must implement :meth:`get_mean_and_stddevs`
and all the class attributes with names starting from ``DEFINED_FOR``
and ``REQUIRES``.
"""
#: Reference to a
#: :class:`tectonic region type <openquake.hazardlib.const.TRT>` this GSIM
#: is defined for. One GSIM can implement only one tectonic region type.
DEFINED_FOR_TECTONIC_REGION_TYPE = abc.abstractproperty()
#: Set of :mod:`intensity measure types <openquake.hazardlib.imt>`
#: this GSIM can
#: calculate. A set should contain classes from module
#: :mod:`openquake.hazardlib.imt`.
DEFINED_FOR_INTENSITY_MEASURE_TYPES = abc.abstractproperty()
#: Reference to a :class:`intensity measure component type
#: <openquake.hazardlib.const.IMC>` this GSIM can calculate mean
#: and standard
#: deviation for.
DEFINED_FOR_INTENSITY_MEASURE_COMPONENT = abc.abstractproperty()
#: Set of
#: :class:`standard deviation types <openquake.hazardlib.const.StdDev>`
#: this GSIM can calculate.
DEFINED_FOR_STANDARD_DEVIATION_TYPES = abc.abstractproperty()
#: Set of site parameters names this GSIM needs. The set should include
#: strings that match names of the attributes of a :class:`site
#: <openquake.hazardlib.site.Site>` object.
#: Those attributes are then available in the
#: :class:`SitesContext` object with the same names.
REQUIRES_SITES_PARAMETERS = abc.abstractproperty()
#: Set of rupture parameters (excluding distance information) required
#: by GSIM. Supported parameters are:
#:
#: ``mag``
#: Magnitude of the rupture.
#: ``dip``
#: Rupture's surface dip angle in decimal degrees.
#: ``rake``
#: Angle describing the slip propagation on the rupture surface,
#: in decimal degrees. See :mod:`~openquake.hazardlib.geo.nodalplane`
#: for more detailed description of dip and rake.
#: ``ztor``
#: Depth of rupture's top edge in km. See
#: :meth:`~openquake.hazardlib.geo.surface.base.BaseSurface.get_top_edge_depth`.
#:
#: These parameters are available from the :class:`RuptureContext` object
#: attributes with same names.
REQUIRES_RUPTURE_PARAMETERS = abc.abstractproperty()
#: Set of types of distance measures between rupture and sites. Possible
#: values are:
#:
#: ``rrup``
#: Closest distance to rupture surface. See
#: :meth:`~openquake.hazardlib.geo.surface.base.BaseSurface.get_min_distance`.
#: ``rjb``
#: Distance to rupture's surface projection. See
#: :meth:`~openquake.hazardlib.geo.surface.base.BaseSurface.get_joyner_boore_distance`.
#: ``rx``
#: Perpendicular distance to rupture top edge projection.
#: See :meth:`~openquake.hazardlib.geo.surface.base.BaseSurface.get_rx_distance`.
#: ``ry0``
#: Horizontal distance off the end of the rupture measured parallel to
# strike. See:
#: See :meth:`~openquake.hazardlib.geo.surface.base.BaseSurface.get_ry0_distance`.
#: ``rcdpp``
#: Direct point parameter for directivity effect centered on the site- and earthquake-specific
# average DPP used. See:
#: See :meth:`~openquake.hazardlib.source.rupture.ParametricProbabilisticRupture.get_dppvalue`.
#: ``rvolc``
#: Source to site distance passing through surface projection of volcanic zone
#:
#: All the distances are available from the :class:`DistancesContext`
#: object attributes with same names. Values are in kilometers.
REQUIRES_DISTANCES = abc.abstractproperty()
minimum_distance = 0 # can be set by the engine
@abc.abstractmethod
def get_mean_and_stddevs(self, sites, rup, dists, imt, stddev_types):
"""
Calculate and return mean value of intensity distribution and it's
standard deviation.
Method must be implemented by subclasses.
:param sites:
Instance of :class:`openquake.hazardlib.site.SiteCollection`
with parameters of sites
collection assigned to respective values as numpy arrays.
Only those attributes that are listed in class'
:attr:`REQUIRES_SITES_PARAMETERS` set are available.
:param rup:
Instance of :class:`openquake.hazardlib.source.rupture.BaseRupture`
with parameters of a rupture
assigned to respective values. Only those attributes that are
listed in class' :attr:`REQUIRES_RUPTURE_PARAMETERS` set are
available.
:param dists:
Instance of :class:`DistancesContext` with values of distance
measures between the rupture and each site of the collection
assigned to respective values as numpy arrays. Only those
attributes that are listed in class' :attr:`REQUIRES_DISTANCES`
set are available.
:param imt:
An instance (not a class) of intensity measure type.
See :mod:`openquake.hazardlib.imt`.
:param stddev_types:
List of standard deviation types, constants from
:class:`openquake.hazardlib.const.StdDev`.
Method result value should include
standard deviation values for each of types in this list.
:returns:
Method should return a tuple of two items. First item should be
a numpy array of floats -- mean values of respective component
of a chosen intensity measure type, and the second should be
a list of numpy arrays of standard deviation values for the same
single component of the same single intensity measure type, one
array for each type in ``stddev_types`` parameter, preserving
the order.
Combining interface to mean and standard deviation values in a single
method allows to avoid redoing the same intermediate calculations
if there are some shared between stddev and mean formulae without
resorting to keeping any sort of internal state (and effectively
making GSIM not reenterable).
However it is advised to split calculation of mean and stddev values
and make ``get_mean_and_stddevs()`` just combine both (and possibly
compute interim steps).
"""
def get_poes(self, sctx, rctx, dctx, imt, imls, truncation_level):
"""
Calculate and return probabilities of exceedance (PoEs) of one or more
intensity measure levels (IMLs) of one intensity measure type (IMT)
for one or more pairs "site -- rupture".
:param sctx:
An instance of :class:`SitesContext` with sites information
to calculate PoEs on.
:param rctx:
An instance of :class:`RuptureContext` with a single rupture
information.
:param dctx:
An instance of :class:`DistancesContext` with information about
the distances between sites and a rupture.
All three contexts (``sctx``, ``rctx`` and ``dctx``) must conform
to each other. The easiest way to get them is to call
ContextMaker.make_contexts.
:param imt:
An intensity measure type object (that is, an instance of one
of classes from :mod:`openquake.hazardlib.imt`).
:param imls:
List of interested intensity measure levels (of type ``imt``).
:param truncation_level:
Can be ``None``, which means that the distribution of intensity
is treated as Gaussian distribution with possible values ranging
from minus infinity to plus infinity.
When set to zero, the mean intensity is treated as an exact
value (standard deviation is not even computed for that case)
and resulting array contains 0 in places where IMT is strictly
lower than the mean value of intensity and 1.0 where IMT is equal
or greater.
When truncation level is positive number, the intensity
distribution is processed as symmetric truncated Gaussian with
range borders being ``mean - truncation_level * stddev`` and
``mean + truncation_level * stddev``. That is, the truncation
level expresses how far the range borders are from the mean
value and is defined in units of sigmas. The resulting PoEs
for that mode are values of complementary cumulative distribution
function of that truncated Gaussian applied to IMLs.
:returns:
A dictionary of the same structure as parameter ``imts`` (see
above). Instead of lists of IMLs values of the dictionaries
have 2d numpy arrays of corresponding PoEs, first dimension
represents sites and the second represents IMLs.
:raises ValueError:
If truncation level is not ``None`` and neither non-negative
float number, and if ``imts`` dictionary contain wrong or
unsupported IMTs (see :attr:`DEFINED_FOR_INTENSITY_MEASURE_TYPES`).
"""
if truncation_level is not None and truncation_level < 0:
raise ValueError('truncation level must be zero, positive number '
'or None')
self._check_imt(imt)
if truncation_level == 0:
# zero truncation mode, just compare imls to mean
imls = self.to_distribution_values(imls)
mean, _ = self.get_mean_and_stddevs(sctx, rctx, dctx, imt, [])
mean = mean.reshape(mean.shape + (1, ))
return (imls <= mean).astype(float)
else:
# use real normal distribution
assert (const.StdDev.TOTAL
in self.DEFINED_FOR_STANDARD_DEVIATION_TYPES)
imls = self.to_distribution_values(imls)
mean, [stddev] = self.get_mean_and_stddevs(sctx, rctx, dctx, imt,
[const.StdDev.TOTAL])
mean = mean.reshape(mean.shape + (1, ))
stddev = stddev.reshape(stddev.shape + (1, ))
values = (imls - mean) / stddev
if truncation_level is None:
return _norm_sf(values)
else:
return _truncnorm_sf(truncation_level, values)
def disaggregate_pne(self, rupture, sctx, dctx, imt, iml, truncnorm,
epsilons):
"""
Disaggregate (separate) PoE of ``iml`` in different contributions
each coming from ``epsilons`` distribution bins.
Other parameters are the same as for :meth:`get_poes`, with
differences that ``truncation_level`` is required to be positive.
:returns:
Contribution to probability of exceedance of ``iml`` coming
from different sigma bands in the form of a 2d numpy array of
probabilities with shape (n_sites, n_epsilons)
"""
# compute mean and standard deviations
mean, [stddev] = self.get_mean_and_stddevs(sctx, rupture, dctx, imt,
[const.StdDev.TOTAL])
# compute iml value with respect to standard (mean=0, std=1)
# normal distributions
standard_imls = (self.to_distribution_values(iml) - mean) / stddev
# compute epsilon bins contributions
contribution_by_bands = (truncnorm.cdf(epsilons[1:]) -
truncnorm.cdf(epsilons[:-1]))
# take the minimum epsilon larger than standard_iml
bins = numpy.searchsorted(epsilons, standard_imls)
poe_by_site = []
n_epsilons = len(epsilons) - 1
for lvl, bin in zip(standard_imls, bins): # one per site
if bin == 0:
poe_by_site.append(contribution_by_bands)
elif bin > n_epsilons:
poe_by_site.append(numpy.zeros(n_epsilons))
else:
# for other cases (when ``lvl`` falls somewhere in the
# histogram):
poe = numpy.concatenate([
# take zeros for bins that are on the left hand side
# from the bin ``lvl`` falls into,
numpy.zeros(bin - 1),
# ... area of the portion of the bin containing ``lvl``
# (the portion is limited on the left hand side by
# ``lvl`` and on the right hand side by the bin edge),
[truncnorm.sf(lvl) - contribution_by_bands[bin:].sum()],
# ... and all bins on the right go unchanged.
contribution_by_bands[bin:]
])
poe_by_site.append(poe)
poes = numpy.array(poe_by_site) # shape (n_sites, n_epsilons)
return rupture.get_probability_no_exceedance(poes)
@abc.abstractmethod
def to_distribution_values(self, values):
"""
Convert a list or array of values in units of IMT to a numpy array
of values of intensity measure distribution (like taking the natural
logarithm for :class:`GMPE`).
This method is implemented by both :class:`GMPE` and :class:`IPE`
so there is no need to override it in actual GSIM implementations.
"""
@abc.abstractmethod
def to_imt_unit_values(self, values):
"""
Convert a list or array of values of intensity measure distribution
(like ones returned from :meth:`get_mean_and_stddevs`) to values
in units of IMT. This is the opposite operation
to :meth:`to_distribution_values`.
This method is implemented by both :class:`GMPE` and :class:`IPE`
so there is no need to override it in actual GSIM implementations.
"""
def _check_imt(self, imt):
"""
Make sure that ``imt`` is valid and is supported by this GSIM.
"""
if not issubclass(type(imt), imt_module._IMT):
raise ValueError('imt must be an instance of IMT subclass')
if not type(imt) in self.DEFINED_FOR_INTENSITY_MEASURE_TYPES:
raise ValueError('imt %s is not supported by %s' %
(type(imt).__name__, type(self).__name__))
def __lt__(self, other):
"""
The GSIMs are ordered according to string representation
"""
return str(self) < str(other)
def __eq__(self, other):
"""
The GSIMs are equal if their string representations are equal
"""
return str(self) == str(other)
def __hash__(self):
"""
We use the __str__ representation as hash: it means that we can
use equivalently GSIM instances or strings as dictionary keys.
"""
return hash(str(self))
def __str__(self):
kwargs = ', '.join('%s=%r' % kv for kv in sorted(self.kwargs.items()))
return "%s(%s)" % (self.__class__.__name__, kwargs)
def __repr__(self):
"""
Default string representation for GSIM instances. It contains
the name and values of the arguments, if any.
"""
return repr(str(self))
class BaseSurface(with_metaclass(abc.ABCMeta)):
"""
Base class for a surface in 3D-space.
"""
@abc.abstractmethod
def get_min_distance(self, mesh):
"""
Compute and return the minimum distance from the surface to each point
of ``mesh``. This distance is sometimes called ``Rrup``.
:param mesh:
:class:`~openquake.hazardlib.geo.mesh.Mesh` of points to calculate
minimum distance to.
:returns:
A numpy array of distances in km.
"""
@abc.abstractmethod
def get_closest_points(self, mesh):
"""
For each point from ``mesh`` find a closest point belonging to surface.
:param mesh:
:class:`~openquake.hazardlib.geo.mesh.Mesh` of points to find
closest points to.
:returns:
:class:`~openquake.hazardlib.geo.mesh.Mesh` of the same shape as
``mesh`` with closest surface's points on respective indices.
"""
@abc.abstractmethod
def get_joyner_boore_distance(self, mesh):
"""
Compute and return Joyner-Boore (also known as ``Rjb``) distance
to each point of ``mesh``.
:param mesh:
:class:`~openquake.hazardlib.geo.mesh.Mesh` of points to calculate
Joyner-Boore distance to.
:returns:
Numpy array of closest distances between the projections of surface
and each point of the ``mesh`` to the earth surface.
"""
@abc.abstractmethod
def get_ry0_distance(self, mesh):
"""
Compute the minimum distance between each point of a mesh and the great
circle arcs perpendicular to the average strike direction of the
fault trace and passing through the end-points of the trace.
:param mesh:
:class:`~openquake.hazardlib.geo.mesh.Mesh` of points to calculate
Ry0-distance to.
:returns:
Numpy array of distances in km.
"""
@abc.abstractmethod
def get_rx_distance(self, mesh):
"""
Compute distance between each point of mesh and surface's great circle
arc.
Distance is measured perpendicular to the rupture strike, from
the surface projection of the updip edge of the rupture, with
the down dip direction being positive (this distance is usually
called ``Rx``).
In other words, is the horizontal distance to top edge of rupture
measured perpendicular to the strike. Values on the hanging wall
are positive, values on the footwall are negative.
:param mesh:
:class:`~openquake.hazardlib.geo.mesh.Mesh` of points to calculate
Rx-distance to.
:returns:
Numpy array of distances in km.
"""
@abc.abstractmethod
def get_top_edge_depth(self):
"""
Compute minimum depth of surface's top edge.
:returns:
Float value, the vertical distance between the earth surface
and the shallowest point in surface's top edge in km.
"""
@abc.abstractmethod
def get_strike(self):
"""
Compute surface's strike as decimal degrees in a range ``[0, 360)``.
The actual definition of the strike might depend on surface geometry.
:returns:
Float value, the azimuth (in degrees) of the surface top edge
"""
@abc.abstractmethod
def get_dip(self):
"""
Compute surface's dip as decimal degrees in a range ``(0, 90]``.
The actual definition of the dip might depend on surface geometry.
:returns:
Float value, the inclination (in degrees) of the surface with
respect to the Earth surface
"""
@abc.abstractmethod
def get_width(self):
"""
Compute surface's width (that is surface extension along the
dip direction) in km.
The actual definition depends on the type of surface geometry.
:returns:
Float value, the surface width
"""
@abc.abstractmethod
def get_area(self):
"""
Compute surface's area in squared km.
:returns:
Float value, the surface area
"""
@abc.abstractmethod
def get_bounding_box(self):
"""
Compute surface geographical bounding box.
:return:
A tuple of four items. These items represent western, eastern,
northern and southern borders of the bounding box respectively.
Values are floats in decimal degrees.
"""
@abc.abstractmethod
def get_middle_point(self):
"""
class BaseQuadrilateralSurface(with_metaclass(abc.ABCMeta, BaseSurface)):
"""
Base class for a quadrilateral surface in 3D-space.
Subclasses must implement :meth:`_create_mesh`, and superclass methods
:meth:`get_strike() <.base.BaseSurface.get_strike>`,
:meth:`get_dip() <.base.BaseSurface.get_dip>` and
:meth:`get_width() <.base.BaseSurface.get_width>`,
and can override any others just for the sake of performance
"""
def __init__(self):
self._mesh = None
def get_min_distance(self, mesh):
"""
See :meth:`superclass method
<.base.BaseSurface.get_min_distance>`
for spec of input and result values.
Base class implementation calls the :meth:`corresponding
<openquake.hazardlib.geo.mesh.Mesh.get_min_distance>` method of the
surface's :meth:`mesh <get_mesh>`.
Subclasses may override this method in order to make use
of knowledge of a specific surface shape and thus perform
better.
"""
return self.get_mesh().get_min_distance(mesh)
def get_closest_points(self, mesh):
"""
See :meth:`superclass method
<.base.BaseSurface.get_closest_points>`
for spec of input and result values.
Base class implementation calls the :meth:`corresponding
<openquake.hazardlib.geo.mesh.Mesh.get_closest_points>` method of the
surface's :meth:`mesh <get_mesh>`.
"""
return self.get_mesh().get_closest_points(mesh)
def get_joyner_boore_distance(self, mesh):
"""
See :meth:`superclass method
<.base.BaseSurface.get_joyner_boore_distance>`
for spec of input and result values.
Base class calls surface mesh's method
:meth:`~openquake.hazardlib.geo.mesh.Mesh.get_joyner_boore_distance`.
"""
return self.get_mesh().get_joyner_boore_distance(mesh)
def get_ry0_distance(self, mesh):
"""
:param mesh:
:class:`~openquake.hazardlib.geo.mesh.Mesh` of points to calculate
Ry0-distance to.
:returns:
Numpy array of distances in km.
See also :meth:`superclass method <.base.BaseSurface.get_ry0_distance>`
for spec of input and result values.
This method uses an average strike direction to compute ry0.
"""
# This computes ry0 by using an average strike direction
top_edge = self.get_mesh()[0:1]
mean_strike = self.get_strike()
dst1 = geodetic.distance_to_arc(top_edge.lons[0, 0],
top_edge.lats[0, 0],
(mean_strike + 90.) % 360,
mesh.lons, mesh.lats)
dst2 = geodetic.distance_to_arc(top_edge.lons[0, -1],
top_edge.lats[0, -1],
(mean_strike + 90.) % 360,
mesh.lons, mesh.lats)
# Find the points on the rupture
# Get the shortest distance from the two lines
idx = numpy.sign(dst1) == numpy.sign(dst2)
dst = numpy.zeros_like(dst1)
dst[idx] = numpy.fmin(numpy.abs(dst1[idx]), numpy.abs(dst2[idx]))
return dst
def get_rx_distance(self, mesh):
"""
See :meth:`superclass method
<.base.BaseSurface.get_rx_distance>`
for spec of input and result values.
The method extracts the top edge of the surface. For each point in mesh
it computes the Rx distance to each segment the top edge is made
of. The calculation is done by calling the function
:func:`openquake.hazardlib.geo.geodetic.distance_to_arc`. The final Rx
distance matrix is then constructed by taking, for each point in mesh,
the minimum Rx distance value computed.
"""
top_edge = self.get_mesh()[0:1]
dists = []
if top_edge.lons.shape[1] < 3:
i = 0
p1 = Point(
top_edge.lons[0, i],
top_edge.lats[0, i],
top_edge.depths[0, i]
)
p2 = Point(
top_edge.lons[0, i + 1], top_edge.lats[0, i + 1],
top_edge.depths[0, i + 1]
)
azimuth = p1.azimuth(p2)
dists.append(
geodetic.distance_to_arc(
p1.longitude, p1.latitude, azimuth,
mesh.lons, mesh.lats
)
)
else:
for i in range(top_edge.lons.shape[1] - 1):
p1 = Point(
top_edge.lons[0, i],
top_edge.lats[0, i],
top_edge.depths[0, i]
)
p2 = Point(
top_edge.lons[0, i + 1],
top_edge.lats[0, i + 1],
top_edge.depths[0, i + 1]
)
# Swapping
if i == 0:
pt = p1
p1 = p2
p2 = pt
# Computing azimuth and distance
if i == 0 or i == top_edge.lons.shape[1] - 2:
azimuth = p1.azimuth(p2)
tmp = geodetic.distance_to_semi_arc(p1.longitude,
p1.latitude,
azimuth,
mesh.lons, mesh.lats)
else:
tmp = geodetic.min_distance_to_segment([p1.longitude,
p2.longitude],
[p1.latitude,
p2.latitude],
mesh.lons,
mesh.lats)
# Correcting the sign of the distance
if i == 0:
tmp *= -1
dists.append(tmp)
# Computing distances
dists = numpy.array(dists)
iii = abs(dists).argmin(axis=0)
dst = dists[iii, list(range(dists.shape[1]))]
return dst
def get_top_edge_depth(self):
"""
Return minimum depth of surface's top edge.
:returns:
Float value, the vertical distance between the earth surface
and the shallowest point in surface's top edge in km.
"""
top_edge = self.get_mesh()[0:1]
if top_edge.depths is None:
return 0
else:
return numpy.min(top_edge.depths)
def _get_top_edge_centroid(self):
"""
Return :class:`~openquake.hazardlib.geo.point.Point` representing the
surface's top edge centroid.
"""
top_edge = self.get_mesh()[0:1]
return top_edge.get_middle_point()
def get_mesh(self):
"""
Return surface's mesh.
Uses :meth:`_create_mesh` for creating the mesh for the first time.
All subsequent calls to :meth:`get_mesh` return the same mesh object.
.. warning::
It is required that the mesh is constructed "top-to-bottom".
That is, the first row of points should be the shallowest.
"""
if self._mesh is None:
self._mesh = self._create_mesh()
assert (
self._mesh.depths is None or len(self._mesh.depths) == 1
or self._mesh.depths[0][0] < self._mesh.depths[-1][0]
), "the first row of points in the mesh must be the shallowest"
return self._mesh
def get_area(self):
"""
Compute area as the sum of the mesh cells area values.
"""
mesh = self.get_mesh()
_, _, _, area = mesh.get_cell_dimensions()
return numpy.sum(area)
def get_bounding_box(self):
"""
Compute surface bounding box from surface mesh representation. That is
extract longitudes and latitudes of mesh points and calls:
:meth:`openquake.hazardlib.geo.utils.get_spherical_bounding_box`
:return:
A tuple of four items. These items represent western, eastern,
northern and southern borders of the bounding box respectively.
Values are floats in decimal degrees.
"""
mesh = self.get_mesh()
return utils.get_spherical_bounding_box(mesh.lons, mesh.lats)
def get_middle_point(self):
"""
Compute middle point from surface mesh representation. Calls
:meth:`openquake.hazardlib.geo.mesh.RectangularMesh.get_middle_point`
"""
mesh = self.get_mesh()
return mesh.get_middle_point()
def get_resampled_top_edge(self, angle_var=0.1):
"""
This methods computes a simplified representation of a fault top edge
by removing the points that are not describing a change of direction,
provided a certain tolerance angle.
:param float angle_var:
Number representing the maximum deviation (in degrees) admitted
without the creation of a new segment
:returns:
A :class:`~openquake.hazardlib.geo.line.Line` representing the
rupture surface's top edge.
"""
mesh = self.get_mesh()
top_edge = [Point(mesh.lons[0][0], mesh.lats[0][0], mesh.depths[0][0])]
for i in range(len(mesh.triangulate()[1][0]) - 1):
v1 = numpy.asarray(mesh.triangulate()[1][0][i])
v2 = numpy.asarray(mesh.triangulate()[1][0][i + 1])
cosang = numpy.dot(v1, v2)
sinang = numpy.linalg.norm(numpy.cross(v1, v2))
angle = math.degrees(numpy.arctan2(sinang, cosang))
if abs(angle) > angle_var:
top_edge.append(Point(mesh.lons[0][i + 1],
mesh.lats[0][i + 1],
mesh.depths[0][i + 1]))
top_edge.append(Point(mesh.lons[0][-1],
mesh.lats[0][-1], mesh.depths[0][-1]))
line_top_edge = Line(top_edge)
return line_top_edge
@abc.abstractmethod
def _create_mesh(self):
"""
Create and return the mesh of points covering the surface.
:returns:
An instance of
:class:`openquake.hazardlib.geo.mesh.RectangularMesh`.
"""
def get_hypo_location(self, mesh_spacing, hypo_loc=None):
"""
The method determines the location of the hypocentre within the rupture
:param mesh:
:class:`~openquake.hazardlib.geo.mesh.Mesh` of points
:param mesh_spacing:
The desired distance between two adjacent points in source's
ruptures' mesh, in km. Mainly this parameter allows to balance
the trade-off between time needed to compute the distance
between the rupture surface and a site and the precision of that
computation.
:param hypo_loc:
Hypocentre location as fraction of rupture plane, as a tuple of
(Along Strike, Down Dip), e.g. a hypocentre located in the centroid
of the rupture would be input as (0.5, 0.5), whereas a
hypocentre located in a position 3/4 along the length, and 1/4 of
the way down dip of the rupture plane would be entered as
(0.75, 0.25).
:returns:
Hypocentre location as instance of
:class:`~openquake.hazardlib.geo.point.Point`
"""
mesh = self.get_mesh()
centroid = mesh.get_middle_point()
if hypo_loc is None:
return centroid
total_len_y = (len(mesh.depths) - 1) * mesh_spacing
y_distance = hypo_loc[1] * total_len_y
y_node = int(numpy.round(y_distance / mesh_spacing))
total_len_x = (len(mesh.lons[y_node]) - 1) * mesh_spacing
x_distance = hypo_loc[0] * total_len_x
x_node = int(numpy.round(x_distance / mesh_spacing))
hypocentre = Point(mesh.lons[y_node][x_node],
mesh.lats[y_node][x_node],
mesh.depths[y_node][x_node])
return hypocentre
class BaseCalculator(with_metaclass(abc.ABCMeta)):
"""
Abstract base class for all calculators.
:param oqparam: OqParam object
:param monitor: monitor object
:param calc_id: numeric calculation ID
"""
oqparam = datastore.persistent_attribute('oqparam')
sitemesh = datastore.persistent_attribute('sitemesh')
sitecol = datastore.persistent_attribute('sitecol')
rlzs_assoc = datastore.persistent_attribute('rlzs_assoc')
realizations = datastore.persistent_attribute('realizations')
assets_by_site = datastore.persistent_attribute('assets_by_site')
assetcol = datastore.persistent_attribute('assetcol')
cost_types = datastore.persistent_attribute('cost_types')
taxonomies = datastore.persistent_attribute('taxonomies')
job_info = datastore.persistent_attribute('job_info')
source_chunks = datastore.persistent_attribute('source_chunks')
source_pre_info = datastore.persistent_attribute('source_pre_info')
performance = datastore.persistent_attribute('performance')
csm = datastore.persistent_attribute('composite_source_model')
pre_calculator = None # to be overridden
is_stochastic = False # True for scenario and event based calculators
def __init__(self,
oqparam,
monitor=DummyMonitor(),
calc_id=None,
persistent=True):
self.monitor = monitor
if persistent:
self.datastore = datastore.DataStore(calc_id)
else:
self.datastore = general.AccumDict()
self.datastore.hdf5 = {}
self.datastore.export_dir = oqparam.export_dir
if 'oqparam' not in self.datastore: # new datastore
self.oqparam = oqparam
# else we are doing a precalculation; oqparam has been already stored
self.persistent = persistent
def run(self,
pre_execute=True,
clean_up=True,
concurrent_tasks=None,
**kw):
"""
Run the calculation and return the exported outputs.
"""
if concurrent_tasks is not None:
self.oqparam.concurrent_tasks = concurrent_tasks
vars(self.oqparam).update(kw)
exported = {}
try:
if pre_execute:
with self.monitor('pre_execute', autoflush=True):
self.pre_execute()
with self.monitor('execute', autoflush=True):
result = self.execute()
with self.monitor('post_execute', autoflush=True):
self.post_execute(result)
with self.monitor('export', autoflush=True):
exported = self.export()
finally:
etype = sys.exc_info()[0]
if etype:
logging.critical('', exc_info=True)
if clean_up:
try:
self.clean_up()
except:
logging.error('Cleanup error', exc_info=True)
return exported
def core_func(*args):
"""
Core routine running on the workers.
"""
raise NotImplementedError
@abc.abstractmethod
def pre_execute(self):
"""
Initialization phase.
"""
@abc.abstractmethod
def execute(self):
"""
Execution phase. Usually will run in parallel the core
function and return a dictionary with the results.
"""
@abc.abstractmethod
def post_execute(self, result):
"""
Post-processing phase of the aggregated output. It must be
overridden with the export code. It will return a dictionary
of output files.
"""
def export(self, exports=None):
"""
Export all the outputs in the datastore in the given export formats.
:returns: dictionary output_key -> sorted list of exported paths
"""
exported = {}
individual_curves = self.oqparam.individual_curves
fmts = exports.split(',') if exports else self.oqparam.exports
for fmt in fmts:
if not fmt:
continue
for key in self.datastore:
if 'rlzs' in key and not individual_curves:
continue # skip individual curves
ekey = (key, fmt)
try:
exported[ekey] = sorted(export.export(
ekey, self.datastore))
logging.info('exported %s: %s', key, exported[ekey])
except KeyError:
logging.info('%s is not exportable in %s', key, fmt)
return exported
def clean_up(self):
"""
Collect the realizations and the monitoring information,
then close the datastore.
"""
self.realizations = numpy.array(
[(r.uid, r.weight) for r in self.rlzs_assoc.realizations], rlz_dt)
performance = self.monitor.collect_performance()
if performance is not None:
self.performance = performance
self.datastore.close()
self.datastore.symlink(os.path.dirname(self.oqparam.inputs['job_ini']))
class BaseCalculator(with_metaclass(abc.ABCMeta)):
"""
Abstract base class for all calculators.
:param oqparam: OqParam object
:param monitor: monitor object
:param calc_id: numeric calculation ID
"""
from_engine = False # set by engine.run_calc
sitecol = datastore.persistent_attribute('sitecol')
assetcol = datastore.persistent_attribute('assetcol')
performance = datastore.persistent_attribute('performance')
pre_calculator = None # to be overridden
is_stochastic = False # True for scenario and event based calculators
@property
def taxonomies(self):
return self.datastore['assetcol/taxonomies'].value
def __init__(self, oqparam, monitor=Monitor(), calc_id=None):
self._monitor = monitor
self.datastore = datastore.DataStore(calc_id)
self.oqparam = oqparam
def monitor(self, operation, **kw):
"""
Return a new Monitor instance
"""
mon = self._monitor(operation, hdf5path=self.datastore.hdf5path)
self._monitor.calc_id = mon.calc_id = self.datastore.calc_id
vars(mon).update(kw)
return mon
def save_params(self, **kw):
"""
Update the current calculation parameters and save engine_version
"""
vars(self.oqparam).update(**kw)
self.datastore['oqparam'] = self.oqparam # save the updated oqparam
attrs = self.datastore['/'].attrs
attrs['engine_version'] = engine_version
self.datastore.flush()
def set_log_format(self):
"""Set the format of the root logger"""
fmt = '[%(asctime)s #{} %(levelname)s] %(message)s'.format(
self.datastore.calc_id)
for handler in logging.root.handlers:
handler.setFormatter(logging.Formatter(fmt))
def run(self, pre_execute=True, concurrent_tasks=None, close=True, **kw):
"""
Run the calculation and return the exported outputs.
"""
global logversion
self.close = close
self.set_log_format()
if logversion: # make sure this is logged only once
logging.info('Running %s', self.oqparam.inputs['job_ini'])
logging.info('Using engine version %s', engine_version)
logversion = False
if concurrent_tasks is None: # use the job.ini parameter
ct = self.oqparam.concurrent_tasks
else: # used the parameter passed in the command-line
ct = concurrent_tasks
if ct == 0: # disable distribution temporarily
oq_distribute = os.environ.get('OQ_DISTRIBUTE')
os.environ['OQ_DISTRIBUTE'] = 'no'
if ct != self.oqparam.concurrent_tasks:
# save the used concurrent_tasks
self.oqparam.concurrent_tasks = ct
self.save_params(**kw)
exported = {}
try:
if pre_execute:
self.pre_execute()
self.result = self.execute()
if self.result is not None:
self.post_execute(self.result)
self.before_export()
exported = self.export(kw.get('exports', ''))
except KeyboardInterrupt:
pids = ' '.join(str(p.pid) for p in executor._processes)
sys.stderr.write(
'You can manually kill the workers with kill %s\n' % pids)
raise
except:
if kw.get('pdb'): # post-mortem debug
tb = sys.exc_info()[2]
traceback.print_tb(tb)
pdb.post_mortem(tb)
else:
logging.critical('', exc_info=True)
raise
finally:
if ct == 0: # restore OQ_DISTRIBUTE
if oq_distribute is None: # was not set
del os.environ['OQ_DISTRIBUTE']
else:
os.environ['OQ_DISTRIBUTE'] = oq_distribute
return exported
def core_task(*args):
"""
Core routine running on the workers.
"""
raise NotImplementedError
@abc.abstractmethod
def pre_execute(self):
"""
Initialization phase.
"""
@abc.abstractmethod
def execute(self):
"""
Execution phase. Usually will run in parallel the core
function and return a dictionary with the results.
"""
@abc.abstractmethod
def post_execute(self, result):
"""
Post-processing phase of the aggregated output. It must be
overridden with the export code. It will return a dictionary
of output files.
"""
def export(self, exports=None):
"""
Export all the outputs in the datastore in the given export formats.
Individual outputs are not exported if there are multiple realizations.
:returns: dictionary output_key -> sorted list of exported paths
"""
num_rlzs = len(self.datastore['realizations'])
exported = {}
if isinstance(exports, tuple):
fmts = exports
elif exports: # is a string
fmts = exports.split(',')
elif isinstance(self.oqparam.exports, tuple):
fmts = self.oqparam.exports
else: # is a string
fmts = self.oqparam.exports.split(',')
keys = set(self.datastore)
has_hcurves = 'hcurves' in self.datastore or 'poes' in self.datastore
if has_hcurves:
keys.add('hcurves')
for fmt in fmts:
if not fmt:
continue
for key in sorted(keys): # top level keys
if 'rlzs' in key and num_rlzs > 1:
continue # skip individual curves
self._export((key, fmt), exported)
if has_hcurves and self.oqparam.hazard_maps:
self._export(('hmaps', fmt), exported)
if has_hcurves and self.oqparam.uniform_hazard_spectra:
self._export(('uhs', fmt), exported)
if self.close: # in the engine we close later
self.result = None
try:
self.datastore.close()
except (RuntimeError, ValueError):
# sometimes produces errors but they are difficult to
# reproduce
logging.warn('', exc_info=True)
return exported
def _export(self, ekey, exported):
if ekey in exp:
with self.monitor('export'):
exported[ekey] = exp(ekey, self.datastore)
logging.info('exported %s: %s', ekey[0], exported[ekey])
def before_export(self):
"""
Collect the realizations and set the attributes nbytes
"""
sm_by_rlz = self.datastore['csm_info'].get_sm_by_rlz(
self.rlzs_assoc.realizations) or collections.defaultdict(
lambda: 'NA')
self.datastore['realizations'] = numpy.array(
[(r.uid, sm_by_rlz[r], gsim_names(r), r.weight)
for r in self.rlzs_assoc.realizations], rlz_dt)
if 'hcurves' in set(self.datastore):
self.datastore.set_nbytes('hcurves')
self.datastore.flush()
class BaseRupture(with_metaclass(abc.ABCMeta)):
"""
Rupture object represents a single earthquake rupture.
:param mag:
Magnitude of the rupture.
:param rake:
Rake value of the rupture.
See :class:`~openquake.hazardlib.geo.nodalplane.NodalPlane`.
:param tectonic_region_type:
Rupture's tectonic regime. One of constants
in :class:`openquake.hazardlib.const.TRT`.
:param hypocenter:
A :class:`~openquake.hazardlib.geo.point.Point`, rupture's hypocenter.
:param surface:
An instance of subclass of
:class:`~openquake.hazardlib.geo.surface.base.BaseSurface`.
Object representing the rupture surface geometry.
:param source_typology:
Subclass of :class:`~openquake.hazardlib.source.base.BaseSeismicSource`
(class object, not an instance) referencing the typology
of the source that produced this rupture.
:param rupture_slip_direction:
Angle describing rupture propagation direction in decimal degrees.
:raises ValueError:
If magnitude value is not positive, or tectonic region type is unknown.
NB: if you want to convert the rupture into XML, you should set the
attribute surface_nodes to an appropriate value.
"""
_slots_ = '''mag rake tectonic_region_type hypocenter surface
source_typology rupture_slip_direction'''.split()
@classmethod
def init(cls):
"""
Initialize the class dictionaries `._code` and .`types` encoding the
bidirectional correspondence between an integer in the range 0..255
(the code) and a triplet of classes (rupture_class, surface_class,
source_class). This is useful when serializing the rupture to and
from HDF5.
"""
source_classes = get_subclasses(BaseSeismicSource)
rupture_classes = [BaseRupture] + list(get_subclasses(BaseRupture))
surface_classes = get_subclasses(BaseSurface)
code, types, n = {}, {}, 0
for src, rup, sur in itertools.product(source_classes, rupture_classes,
surface_classes):
code[rup, sur, src] = n
types[n] = rup, sur, src
n += 1
if n >= 256:
raise ValueError('Too many rupture codes: %d' % n)
cls._code = code
cls.types = types
def __init__(self,
mag,
rake,
tectonic_region_type,
hypocenter,
surface,
source_typology,
rupture_slip_direction=None):
if not mag > 0:
raise ValueError('magnitude must be positive')
NodalPlane.check_rake(rake)
self.tectonic_region_type = tectonic_region_type
self.rake = rake
self.mag = mag
self.hypocenter = hypocenter
self.surface = surface
self.source_typology = source_typology
self.rupture_slip_direction = rupture_slip_direction
@property
def code(self):
"""Returns the code (integer in the range 0 .. 255) of the rupture"""
return self._code[self.__class__, self.surface.__class__,
self.source_typology]
def get_probability_no_exceedance(self, poes):
"""
Compute and return the probability that in the time span for which the
rupture is defined, the rupture itself never generates a ground motion
value higher than a given level at a given site.
Such calculation is performed starting from the conditional probability
that an occurrence of the current rupture is producing a ground motion
value higher than the level of interest at the site of interest.
The actual formula used for such calculation depends on the temporal
occurrence model the rupture is associated with.
The calculation can be performed for multiple intensity measure levels
and multiple sites in a vectorized fashion.
:param poes:
2D numpy array containing conditional probabilities the the a
rupture occurrence causes a ground shaking value exceeding a
ground motion level at a site. First dimension represent sites,
second dimension intensity measure levels. ``poes`` can be obtained
calling the :meth:`method
<openquake.hazardlib.gsim.base.GroundShakingIntensityModel.get_poes>`.
"""
raise NotImplementedError
def sample_number_of_occurrences(self):
"""
Randomly sample number of occurrences from temporal occurrence model
probability distribution.
.. note::
This method is using random numbers. In order to reproduce the
same results numpy random numbers generator needs to be seeded, see
http://docs.scipy.org/doc/numpy/reference/generated/numpy.random.seed.html
:returns:
int, Number of rupture occurrences
"""
raise NotImplementedError
class BaseSeismicSource(with_metaclass(abc.ABCMeta)):
"""
Base class representing a seismic source, that is a structure generating
earthquake ruptures.
:param source_id:
Some (numeric or literal) source identifier. Supposed to be unique
within the source model.
:param name:
String, a human-readable name of the source.
:param tectonic_region_type:
Source's tectonic regime. See :class:`openquake.hazardlib.const.TRT`.
"""
_slots_ = [
'source_id', 'name', 'tectonic_region_type', 'src_group_id',
'num_ruptures', 'seed', 'id'
]
RUPTURE_WEIGHT = 1. # overridden in (Multi)PointSource, AreaSource
nsites = 1 # FIXME: remove this and fix all hazardlib tests
ngsims = 1
@abc.abstractproperty
def MODIFICATIONS(self):
pass
@property
def weight(self):
"""
Determine the source weight from the number of ruptures, by
multiplying with the scale factor RUPTURE_WEIGHT
"""
if not self.num_ruptures:
self.num_ruptures = self.count_ruptures()
return (self.num_ruptures * self.RUPTURE_WEIGHT * self.nsites
) # * self.ngsims)
@property
def src_group_ids(self):
"""
:returns: a list of source group IDs (usually of 1 element)
"""
grp_id = self.src_group_id
return [grp_id] if isinstance(grp_id, int) else grp_id
def __init__(self, source_id, name, tectonic_region_type):
self.source_id = source_id
self.name = name
self.tectonic_region_type = tectonic_region_type
self.src_group_id = None # set by the engine
self.num_ruptures = 0 # set by the engine
self.seed = None # set by the engine
self.id = None # set by the engine
@abc.abstractmethod
def iter_ruptures(self):
"""
Get a generator object that yields probabilistic ruptures the source
consists of.
:returns:
Generator of instances of sublclass of :class:
`~openquake.hazardlib.source.rupture.BaseProbabilisticRupture`.
"""
@abc.abstractmethod
def count_ruptures(self):
"""
Return the number of ruptures that will be generated by the source.
"""
@abc.abstractmethod
def get_min_max_mag(self):
"""
Return minimum and maximum magnitudes of the ruptures generated
by the source.
"""
@abc.abstractmethod
def get_rupture_enclosing_polygon(self, dilation=0):
"""
Get a polygon which encloses all the ruptures generated by the source.
The rupture enclosing polygon is meant to be used in all hazard
calculators to filter out sources whose ruptures the user wants
to be neglected because they are too far from the locations
of interest.
For performance reasons, the ``get_rupture_enclosing_polygon()``
should compute the polygon, without creating all the ruptures.
The rupture enclosing polygon may not be necessarily the *minimum*
enclosing polygon, but must guarantee that all ruptures are within
the polygon.
This method must be implemented by subclasses.
:param dilation:
A buffer distance in km to extend the polygon borders to.
:returns:
Instance of :class:`openquake.hazardlib.geo.polygon.Polygon`.
"""
def get_bounding_box(self, dilation=0):
"""
Returns the bounding box of all the ruptures generated by the source,
enlarged by the integration distance (dilation).
"""
return self.get_rupture_enclosing_polygon(dilation).get_bbox()
def filter_sites_by_distance_to_source(self, integration_distance, sites):
"""
Filter out sites from the collection that are further from the source
than some arbitrary threshold.
:param integration_distance:
Distance in km representing a threshold: sites that are further
than that distance from the closest rupture produced by the source
should be excluded.
:param sites:
Instance of :class:`openquake.hazardlib.site.SiteCollection`
to filter.
:returns:
Filtered :class:`~openquake.hazardlib.site.SiteCollection`.
Method can be overridden by subclasses in order to achieve
higher performance for a specific typology. Base class method calls
:meth:`get_rupture_enclosing_polygon` with ``integration_distance``
as a dilation value and then filters site collection by checking
:meth:
`containment <openquake.hazardlib.geo.polygon.Polygon.intersects>`
of site locations.
The main criteria for this method to decide whether a site should be
filtered out or not is the minimum distance between the site and all
the ruptures produced by the source. If at least one rupture is closer
(in terms of great circle distance between surface projections) than
integration distance to a site, it should not be filtered out. However,
it is important not to make this method too computationally intensive.
If short-circuits are taken, false positives are generally better than
false negatives (it's better not to filter a site out if there is some
uncertainty about its distance).
"""
if integration_distance is None: # no filtering
return sites
rup_enc_poly = self.get_rupture_enclosing_polygon(integration_distance)
return sites.filter(rup_enc_poly.intersects(sites.mesh))
def modify(self, modification, parameters):
"""
Apply a single modificaton to the source parameters
Reflects the modification method and calls it passing ``parameters``
as keyword arguments.
Modifications can be applied one on top of another. The logic
of stacking modifications is up to a specific source implementation.
:param modification:
String name representing the type of modification.
:param parameters:
Dictionary of parameters needed for modification.
:raises ValueError:
If ``modification`` is missing from the attribute `MODIFICATIONS`.
"""
if modification not in self.MODIFICATIONS:
raise ValueError('Modification %s is not supported by %s' %
(modification, type(self).__name__))
meth = getattr(self, 'modify_%s' % modification)
meth(**parameters)
class BaseCalculator(with_metaclass(abc.ABCMeta)):
"""
Abstract base class for all calculators.
:param oqparam: OqParam object
:param monitor: monitor object
:param calc_id: numeric calculation ID
"""
from_engine = False # set by engine.run_calc
sitecol = datastore.persistent_attribute('sitecol')
performance = datastore.persistent_attribute('performance')
pre_calculator = None # to be overridden
is_stochastic = False # True for scenario and event based calculators
def __init__(self, oqparam, monitor=Monitor(), calc_id=None):
self._monitor = monitor
self.datastore = datastore.DataStore(calc_id)
self.oqparam = oqparam
def monitor(self, operation, **kw):
"""
:returns: a new Monitor instance
"""
mon = self._monitor(operation, hdf5path=self.datastore.hdf5path)
self._monitor.calc_id = mon.calc_id = self.datastore.calc_id
vars(mon).update(kw)
return mon
def save_params(self, **kw):
"""
Update the current calculation parameters and save engine_version
"""
vars(self.oqparam).update(**kw)
self.datastore['oqparam'] = self.oqparam # save the updated oqparam
attrs = self.datastore['/'].attrs
attrs['engine_version'] = engine_version
attrs['date'] = datetime.now().isoformat()[:19]
if 'checksum32' not in attrs:
attrs['checksum32'] = readinput.get_checksum32(self.oqparam)
self.datastore.flush()
def set_log_format(self):
"""Set the format of the root logger"""
fmt = '[%(asctime)s #{} %(levelname)s] %(message)s'.format(
self.datastore.calc_id)
for handler in logging.root.handlers:
handler.setFormatter(logging.Formatter(fmt))
def run(self, pre_execute=True, concurrent_tasks=None, close=True, **kw):
"""
Run the calculation and return the exported outputs.
"""
global logversion
self.close = close
self.set_log_format()
if logversion: # make sure this is logged only once
logging.info('Running %s', self.oqparam.inputs['job_ini'])
logging.info('Using engine version %s', engine_version)
logversion = False
if concurrent_tasks is None: # use the job.ini parameter
ct = self.oqparam.concurrent_tasks
else: # used the parameter passed in the command-line
ct = concurrent_tasks
if ct == 0: # disable distribution temporarily
oq_distribute = os.environ.get('OQ_DISTRIBUTE')
os.environ['OQ_DISTRIBUTE'] = 'no'
if ct != self.oqparam.concurrent_tasks:
# save the used concurrent_tasks
self.oqparam.concurrent_tasks = ct
self.save_params(**kw)
Starmap.init()
try:
if pre_execute:
self.pre_execute()
self.result = self.execute()
if self.result is not None:
self.post_execute(self.result)
self.before_export()
self.export(kw.get('exports', ''))
except:
if kw.get('pdb'): # post-mortem debug
tb = sys.exc_info()[2]
traceback.print_tb(tb)
pdb.post_mortem(tb)
else:
logging.critical('', exc_info=True)
raise
finally:
# cleanup globals
if ct == 0: # restore OQ_DISTRIBUTE
if oq_distribute is None: # was not set
del os.environ['OQ_DISTRIBUTE']
else:
os.environ['OQ_DISTRIBUTE'] = oq_distribute
readinput.pmap = None
readinput.exposure = None
Starmap.shutdown()
return getattr(self, 'exported', {})
def core_task(*args):
"""
Core routine running on the workers.
"""
raise NotImplementedError
@abc.abstractmethod
def pre_execute(self):
"""
Initialization phase.
"""
@abc.abstractmethod
def execute(self):
"""
Execution phase. Usually will run in parallel the core
function and return a dictionary with the results.
"""
@abc.abstractmethod
def post_execute(self, result):
"""
Post-processing phase of the aggregated output. It must be
overridden with the export code. It will return a dictionary
of output files.
"""
def export(self, exports=None):
"""
Export all the outputs in the datastore in the given export formats.
Individual outputs are not exported if there are multiple realizations.
"""
self.exported = getattr(self.precalc, 'exported', {})
if isinstance(exports, tuple):
fmts = exports
elif exports: # is a string
fmts = exports.split(',')
elif isinstance(self.oqparam.exports, tuple):
fmts = self.oqparam.exports
else: # is a string
fmts = self.oqparam.exports.split(',')
keys = set(self.datastore)
has_hcurves = 'hcurves' in self.datastore or 'poes' in self.datastore
if has_hcurves:
keys.add('hcurves')
for fmt in fmts:
if not fmt:
continue
for key in sorted(keys): # top level keys
if 'rlzs' in key and self.R > 1:
continue # skip individual curves
self._export((key, fmt))
if has_hcurves and self.oqparam.hazard_maps:
self._export(('hmaps', fmt))
if has_hcurves and self.oqparam.uniform_hazard_spectra:
self._export(('uhs', fmt))
if self.close: # in the engine we close later
self.result = None
try:
self.datastore.close()
except (RuntimeError, ValueError):
# sometimes produces errors but they are difficult to
# reproduce
logging.warn('', exc_info=True)
def _export(self, ekey):
if ekey not in exp or self.exported.get(ekey): # already exported
return
with self.monitor('export'):
self.exported[ekey] = fnames = exp(ekey, self.datastore)
if fnames:
logging.info('exported %s: %s', ekey[0], fnames)
def before_export(self):
"""
Set the attributes nbytes
"""
# sanity check that eff_ruptures have been set, i.e. are not -1
csm_info = self.datastore['csm_info']
for sm in csm_info.source_models:
for sg in sm.src_groups:
assert sg.eff_ruptures != -1, sg
for key in self.datastore:
self.datastore.set_nbytes(key)
self.datastore.flush()
class BaseMFD(with_metaclass(abc.ABCMeta)):
"""
Abstract base class for Magnitude-Frequency Distribution function.
"""
#: The set of modification type names that are supported by an MFD.
#: Each modification should have a corresponding method named
#: ``modify_modificationname()`` where the actual modification
#: logic resides.
MODIFICATIONS = abc.abstractproperty()
def modify(self, modification, parameters):
"""
Apply a single modification to an MFD parameters.
Reflects the modification method and calls it passing ``parameters``
as keyword arguments. See also :attr:`MODIFICATIONS`.
Modifications can be applied one on top of another. The logic
of stacking modifications is up to a specific MFD implementation.
:param modification:
String name representing the type of modification.
:param parameters:
Dictionary of parameters needed for modification.
:raises ValueError:
If ``modification`` is missing from :attr:`MODIFICATIONS`.
"""
if not modification in self.MODIFICATIONS:
raise ValueError('Modification %s is not supported by %s' %
(modification, type(self).__name__))
meth = getattr(self, 'modify_%s' % modification)
meth(**parameters)
self.check_constraints()
@abc.abstractmethod
def check_constraints(self):
"""
Check MFD-specific constraints and raise :exc:`ValueError`
in case of violation.
This method must be implemented by subclasses.
"""
@abc.abstractmethod
def get_annual_occurrence_rates(self):
"""
Return an MFD annual occurrence rates histogram.
This method must be implemented by subclasses.
:return:
The list of tuples, each tuple containing a pair
``(magnitude, occurrence_rate)``. Each pair represents
a single bin of the histogram with ``magnitude`` being
the center of the bin. Magnitude values are monotonically
increasing by value of bin width. ``occurence_rate``
represents the number of events per year with magnitude
that falls in between bin's boundaries.
"""
@abc.abstractmethod
def get_min_max_mag(self):
"""
Return the minimum and maximum magnitudes this MFD is defined for.
This method must be implemented by subclasses.
:return:
Magnitude value, float number.
"""
def __repr__(self):
"""
Returns the name of the magnitude frequency distribution class
"""
return "<%s>" % self.__class__.__name__
class BaseSeismicSource(with_metaclass(abc.ABCMeta)):
"""
Base class representing a seismic source, that is a structure generating
earthquake ruptures.
:param source_id:
Some (numeric or literal) source identifier. Supposed to be unique
within the source model.
:param name:
String, a human-readable name of the source.
:param tectonic_region_type:
Source's tectonic regime. See :class:`openquake.hazardlib.const.TRT`.
"""
_slots_ = [
'source_id', 'name', 'tectonic_region_type', 'src_group_id',
'num_ruptures', 'seed', 'id'
]
RUPTURE_WEIGHT = 1. # overridden in (Multi)PointSource, AreaSource
ngsims = 1
@abc.abstractproperty
def MODIFICATIONS(self):
pass
@property
def weight(self):
"""
Determine the source weight from the number of ruptures, by
multiplying with the scale factor RUPTURE_WEIGHT
"""
if not self.num_ruptures:
self.num_ruptures = self.count_ruptures()
# (MS) the weight is proportional to the number of ruptures and GSIMs
# the relation to the number of sites is unclear, but for sure less
# than linear and I am using a sqrt here (totally made up but good)
return (self.num_ruptures * self.RUPTURE_WEIGHT *
math.sqrt(self.nsites) * self.ngsims)
@property
def nsites(self):
"""
:returns: the number of sites affected by this source
"""
try:
# the engine sets self.indices when filtering the sources
return len(self.indices)
except AttributeError:
# this happens in several hazardlib tests, therefore we return
# a fake number of affected sites to avoid changing all tests
return 1
@property
def src_group_ids(self):
"""
:returns: a list of source group IDs (usually of 1 element)
"""
grp_id = getattr(self, 'src_group_id', [0])
return [grp_id] if isinstance(grp_id, int) else grp_id
def __init__(self, source_id, name, tectonic_region_type):
self.source_id = source_id
self.name = name
self.tectonic_region_type = tectonic_region_type
self.src_group_id = 0 # set by the engine
self.num_ruptures = 0 # set by the engine
self.seed = None # set by the engine
self.id = None # set by the engine
@abc.abstractmethod
def iter_ruptures(self):
"""
Get a generator object that yields probabilistic ruptures the source
consists of.
:returns:
Generator of instances of sublclass of :class:
`~openquake.hazardlib.source.rupture.BaseProbabilisticRupture`.
"""
def __iter__(self):
"""
Override to implement source splitting
"""
yield self
@abc.abstractmethod
def count_ruptures(self):
"""
Return the number of ruptures that will be generated by the source.
"""
@abc.abstractmethod
def get_min_max_mag(self):
"""
Return minimum and maximum magnitudes of the ruptures generated
by the source.
"""
def modify(self, modification, parameters):
"""
Apply a single modificaton to the source parameters
Reflects the modification method and calls it passing ``parameters``
as keyword arguments.
Modifications can be applied one on top of another. The logic
of stacking modifications is up to a specific source implementation.
:param modification:
String name representing the type of modification.
:param parameters:
Dictionary of parameters needed for modification.
:raises ValueError:
If ``modification`` is missing from the attribute `MODIFICATIONS`.
"""
if modification not in self.MODIFICATIONS:
raise ValueError('Modification %s is not supported by %s' %
(modification, type(self).__name__))
meth = getattr(self, 'modify_%s' % modification)
meth(**parameters)
class ParamSet(with_metaclass(MetaParamSet, hdf5.LiteralAttrs)):
"""
A set of valid interrelated parameters. Here is an example
of usage:
>>> class MyParams(ParamSet):
... a = Param(positiveint)
... b = Param(positivefloat)
...
... def is_valid_not_too_big(self):
... "The sum of a and b must be under 10: a={a} and b={b}"
... return self.a + self.b < 10
>>> mp = MyParams(a='1', b='7.2')
>>> mp
<MyParams a=1, b=7.2>
>>> MyParams(a='1', b='9.2').validate()
Traceback (most recent call last):
...
ValueError: The sum of a and b must be under 10: a=1 and b=9.2
The constrains are applied in lexicographic order. The attribute
corresponding to a Param descriptor can be set as usual:
>>> mp.a = '2'
>>> mp.a
'2'
A list with the literal strings can be extracted as follows:
>>> mp.to_params()
[('a', "'2'"), ('b', '7.2')]
It is possible to build a new object from a dictionary of parameters
which are assumed to be already validated:
>>> MyParams.from_(dict(a="'2'", b='7.2'))
<MyParams a='2', b=7.2>
"""
params = {}
@classmethod
def check(cls, dic):
"""
Convert a dictionary name->string into a dictionary name->value
by converting the string. If the name does not correspond to a
known parameter, just ignore it and print a warning.
"""
res = {}
for name, text in dic.items():
try:
p = getattr(cls, name)
except AttributeError:
logging.warn('Ignored unknown parameter %s', name)
else:
res[name] = p.validator(text)
return res
@classmethod
def from_(cls, dic):
"""
Build a new ParamSet from a dictionary of string-valued parameters
which are assumed to be already valid.
"""
self = cls.__new__(cls)
for k, v in dic.items():
setattr(self, k, ast.literal_eval(v))
return self
def to_params(self):
"""
Convert the instance dictionary into a sorted list of pairs
(name, valrepr) where valrepr is the string representation of
the underlying value.
"""
dic = self.__dict__
return [(k, repr(dic[k])) for k in sorted(dic)
if not k.startswith('_')]
def __init__(self, **names_vals):
for name, val in names_vals.items():
if name.startswith(('_', 'is_valid_')):
raise NameError('The parameter name %s is not acceptable' %
name)
try:
convert = getattr(self.__class__, name).validator
except AttributeError:
logging.warn("The parameter '%s' is unknown, ignoring" % name)
continue
try:
value = convert(val)
except Exception as exc:
raise ValueError('%s: could not convert to %s: %s=%s' %
(exc, convert.__name__, name, val))
setattr(self, name, value)
def validate(self):
"""
Apply the `is_valid` methods to self and possibly raise a ValueError.
"""
# it is important to have the validator applied in a fixed order
valids = [
getattr(self, valid) for valid in sorted(dir(self.__class__))
if valid.startswith('is_valid_')
]
for is_valid in valids:
if not is_valid():
docstring = is_valid.__doc__.strip()
doc = textwrap.fill(docstring.format(**vars(self)))
raise ValueError(doc)
def __iter__(self):
for item in sorted(vars(self).items()):
yield item
class BaseSeismicSource(with_metaclass(abc.ABCMeta)):
"""
Base class representing a seismic source, that is a structure generating
earthquake ruptures.
:param source_id:
Some (numeric or literal) source identifier. Supposed to be unique
within the source model.
:param name:
String, a human-readable name of the source.
:param tectonic_region_type:
Source's tectonic regime. See :class:`openquake.hazardlib.const.TRT`.
"""
__slots__ = [
'source_id', 'name', 'tectonic_region_type', 'trt_model_id', 'weight',
'seed', 'id'
]
def __init__(self, source_id, name, tectonic_region_type):
self.source_id = source_id
self.name = name
self.tectonic_region_type = tectonic_region_type
self.trt_model_id = None # set by the engine
self.weight = 1 # set by the engine
self.seed = None # set by the engine
self.id = None # set by the engine
@abc.abstractmethod
def iter_ruptures(self):
"""
Get a generator object that yields probabilistic ruptures the source
consists of.
:returns:
Generator of instances of sublclass of :class:
`~openquake.hazardlib.source.rupture.BaseProbabilisticRupture`.
"""
@abc.abstractmethod
def count_ruptures(self):
"""
Return the number of ruptures that will be generated by the source.
"""
@abc.abstractmethod
def get_min_max_mag(self):
"""
Return minimum and maximum magnitudes of the ruptures generated
by the source.
"""
@abc.abstractmethod
def get_rupture_enclosing_polygon(self, dilation=0):
"""
Get a polygon which encloses all the ruptures generated by the source.
The rupture enclosing polygon is meant to be used in all hazard
calculators to filter out sources whose ruptures the user wants
to be neglected because they are too far from the locations
of interest.
For performance reasons, the ``get_rupture_enclosing_polygon()``
should compute the polygon, without creating all the ruptures.
The rupture enclosing polygon may not be necessarily the *minimum*
enclosing polygon, but must guarantee that all ruptures are within
the polygon.
This method must be implemented by subclasses.
:param dilation:
A buffer distance in km to extend the polygon borders to.
:returns:
Instance of :class:`openquake.hazardlib.geo.polygon.Polygon`.
"""
def filter_sites_by_distance_to_source(self, integration_distance, sites):
"""
Filter out sites from the collection that are further from the source
than some arbitrary threshold.
:param integration_distance:
Distance in km representing a threshold: sites that are further
than that distance from the closest rupture produced by the source
should be excluded.
:param sites:
Instance of :class:`openquake.hazardlib.site.SiteCollection`
to filter.
:returns:
Filtered :class:`~openquake.hazardlib.site.SiteCollection`.
Method can be overridden by subclasses in order to achieve
higher performance for a specific typology. Base class method calls
:meth:`get_rupture_enclosing_polygon` with ``integration_distance``
as a dilation value and then filters site collection by checking
:meth:
`containment <openquake.hazardlib.geo.polygon.Polygon.intersects>`
of site locations.
The main criteria for this method to decide whether a site should be
filtered out or not is the minimum distance between the site and all
the ruptures produced by the source. If at least one rupture is closer
(in terms of great circle distance between surface projections) than
integration distance to a site, it should not be filtered out. However,
it is important not to make this method too computationally intensive.
If short-circuits are taken, false positives are generally better than
false negatives (it's better not to filter a site out if there is some
uncertainty about its distance).
"""
rup_enc_poly = self.get_rupture_enclosing_polygon(integration_distance)
return sites.filter(rup_enc_poly.intersects(sites.mesh))
