def pre_execute(self):
"""
Check if there is a pre_calculator or a previous calculation ID.
If yes, read the inputs by invoking the precalculator or by retrieving
the previous calculation; if not, read the inputs directly.
"""
job_info = hdf5.LiteralAttrs()
if self.pre_calculator is not None:
# the parameter hazard_calculation_id is only meaningful if
# there is a precalculator
precalc_id = self.oqparam.hazard_calculation_id
if precalc_id is None: # recompute everything
self.compute_previous()
else: # read previously computed data
self.read_previous(precalc_id)
self.init()
else: # we are in a basic calculator
self.basic_pre_execute()
if 'source' in self.oqparam.inputs:
vars(job_info).update(readinput.get_job_info(
self.oqparam, self.csm, self.sitecol))
job_info.hostname = socket.gethostname()
if hasattr(self, 'riskmodel'):
job_info.require_epsilons = bool(self.riskmodel.covs)
self.job_info = job_info
self.datastore.flush()
try:
csm_info = self.datastore['csm_info']
except KeyError:
pass
else:
csm_info.gsim_lt.check_imts(self.oqparam.imtls)
def pre_execute(self):
"""
Check if there is a pre_calculator or a previous calculation ID.
If yes, read the inputs by invoking the precalculator or by retrieving
the previous calculation; if not, read the inputs directly.
"""
precalc_id = self.oqparam.hazard_calculation_id
job_info = {}
if self.pre_calculator is not None:
# the parameter hazard_calculation_id is only meaningful if
# there is a precalculator
self.precalc = (self.compute_previous() if precalc_id is None else
self.read_previous(precalc_id))
self.init()
else: # we are in a basic calculator
self.precalc = None
self.basic_pre_execute()
if 'source' in self.oqparam.inputs:
job_info.update(
readinput.get_job_info(self.oqparam, self.csm,
self.sitecol))
if hasattr(self, 'riskmodel'):
job_info['require_epsilons'] = bool(self.riskmodel.covs)
self._monitor.save_info(job_info)
self.param = {} # used in the risk calculators
def read_sources(self):
"""
Read the composite source model (if any).
This method must be called after read_exposure_sitecol, to be able
to filter to sources according to the site collection.
"""
if 'source' in self.oqparam.inputs:
logging.info('Reading the composite source model')
with self.monitor(
'reading composite source model', autoflush=True):
self.csm = readinput.get_composite_source_model(
self.oqparam, self.sitecol, self.SourceProcessor,
self.monitor)
# we could manage limits here
self.source_pre_info = self.csm.source_info
self.job_info = readinput.get_job_info(
self.oqparam, self.csm, self.sitecol)
self.csm.count_ruptures()
self.rlzs_assoc = self.csm.get_rlzs_assoc()
logging.info(
'Total weight of the sources=%s',
self.job_info['input_weight'])
logging.info(
'Expected output size=%s',
self.job_info['output_weight'])
def pre_execute(self):
"""
Check if there is a pre_calculator or a previous calculation ID.
If yes, read the inputs by invoking the precalculator or by retrieving
the previous calculation; if not, read the inputs directly.
"""
job_info = hdf5.LiteralAttrs()
if self.pre_calculator is not None:
# the parameter hazard_calculation_id is only meaningful if
# there is a precalculator
precalc_id = self.oqparam.hazard_calculation_id
if precalc_id is None: # recompute everything
self.compute_previous()
else: # read previously computed data
self.read_previous(precalc_id)
self.init()
else: # we are in a basic calculator
self.basic_pre_execute()
if 'source' in self.oqparam.inputs:
vars(job_info).update(
readinput.get_job_info(self.oqparam, self.csm,
self.sitecol))
job_info.hostname = socket.gethostname()
if hasattr(self, 'riskmodel'):
job_info.require_epsilons = bool(self.riskmodel.covs)
self.job_info = job_info
self.datastore.flush()
try:
csm_info = self.datastore['csm_info']
except KeyError:
pass
else:
csm_info.gsim_lt.check_imts(self.oqparam.imtls)
def read_sources(self):
"""
Read the composite source model (if any).
This method must be called after read_risk_data, to be able
to filter to sources according to the site collection.
"""
if 'source' in self.oqparam.inputs:
logging.info('Reading the composite source model')
with self.monitor('reading composite source model',
autoflush=True):
self.csm = readinput.get_composite_source_model(
self.oqparam,
self.sitecol,
self.SourceProcessor,
self.monitor,
dstore=self.datastore)
# we could manage limits here
self.job_info = readinput.get_job_info(self.oqparam, self.csm,
self.sitecol)
self.rlzs_assoc = self.csm.get_rlzs_assoc()
logging.info('Total weight of the sources=%s',
self.job_info['input_weight'])
logging.info('Expected output size=%s',
self.job_info['output_weight'])
def pre_execute(self):
"""
Check if there is a pre_calculator or a previous calculation ID.
If yes, read the inputs by invoking the precalculator or by retrieving
the previous calculation; if not, read the inputs directly.
"""
job_info = hdf5.LiteralAttrs()
if self.pre_calculator is not None:
# the parameter hazard_calculation_id is only meaningful if
# there is a precalculator
precalc_id = self.oqparam.hazard_calculation_id
if precalc_id is None: # recompute everything
precalc = calculators[self.pre_calculator](
self.oqparam, self.monitor("precalculator"), self.datastore.calc_id
)
precalc.run()
if "scenario" not in self.oqparam.calculation_mode:
self.csm = precalc.csm
pre_attrs = vars(precalc)
for name in ("riskmodel", "assets_by_site"):
if name in pre_attrs:
setattr(self, name, getattr(precalc, name))
else: # read previously computed data
parent = datastore.read(precalc_id)
self.datastore.set_parent(parent)
# copy missing parameters from the parent
params = {
name: value for name, value in vars(parent["oqparam"]).items() if name not in vars(self.oqparam)
}
self.save_params(**params)
self.read_risk_data()
self.init()
else: # we are in a basic calculator
self.read_risk_data()
if "source" in self.oqparam.inputs:
with self.monitor("reading composite source model", autoflush=True):
self.csm = readinput.get_composite_source_model(self.oqparam)
self.datastore["csm_info"] = self.csm.info
self.rup_data = {}
# we could manage limits here
vars(job_info).update(readinput.get_job_info(self.oqparam, self.csm, self.sitecol))
logging.info("Expected output size=%s", job_info.hazard["output_weight"])
logging.info("Total weight of the sources=%s", job_info.hazard["input_weight"])
self.init()
if "source" in self.oqparam.inputs:
with self.monitor("managing sources", autoflush=True):
self.send_sources()
self.manager.store_source_info(self.datastore)
attrs = self.datastore.hdf5["composite_source_model"].attrs
attrs["weight"] = self.csm.weight
attrs["filtered_weight"] = self.csm.filtered_weight
attrs["maxweight"] = self.csm.maxweight
job_info.hostname = socket.gethostname()
if hasattr(self, "riskmodel"):
job_info.require_epsilons = bool(self.riskmodel.covs)
self.job_info = job_info
self.datastore.flush()
def pre_execute(self):
"""
Read the full source model and sites and build the needed tiles
"""
self.oqparam = self.job.get_oqparam()
source_model_lt = readinput.get_source_model_lt(self.oqparam)
source_models = list(
readinput.get_source_models(self.oqparam, source_model_lt))
self.parse_risk_model()
self.initialize_site_collection()
info = readinput.get_job_info(self.oqparam, source_models,
self.site_collection)
self.imtls = self.oqparam.imtls
weight = info['n_sites'] * info['n_levels'] * info['max_realizations']
nblocks = math.ceil(weight / self.oqparam.maximum_tile_weight)
self.tiles = list(split_in_blocks(self.site_collection, nblocks))
self.num_tiles = len(self.tiles)
def _print_info(assoc, oqparam, csm, sitecol,
filtersources=True, weightsources=True):
print(assoc.csm_info)
print('See https://github.com/gem/oq-risklib/blob/master/doc/'
'effective-realizations.rst for an explanation')
print(assoc)
if filtersources or weightsources:
info = readinput.get_job_info(oqparam, csm, sitecol)
info['n_sources'] = csm.get_num_sources()
info['c_matrix'] = humansize(
info['n_sites'] * info['n_levels'] *
info['n_imts'] * len(assoc) * 8)
for k in sorted(info):
if k == 'input_weight' and not weightsources:
pass
else:
print(k, info[k])
def pre_execute(self):
"""
Read the full source model and sites and build the needed tiles
"""
self.oqparam = self.job.get_oqparam()
source_model_lt = readinput.get_source_model_lt(self.oqparam)
source_models = list(readinput.get_source_models(
self.oqparam, source_model_lt))
self.parse_risk_model()
self.initialize_site_collection()
info = readinput.get_job_info(
self.oqparam, source_models, self.site_collection)
self.imtls = self.oqparam.imtls
weight = info['n_sites'] * info['n_levels'] * info['max_realizations']
nblocks = math.ceil(weight / self.oqparam.maximum_tile_weight)
self.tiles = list(split_in_blocks(self.site_collection, nblocks))
self.num_tiles = len(self.tiles)
def pre_execute(self):
"""
Initialize risk models, site model and sources
"""
self.parse_risk_model()
self.initialize_site_collection()
self.initialize_sources()
info = readinput.get_job_info(
self.oqparam, self.composite_model, self.site_collection)
models.JobInfo.objects.create(
oq_job=self.job,
num_sites=info['n_sites'],
num_realizations=info['max_realizations'],
num_imts=info['n_imts'],
num_levels=info['n_levels'],
input_weight=info['input_weight'],
output_weight=info['output_weight'])
self.check_limits(info['input_weight'], info['output_weight'])
self.init_zeros_ones()
return info['input_weight'], info['output_weight']
def _print_info(dstore, filtersources=True, weightsources=True):
assoc = dstore["rlzs_assoc"]
oqparam = OqParam.from_(dstore.attrs)
csm = dstore["composite_source_model"]
sitecol = dstore["sitecol"]
print(csm.get_info())
print("See https://github.com/gem/oq-risklib/blob/master/doc/" "effective-realizations.rst for an explanation")
print(assoc)
if filtersources or weightsources:
[info] = readinput.get_job_info(oqparam, csm, sitecol)
info["n_sources"] = csm.get_num_sources()
curve_matrix_size = info["n_sites"] * info["n_levels"] * info["n_imts"] * len(assoc) * 8
for k in info.dtype.fields:
if k == "input_weight" and not weightsources:
pass
else:
print(k, info[k])
print("curve_matrix_size", humansize(curve_matrix_size))
if "num_ruptures" in dstore:
print(datastore.view("rupture_collections", dstore))
def pre_execute(self):
"""
Initialize risk models, site model and sources
"""
self.parse_risk_model()
self.initialize_site_collection()
self.initialize_sources()
info = readinput.get_job_info(self.oqparam, self.composite_model,
self.site_collection)
models.JobInfo.objects.create(
oq_job=self.job,
num_sites=info['n_sites'],
num_realizations=info['max_realizations'],
num_imts=info['n_imts'],
num_levels=info['n_levels'],
input_weight=info['input_weight'],
output_weight=info['output_weight'])
self.check_limits(info['input_weight'], info['output_weight'])
self.init_zeros_ones()
return info['input_weight'], info['output_weight']
def pre_execute(self):
"""
Read the site collection and the sources.
"""
if 'exposure' in self.oqparam.inputs:
logging.info('Reading the exposure')
exposure = readinput.get_exposure(self.oqparam)
self.sitecol, _assets = readinput.get_sitecol_assets(
self.oqparam, exposure)
else:
logging.info('Reading the site collection')
self.sitecol = readinput.get_site_collection(self.oqparam)
logging.info('Reading the effective source models')
source_models = list(
readinput.get_effective_source_models(self.oqparam, self.sitecol))
self.all_sources = [src for src_model in source_models
for trt_model in src_model.trt_models
for src in trt_model]
self.job_info = readinput.get_job_info(
self.oqparam, source_models, self.sitecol)
def _print_info(dstore, filtersources=True, weightsources=True):
assoc = dstore['rlzs_assoc']
oqparam = OqParam.from_(dstore.attrs)
csm = dstore['composite_source_model']
sitecol = dstore['sitecol']
print(csm.get_info())
print('See https://github.com/gem/oq-risklib/blob/master/doc/'
'effective-realizations.rst for an explanation')
print(assoc)
if filtersources or weightsources:
[info] = readinput.get_job_info(oqparam, csm, sitecol)
info['n_sources'] = csm.get_num_sources()
curve_matrix_size = (info['n_sites'] * info['n_levels'] *
info['n_imts'] * len(assoc) * 8)
for k in info.dtype.fields:
if k == 'input_weight' and not weightsources:
pass
else:
print(k, info[k])
print('curve_matrix_size', humansize(curve_matrix_size))
if 'num_ruptures' in dstore:
print(datastore.view('rupture_collections', dstore))
def pre_execute(self):
"""
Initialize risk models, site model and sources
"""
# if you don't use an explicit transaction, errors will be masked
# the problem is that Django by default performs implicit transactions
# without rollback, see
# https://docs.djangoproject.com/en/1.3/topics/db/transactions/
with transaction.commit_on_success(using='job_init'):
self.parse_risk_model()
with transaction.commit_on_success(using='job_init'):
self.initialize_site_collection()
with transaction.commit_on_success(using='job_init'):
source_models = self.initialize_sources()
info = readinput.get_job_info(
self.hc, source_models, self.site_collection)
with transaction.commit_on_success(using='job_init'):
models.JobInfo.objects.create(
oq_job=self.job,
num_sites=info['n_sites'],
num_realizations=info['max_realizations'],
num_imts=info['n_imts'],
num_levels=info['n_levels'],
input_weight=info['input_weight'],
output_weight=info['output_weight'])
self.check_limits(info['input_weight'], info['output_weight'])
self.all_sources = [src for sm in source_models
for trt_model in sm.trt_models
for src in trt_model]
self.imtls = self.hc.intensity_measure_types_and_levels
if info['n_levels']: # we can compute hazard curves
self.zeros = numpy.array(
[numpy.zeros((info['n_sites'], len(self.imtls[imt])))
for imt in sorted(self.imtls)])
self.ones = [numpy.zeros(len(self.imtls[imt]), dtype=float)
for imt in sorted(self.imtls)]
return info['input_weight'], info['output_weight']
def _print_info(dstore, filtersources=True, weightsources=True):
assoc = dstore['rlzs_assoc']
oqparam = dstore['oqparam']
csm = dstore['composite_source_model']
sitecol = dstore['sitecol']
print(csm.get_info())
print('See https://github.com/gem/oq-risklib/blob/master/doc/'
'effective-realizations.rst for an explanation')
print(assoc)
if filtersources or weightsources:
[info] = readinput.get_job_info(oqparam, csm, sitecol)
info['n_sources'] = csm.get_num_sources()
curve_matrix_size = (
info['n_sites'] * info['n_levels'] *
info['n_imts'] * len(assoc) * 8)
for k in info.dtype.fields:
if k == 'input_weight' and not weightsources:
pass
else:
print(k, info[k])
print('curve_matrix_size', humansize(curve_matrix_size))
if 'num_ruptures' in dstore:
print(datastore.view('rupture_collections', dstore))
def pre_execute(self):
"""
Check if there is a pre_calculator or a previous calculation ID.
If yes, read the inputs by invoking the precalculator or by retrieving
the previous calculation; if not, read the inputs directly.
"""
job_info = hdf5.LiteralAttrs()
if self.pre_calculator is not None:
# the parameter hazard_calculation_id is only meaningful if
# there is a precalculator
precalc_id = self.oqparam.hazard_calculation_id
if precalc_id is None: # recompute everything
precalc = calculators[self.pre_calculator](
self.oqparam, self.monitor('precalculator'),
self.datastore.calc_id)
precalc.run()
if 'scenario' not in self.oqparam.calculation_mode:
self.csm = precalc.csm
pre_attrs = vars(precalc)
for name in ('riskmodel', 'assets_by_site'):
if name in pre_attrs:
setattr(self, name, getattr(precalc, name))
else: # read previously computed data
parent = datastore.read(precalc_id)
self.datastore.set_parent(parent)
# copy missing parameters from the parent
params = {name: value for name, value in
vars(parent['oqparam']).items()
if name not in vars(self.oqparam)}
self.save_params(**params)
self.read_risk_data()
else: # we are in a basic calculator
self.read_risk_data()
if 'source' in self.oqparam.inputs:
with self.monitor(
'reading composite source model', autoflush=True):
self.csm = readinput.get_composite_source_model(
self.oqparam)
self.rlzs_assoc = self.csm.info.get_rlzs_assoc()
self.datastore['csm_info'] = self.rlzs_assoc.csm_info
self.rup_data = {}
# we could manage limits here
vars(job_info).update(readinput.get_job_info(
self.oqparam, self.csm, self.sitecol))
logging.info('Expected output size=%s',
job_info.hazard['output_weight'])
logging.info('Total weight of the sources=%s',
job_info.hazard['input_weight'])
self.init()
with self.monitor('managing sources', autoflush=True):
self.send_sources()
self.manager.store_source_info(self.datastore)
attrs = self.datastore.hdf5['composite_source_model'].attrs
attrs['weight'] = self.csm.weight
attrs['filtered_weight'] = self.csm.filtered_weight
attrs['maxweight'] = self.csm.maxweight
job_info.hostname = socket.gethostname()
if hasattr(self, 'riskmodel'):
job_info.require_epsilons = bool(self.riskmodel.covs)
self.job_info = job_info
self.datastore.flush()