def run(self, pre_execute=True, concurrent_tasks=None, close=True, **kw):
"""
Run the calculation and return the exported outputs.
"""
global logversion
with self._monitor:
self.close = close
self.set_log_format()
if logversion: # make sure this is logged only once
logging.info('Running %s', self.oqparam.inputs['job_ini'])
logging.info('Using engine version %s', engine_version)
logversion = False
if concurrent_tasks is None: # use the job.ini parameter
ct = self.oqparam.concurrent_tasks
else: # used the parameter passed in the command-line
ct = concurrent_tasks
if ct == 0: # disable distribution temporarily
oq_distribute = os.environ.get('OQ_DISTRIBUTE')
os.environ['OQ_DISTRIBUTE'] = 'no'
if ct != self.oqparam.concurrent_tasks:
# save the used concurrent_tasks
self.oqparam.concurrent_tasks = ct
self.save_params(**kw)
Starmap.init()
try:
if pre_execute:
self.pre_execute()
self.result = self.execute()
if self.result is not None:
self.post_execute(self.result)
self.before_export()
self.export(kw.get('exports', ''))
except Exception:
if kw.get('pdb'): # post-mortem debug
tb = sys.exc_info()[2]
traceback.print_tb(tb)
pdb.post_mortem(tb)
else:
logging.critical('', exc_info=True)
raise
finally:
# cleanup globals
if ct == 0: # restore OQ_DISTRIBUTE
if oq_distribute is None: # was not set
del os.environ['OQ_DISTRIBUTE']
else:
os.environ['OQ_DISTRIBUTE'] = oq_distribute
readinput.pmap = None
readinput.exposure = None
Starmap.shutdown()
self._monitor.flush()
return getattr(self, 'exported', {})
def find_matching_poe_parallel_runs(self, target, quantity='poe', tol=None, nsol=1, outputfile=None):
"""
Returns a list of vector-valued coordinates corresponding to the Multi-Dimensional Hazard
Curve ARE/POE value TARGET (within tolerance interval +/- TOL). This list of
coordinates is obtained using an optimization algorithm. Parallelization is realized by
sending one individual optimization run on each worker.
:return: Coordinate of vector-sample with matching QUANTITY=TARGET
"""
# TOL: Tolerance on cost-function evaluation w/r to TARGET:
if tol is None:
tol = target/1E3
lower_bound = [np.log(min(self.oqparam.imtls[str(p)])) for p in self.periods]
upper_bound = [np.log(max(self.oqparam.imtls[str(p)])) for p in self.periods]
coord = np.empty( (nsol, 3+len(self.periods)) )
# coord[i,:] = [ ARE_OR_POE, N_ITER, N_FEV, SA_1, ..., SA_N]
worker_args = list()
for i in range(nsol):
rs = np.random.RandomState(seed=np.random.random_integers(0,1E9))
worker_args.append((getattr(self, quantity), target, lower_bound, upper_bound, tol, rs))
i = 0
for res in Starmap(_root_finder_worker, worker_args):
logging.info('Starting point: {}'.format(res.x0))
logging.info('{}/{}: Convergence met for sample {} ({}={})'.format(
i+1,nsol,np.exp(res.x),quantity,res.fun+target))
coord[i, 0] = res.fun+target # Evaluate ARE/POE at solution
coord[i, 1] = res.nit
coord[i, 2] = res.nfev
coord[i, 3:] = np.exp(res.x) # Convert lnSA to SA in units of g
i = i + 1
with open(outputfile, 'ab') as f:
np.savetxt(f, coord, fmt='%.6e', delimiter=',')
def are_parallel(self, lnSA):
"""
Returns the vector-valued annual rate of exceedance
param *lnSA: tuple, natural logarithm of acceleration values, in unit of g.
"""
args_list = list()
for rlz in self.ssm_lt: # Loop over realizations
_, weight = parser.get_value_and_weight_from_rlz(rlz)
srcs = parser.get_sources_from_rlz(rlz, self.oqparam, self.ssm_lt, sourcefilter=self.srcfilter)
for src in srcs: # Loop over (filtered) seismic sources (area, fault, etc...)
for pt in self.srcfilter.filter(src): # Loop over point-sources
gsim_lt = get_gsim_lt(self.oqparam, trts=[src.tectonic_region_type])
for gsim_rlz in gsim_lt: # Loop over GSIM Logic_tree
gsim_model, gsim_weight = parser.get_value_and_weight_from_gsim_rlz(gsim_rlz)
# Distribute ARE:
pt_weight = weight*gsim_weight
args = (self, pt, gsim_model, pt_weight, lnSA)
args_list.append(args)
are = 0
for value in Starmap(self.pt_src_are.__func__, args_list):
are += value
return are
def save_hmaps(self):
"""
Save hazard maps generated from the hazard curves
"""
oq = self.oqparam
if oq.poes:
mon = self.monitor('computing hazard maps')
logging.info('Computing hazard maps for PoEs=%s', oq.poes)
with mon:
N = len(self.sitecol.complete)
ct = oq.concurrent_tasks or 1
if 'hcurves' in self.datastore:
kinds = self.datastore['hcurves']
hmaps_dt = numpy.dtype(
[('%s-%s' % (imt, poe), F32)
for imt in oq.imtls for poe in oq.poes])
for kind in kinds:
self.datastore.create_dset(
'hmaps/' + kind, hmaps_dt, (N,), fillvalue=None)
allargs = []
for slc in general.split_in_slices(N, ct):
hcurves_by_kind = {
kind: self.datastore['hcurves/' + kind][slc]
for kind in kinds}
allargs.append((hcurves_by_kind, slc,
oq.imtls, oq.poes, mon))
for dic, slc in Starmap(build_hmaps, allargs):
for kind, hmaps in dic.items():
self.datastore['hmaps/' + kind][slc] = hmaps
else: # single realization
pg = PmapGetter(self.datastore, self.rlzs_assoc)
self.datastore['hmaps/mean'] = calc.make_hmap_array(
pg.get_mean(), oq.imtls, oq.poes, N)
def pfilter(self, sources, monitor):
"""
Filter the sources in parallel by using Starmap.apply
:param sources: a sequence of sources
:param monitor: a Monitor instance
:returns: a dictionary src_group_id -> sources
"""
sources_by_grp = Starmap.apply(prefilter, (sources, self, monitor),
distribute=self.distribute,
name=self.__class__.__name__).reduce()
Starmap.shutdown() # close the processpool
Starmap.init() # reopen it when necessary
# avoid task ordering issues
for sources in sources_by_grp.values():
sources.sort(key=operator.attrgetter('source_id'))
return sources_by_grp
def execute(self):
"""
Parallelize on the riskinputs and returns a dictionary of results.
Require a `.core_task` to be defined with signature
(riskinputs, riskmodel, rlzs_assoc, monitor).
"""
mon = self.monitor('risk')
all_args = [(riskinput, self.riskmodel, self.param, mon)
for riskinput in self.riskinputs]
res = Starmap(self.core_task.__func__, all_args).reduce(self.combine)
return res
def main(dirname):
dname = pathlib.Path(dirname)
with hdf5new() as hdf5: # create a new datastore
monitor = Monitor('count', hdf5) # create a new monitor
iterargs = ((open(dname/fname, encoding='utf-8').read(),)
for fname in os.listdir(dname)
if fname.endswith('.rst')) # read the docs
c = collections.Counter() # intially empty counter
for counter in Starmap(count, iterargs, monitor):
c += counter
print(c) # total counts
print('Performance info stored in', hdf5)
def start_tasks(self, sm_id, ruptures_by_grp, sitecol, assetcol, riskmodel,
imts, trunc_level, correl_model, min_iml, monitor):
"""
:param sm_id: source model ordinal
:param ruptures_by_grp: dictionary of ruptures by src_group_id
:param sitecol: a SiteCollection instance
:param assetcol: an AssetCollection instance
:param riskmodel: a RiskModel instance
:param imts: a list of Intensity Measure Types
:param trunc_level: truncation level
:param correl_model: correlation model
:param min_iml: vector of minimum intensities, one per IMT
:param monitor: a Monitor instance
:returns: an IterResult instance
"""
csm_info = self.csm_info.get_info(sm_id)
grp_ids = sorted(csm_info.get_sm_by_grp())
rlzs_assoc = csm_info.get_rlzs_assoc(
count_ruptures=lambda grp: len(ruptures_by_grp.get(grp.id, [])))
num_events = sum(ebr.multiplicity for grp in ruptures_by_grp
for ebr in ruptures_by_grp[grp])
seeds = self.oqparam.random_seed + numpy.arange(num_events)
allargs = []
# prepare the risk inputs
ruptures_per_block = self.oqparam.ruptures_per_block
start = 0
ignore_covs = self.oqparam.ignore_covs
for grp_id in grp_ids:
rlzs_by_gsim = rlzs_assoc.get_rlzs_by_gsim(grp_id)
samples = rlzs_assoc.samples[grp_id]
for rupts in block_splitter(ruptures_by_grp.get(grp_id, []),
ruptures_per_block):
if ignore_covs or not self.riskmodel.covs:
eps = None
elif self.oqparam.asset_correlation:
eps = EpsilonMatrix1(num_events, self.oqparam.master_seed)
else:
n_events = sum(ebr.multiplicity for ebr in rupts)
eps = EpsilonMatrix0(len(self.assetcol),
seeds[start:start + n_events])
start += n_events
getter = riskinput.GmfGetter(grp_id, rlzs_by_gsim, rupts,
sitecol, imts, min_iml,
trunc_level, correl_model,
samples)
ri = riskinput.RiskInputFromRuptures(getter, eps)
allargs.append((ri, riskmodel, assetcol, monitor))
self.vals = self.assetcol.values()
taskname = '%s#%d' % (event_based_risk.__name__, sm_id + 1)
ires = Starmap(event_based_risk, allargs, name=taskname).submit_all()
ires.num_ruptures = {
sg_id: len(rupts)
for sg_id, rupts in ruptures_by_grp.items()
}
ires.num_events = num_events
ires.num_rlzs = len(rlzs_assoc.realizations)
ires.sm_id = sm_id
return ires
def execute(self):
# build loss maps
if ('all_loss_ratios' in self.datastore
and self.oqparam.conditional_loss_poes):
assetcol = self.assetcol
rlzs = self.rlzs_assoc.realizations
stats = self.oqparam.risk_stats()
builder = self.riskmodel.curve_builder
A = len(assetcol)
R = len(self.datastore['realizations'])
# create loss_maps datasets
self.datastore.create_dset('loss_maps-rlzs',
builder.loss_maps_dt, (A, R),
fillvalue=None)
if R > 1:
self.datastore.create_dset('loss_maps-stats',
builder.loss_maps_dt,
(A, len(stats)),
fillvalue=None)
mon = self.monitor('loss maps')
if self.oqparam.hazard_calculation_id and (
'asset_loss_table' in self.datastore.parent):
Starmap = parallel.Starmap # we can parallelize fully
lrgetter = riskinput.LossRatiosGetter(self.datastore.parent)
# avoid OSError: Can't read data (Wrong b-tree signature)
self.datastore.parent.close()
else: # there is a single datastore
# we cannot read from it in parallel while writing
Starmap = parallel.Sequential
lrgetter = riskinput.LossRatiosGetter(self.datastore)
Starmap.apply(build_loss_maps,
(assetcol, builder, lrgetter, rlzs, stats, mon),
self.oqparam.concurrent_tasks).reduce(
self.save_loss_maps)
if self.oqparam.hazard_calculation_id:
self.datastore.parent.open()
# build an aggregate loss curve per realization
if 'agg_loss_table' in self.datastore:
self.build_agg_curve()
def hazard_matrix_calculation_parallel(self, quantity='poe'):
"""
Compute exhaustively the full VPSHA hazard matrix of ARE/POE over the N-dimensional space of
spectral periods or parameters.
NOTE: Parallelization occurs on the loop over N-D hazard matrix cells
WARNING !! This computation can be extremely expensive for high-dimensional problems !
"""
# Initialization step:
hc_calc_method = getattr(self, quantity) # self.poe or self.are method
args = list()
shape = (len(self.sites),) + tuple(len(self.oqparam.imtls[str(p)]) for p in self.periods)
max_nb = np.prod(shape)
logging.warning('hazard matrix shape: [N_sites x N_IMT_1 x ... x N_IMT_k]: {}'.format(shape))
logging.warning('hazard matrix has {} elements'.format(max_nb))
acc_discretization = [np.log(self.oqparam.imtls[str(p)]) for p in self.periods]
# create a N-dimensional mesh of spectral acceleration values:
acc_meshes = np.meshgrid(*acc_discretization, indexing='ij', copy=False)
nelts = int(np.prod(shape[1:])) # Number of N-D pseudo spectral values
for i in range(nelts):
indices = np.unravel_index(i, shape[1:]) # Flat to multi-dimensional index
accels = [ acc_meshes[j][indices] for j in range(self.ndims)]
logging.debug(f" # Current acceleration vector: {tuple(str(p) for p in self.periods)} = {accels}\n")
# Call hazard curve computation method:
#hazard_output = hc_calc_method(accels)
args.append((indices, hc_calc_method, accels))
"""
# Sort results for each site:
for k in range(len(hazard_output)):
# Loop on sites, i.e. 1st dimension of "hazard_output"
# indx = np.ravel_multi_index((k,)+indices, shape)
# indices = np.unravel_index(indx, shape)
output[(k,) + indices] = hazard_output[k]
"""
output = np.empty(shape)
for result in Starmap(_matrix_cell_worker, args):
# Sort results for each site:
for k in range(len(result['output'])):
# Loop on sites, i.e. 1st dimension of "hazard_output"
# indx = np.ravel_multi_index((k,)+indices, shape)
# indices = np.unravel_index(indx, shape)
output[(k,) + result['indices']] = result['output'][k]
self.hc.hazard_matrix = output
return self.hc
def execute(self):
"""
Parallelize on the riskinputs and returns a dictionary of results.
Require a `.core_task` to be defined with signature
(riskinputs, riskmodel, rlzs_assoc, monitor).
"""
if not hasattr(self, 'riskinputs'): # in the reportwriter
return
res = Starmap.apply(
self.core_task.__func__,
(self.riskinputs, self.riskmodel, self.param, self.monitor()),
concurrent_tasks=self.oqparam.concurrent_tasks or 1,
weight=get_weight).reduce(self.combine)
return res
def run_calc(request):
"""
Run a calculation.
:param request:
a `django.http.HttpRequest` object.
If the request has the attribute `hazard_job_id`, the results of the
specified hazard calculations will be re-used as input by the risk
calculation.
The request also needs to contain the files needed to perform the
calculation. They can be uploaded as separate files, or zipped
together.
"""
hazard_job_id = request.POST.get('hazard_job_id')
if hazard_job_id:
hazard_job_id = int(hazard_job_id)
candidates = ("job_risk.ini", "job.ini")
else:
candidates = ("job_hazard.ini", "job_haz.ini", "job.ini")
einfo, exctype, monitor = safely_call(_prepare_job, (request, candidates))
if exctype:
return HttpResponse(json.dumps(einfo.splitlines()),
content_type=JSON,
status=500)
if not einfo:
msg = 'Could not find any file of the form %s' % str(candidates)
logging.error(msg)
return HttpResponse(content=json.dumps([msg]),
content_type=JSON,
status=500)
user = utils.get_user_data(request)
try:
job_id, fut = submit_job(einfo[0], user['name'], hazard_job_id)
# restart the process pool at the end of each job
fut.add_done_callback(lambda f: Starmap.restart())
except Exception as exc: # no job created, for instance missing .xml file
# get the exception message
exc_msg = str(exc)
logging.error(exc_msg)
response_data = exc_msg.splitlines()
status = 500
else:
response_data = dict(job_id=job_id, status='created')
status = 200
return HttpResponse(content=json.dumps(response_data),
content_type=JSON,
status=status)
def execute(self):
"""
Parallelize on the riskinputs and returns a dictionary of results.
Require a `.core_task` to be defined with signature
(riskinputs, riskmodel, rlzs_assoc, monitor).
"""
if not hasattr(self, 'riskinputs'): # in the reportwriter
return
res = Starmap.apply(
self.core_task.__func__,
(self.riskinputs, self.riskmodel, self.param, self.monitor()),
concurrent_tasks=self.oqparam.concurrent_tasks or 1,
weight=get_weight
).reduce(self.combine)
return res
def test(self):
mon = Monitor()
iterargs = ((i,) for i in range(10))
smap = Starmap(double, iterargs, mon, distribute='zmq')
self.assertEqual(sum(res for res in smap), 90)
def tearDown(cls):
Starmap.shutdown()
def tearDown(self):
Starmap.shutdown()
