def run(self, pre_execute=True, concurrent_tasks=None, close=True, **kw): """ Run the calculation and return the exported outputs. """ global logversion with self._monitor: self.close = close self.set_log_format() if logversion: # make sure this is logged only once logging.info('Running %s', self.oqparam.inputs['job_ini']) logging.info('Using engine version %s', engine_version) logversion = False if concurrent_tasks is None: # use the job.ini parameter ct = self.oqparam.concurrent_tasks else: # used the parameter passed in the command-line ct = concurrent_tasks if ct == 0: # disable distribution temporarily oq_distribute = os.environ.get('OQ_DISTRIBUTE') os.environ['OQ_DISTRIBUTE'] = 'no' if ct != self.oqparam.concurrent_tasks: # save the used concurrent_tasks self.oqparam.concurrent_tasks = ct self.save_params(**kw) Starmap.init() try: if pre_execute: self.pre_execute() self.result = self.execute() if self.result is not None: self.post_execute(self.result) self.before_export() self.export(kw.get('exports', '')) except Exception: if kw.get('pdb'): # post-mortem debug tb = sys.exc_info()[2] traceback.print_tb(tb) pdb.post_mortem(tb) else: logging.critical('', exc_info=True) raise finally: # cleanup globals if ct == 0: # restore OQ_DISTRIBUTE if oq_distribute is None: # was not set del os.environ['OQ_DISTRIBUTE'] else: os.environ['OQ_DISTRIBUTE'] = oq_distribute readinput.pmap = None readinput.exposure = None Starmap.shutdown() self._monitor.flush() return getattr(self, 'exported', {})
def find_matching_poe_parallel_runs(self, target, quantity='poe', tol=None, nsol=1, outputfile=None): """ Returns a list of vector-valued coordinates corresponding to the Multi-Dimensional Hazard Curve ARE/POE value TARGET (within tolerance interval +/- TOL). This list of coordinates is obtained using an optimization algorithm. Parallelization is realized by sending one individual optimization run on each worker. :return: Coordinate of vector-sample with matching QUANTITY=TARGET """ # TOL: Tolerance on cost-function evaluation w/r to TARGET: if tol is None: tol = target/1E3 lower_bound = [np.log(min(self.oqparam.imtls[str(p)])) for p in self.periods] upper_bound = [np.log(max(self.oqparam.imtls[str(p)])) for p in self.periods] coord = np.empty( (nsol, 3+len(self.periods)) ) # coord[i,:] = [ ARE_OR_POE, N_ITER, N_FEV, SA_1, ..., SA_N] worker_args = list() for i in range(nsol): rs = np.random.RandomState(seed=np.random.random_integers(0,1E9)) worker_args.append((getattr(self, quantity), target, lower_bound, upper_bound, tol, rs)) i = 0 for res in Starmap(_root_finder_worker, worker_args): logging.info('Starting point: {}'.format(res.x0)) logging.info('{}/{}: Convergence met for sample {} ({}={})'.format( i+1,nsol,np.exp(res.x),quantity,res.fun+target)) coord[i, 0] = res.fun+target # Evaluate ARE/POE at solution coord[i, 1] = res.nit coord[i, 2] = res.nfev coord[i, 3:] = np.exp(res.x) # Convert lnSA to SA in units of g i = i + 1 with open(outputfile, 'ab') as f: np.savetxt(f, coord, fmt='%.6e', delimiter=',')
def are_parallel(self, lnSA): """ Returns the vector-valued annual rate of exceedance param *lnSA: tuple, natural logarithm of acceleration values, in unit of g. """ args_list = list() for rlz in self.ssm_lt: # Loop over realizations _, weight = parser.get_value_and_weight_from_rlz(rlz) srcs = parser.get_sources_from_rlz(rlz, self.oqparam, self.ssm_lt, sourcefilter=self.srcfilter) for src in srcs: # Loop over (filtered) seismic sources (area, fault, etc...) for pt in self.srcfilter.filter(src): # Loop over point-sources gsim_lt = get_gsim_lt(self.oqparam, trts=[src.tectonic_region_type]) for gsim_rlz in gsim_lt: # Loop over GSIM Logic_tree gsim_model, gsim_weight = parser.get_value_and_weight_from_gsim_rlz(gsim_rlz) # Distribute ARE: pt_weight = weight*gsim_weight args = (self, pt, gsim_model, pt_weight, lnSA) args_list.append(args) are = 0 for value in Starmap(self.pt_src_are.__func__, args_list): are += value return are
def save_hmaps(self): """ Save hazard maps generated from the hazard curves """ oq = self.oqparam if oq.poes: mon = self.monitor('computing hazard maps') logging.info('Computing hazard maps for PoEs=%s', oq.poes) with mon: N = len(self.sitecol.complete) ct = oq.concurrent_tasks or 1 if 'hcurves' in self.datastore: kinds = self.datastore['hcurves'] hmaps_dt = numpy.dtype( [('%s-%s' % (imt, poe), F32) for imt in oq.imtls for poe in oq.poes]) for kind in kinds: self.datastore.create_dset( 'hmaps/' + kind, hmaps_dt, (N,), fillvalue=None) allargs = [] for slc in general.split_in_slices(N, ct): hcurves_by_kind = { kind: self.datastore['hcurves/' + kind][slc] for kind in kinds} allargs.append((hcurves_by_kind, slc, oq.imtls, oq.poes, mon)) for dic, slc in Starmap(build_hmaps, allargs): for kind, hmaps in dic.items(): self.datastore['hmaps/' + kind][slc] = hmaps else: # single realization pg = PmapGetter(self.datastore, self.rlzs_assoc) self.datastore['hmaps/mean'] = calc.make_hmap_array( pg.get_mean(), oq.imtls, oq.poes, N)
def pfilter(self, sources, monitor): """ Filter the sources in parallel by using Starmap.apply :param sources: a sequence of sources :param monitor: a Monitor instance :returns: a dictionary src_group_id -> sources """ sources_by_grp = Starmap.apply(prefilter, (sources, self, monitor), distribute=self.distribute, name=self.__class__.__name__).reduce() Starmap.shutdown() # close the processpool Starmap.init() # reopen it when necessary # avoid task ordering issues for sources in sources_by_grp.values(): sources.sort(key=operator.attrgetter('source_id')) return sources_by_grp
def execute(self): """ Parallelize on the riskinputs and returns a dictionary of results. Require a `.core_task` to be defined with signature (riskinputs, riskmodel, rlzs_assoc, monitor). """ mon = self.monitor('risk') all_args = [(riskinput, self.riskmodel, self.param, mon) for riskinput in self.riskinputs] res = Starmap(self.core_task.__func__, all_args).reduce(self.combine) return res
def main(dirname): dname = pathlib.Path(dirname) with hdf5new() as hdf5: # create a new datastore monitor = Monitor('count', hdf5) # create a new monitor iterargs = ((open(dname/fname, encoding='utf-8').read(),) for fname in os.listdir(dname) if fname.endswith('.rst')) # read the docs c = collections.Counter() # intially empty counter for counter in Starmap(count, iterargs, monitor): c += counter print(c) # total counts print('Performance info stored in', hdf5)
def start_tasks(self, sm_id, ruptures_by_grp, sitecol, assetcol, riskmodel, imts, trunc_level, correl_model, min_iml, monitor): """ :param sm_id: source model ordinal :param ruptures_by_grp: dictionary of ruptures by src_group_id :param sitecol: a SiteCollection instance :param assetcol: an AssetCollection instance :param riskmodel: a RiskModel instance :param imts: a list of Intensity Measure Types :param trunc_level: truncation level :param correl_model: correlation model :param min_iml: vector of minimum intensities, one per IMT :param monitor: a Monitor instance :returns: an IterResult instance """ csm_info = self.csm_info.get_info(sm_id) grp_ids = sorted(csm_info.get_sm_by_grp()) rlzs_assoc = csm_info.get_rlzs_assoc( count_ruptures=lambda grp: len(ruptures_by_grp.get(grp.id, []))) num_events = sum(ebr.multiplicity for grp in ruptures_by_grp for ebr in ruptures_by_grp[grp]) seeds = self.oqparam.random_seed + numpy.arange(num_events) allargs = [] # prepare the risk inputs ruptures_per_block = self.oqparam.ruptures_per_block start = 0 ignore_covs = self.oqparam.ignore_covs for grp_id in grp_ids: rlzs_by_gsim = rlzs_assoc.get_rlzs_by_gsim(grp_id) samples = rlzs_assoc.samples[grp_id] for rupts in block_splitter(ruptures_by_grp.get(grp_id, []), ruptures_per_block): if ignore_covs or not self.riskmodel.covs: eps = None elif self.oqparam.asset_correlation: eps = EpsilonMatrix1(num_events, self.oqparam.master_seed) else: n_events = sum(ebr.multiplicity for ebr in rupts) eps = EpsilonMatrix0(len(self.assetcol), seeds[start:start + n_events]) start += n_events getter = riskinput.GmfGetter(grp_id, rlzs_by_gsim, rupts, sitecol, imts, min_iml, trunc_level, correl_model, samples) ri = riskinput.RiskInputFromRuptures(getter, eps) allargs.append((ri, riskmodel, assetcol, monitor)) self.vals = self.assetcol.values() taskname = '%s#%d' % (event_based_risk.__name__, sm_id + 1) ires = Starmap(event_based_risk, allargs, name=taskname).submit_all() ires.num_ruptures = { sg_id: len(rupts) for sg_id, rupts in ruptures_by_grp.items() } ires.num_events = num_events ires.num_rlzs = len(rlzs_assoc.realizations) ires.sm_id = sm_id return ires
def execute(self): # build loss maps if ('all_loss_ratios' in self.datastore and self.oqparam.conditional_loss_poes): assetcol = self.assetcol rlzs = self.rlzs_assoc.realizations stats = self.oqparam.risk_stats() builder = self.riskmodel.curve_builder A = len(assetcol) R = len(self.datastore['realizations']) # create loss_maps datasets self.datastore.create_dset('loss_maps-rlzs', builder.loss_maps_dt, (A, R), fillvalue=None) if R > 1: self.datastore.create_dset('loss_maps-stats', builder.loss_maps_dt, (A, len(stats)), fillvalue=None) mon = self.monitor('loss maps') if self.oqparam.hazard_calculation_id and ( 'asset_loss_table' in self.datastore.parent): Starmap = parallel.Starmap # we can parallelize fully lrgetter = riskinput.LossRatiosGetter(self.datastore.parent) # avoid OSError: Can't read data (Wrong b-tree signature) self.datastore.parent.close() else: # there is a single datastore # we cannot read from it in parallel while writing Starmap = parallel.Sequential lrgetter = riskinput.LossRatiosGetter(self.datastore) Starmap.apply(build_loss_maps, (assetcol, builder, lrgetter, rlzs, stats, mon), self.oqparam.concurrent_tasks).reduce( self.save_loss_maps) if self.oqparam.hazard_calculation_id: self.datastore.parent.open() # build an aggregate loss curve per realization if 'agg_loss_table' in self.datastore: self.build_agg_curve()
def hazard_matrix_calculation_parallel(self, quantity='poe'): """ Compute exhaustively the full VPSHA hazard matrix of ARE/POE over the N-dimensional space of spectral periods or parameters. NOTE: Parallelization occurs on the loop over N-D hazard matrix cells WARNING !! This computation can be extremely expensive for high-dimensional problems ! """ # Initialization step: hc_calc_method = getattr(self, quantity) # self.poe or self.are method args = list() shape = (len(self.sites),) + tuple(len(self.oqparam.imtls[str(p)]) for p in self.periods) max_nb = np.prod(shape) logging.warning('hazard matrix shape: [N_sites x N_IMT_1 x ... x N_IMT_k]: {}'.format(shape)) logging.warning('hazard matrix has {} elements'.format(max_nb)) acc_discretization = [np.log(self.oqparam.imtls[str(p)]) for p in self.periods] # create a N-dimensional mesh of spectral acceleration values: acc_meshes = np.meshgrid(*acc_discretization, indexing='ij', copy=False) nelts = int(np.prod(shape[1:])) # Number of N-D pseudo spectral values for i in range(nelts): indices = np.unravel_index(i, shape[1:]) # Flat to multi-dimensional index accels = [ acc_meshes[j][indices] for j in range(self.ndims)] logging.debug(f" # Current acceleration vector: {tuple(str(p) for p in self.periods)} = {accels}\n") # Call hazard curve computation method: #hazard_output = hc_calc_method(accels) args.append((indices, hc_calc_method, accels)) """ # Sort results for each site: for k in range(len(hazard_output)): # Loop on sites, i.e. 1st dimension of "hazard_output" # indx = np.ravel_multi_index((k,)+indices, shape) # indices = np.unravel_index(indx, shape) output[(k,) + indices] = hazard_output[k] """ output = np.empty(shape) for result in Starmap(_matrix_cell_worker, args): # Sort results for each site: for k in range(len(result['output'])): # Loop on sites, i.e. 1st dimension of "hazard_output" # indx = np.ravel_multi_index((k,)+indices, shape) # indices = np.unravel_index(indx, shape) output[(k,) + result['indices']] = result['output'][k] self.hc.hazard_matrix = output return self.hc
def execute(self): """ Parallelize on the riskinputs and returns a dictionary of results. Require a `.core_task` to be defined with signature (riskinputs, riskmodel, rlzs_assoc, monitor). """ if not hasattr(self, 'riskinputs'): # in the reportwriter return res = Starmap.apply( self.core_task.__func__, (self.riskinputs, self.riskmodel, self.param, self.monitor()), concurrent_tasks=self.oqparam.concurrent_tasks or 1, weight=get_weight).reduce(self.combine) return res
def run_calc(request): """ Run a calculation. :param request: a `django.http.HttpRequest` object. If the request has the attribute `hazard_job_id`, the results of the specified hazard calculations will be re-used as input by the risk calculation. The request also needs to contain the files needed to perform the calculation. They can be uploaded as separate files, or zipped together. """ hazard_job_id = request.POST.get('hazard_job_id') if hazard_job_id: hazard_job_id = int(hazard_job_id) candidates = ("job_risk.ini", "job.ini") else: candidates = ("job_hazard.ini", "job_haz.ini", "job.ini") einfo, exctype, monitor = safely_call(_prepare_job, (request, candidates)) if exctype: return HttpResponse(json.dumps(einfo.splitlines()), content_type=JSON, status=500) if not einfo: msg = 'Could not find any file of the form %s' % str(candidates) logging.error(msg) return HttpResponse(content=json.dumps([msg]), content_type=JSON, status=500) user = utils.get_user_data(request) try: job_id, fut = submit_job(einfo[0], user['name'], hazard_job_id) # restart the process pool at the end of each job fut.add_done_callback(lambda f: Starmap.restart()) except Exception as exc: # no job created, for instance missing .xml file # get the exception message exc_msg = str(exc) logging.error(exc_msg) response_data = exc_msg.splitlines() status = 500 else: response_data = dict(job_id=job_id, status='created') status = 200 return HttpResponse(content=json.dumps(response_data), content_type=JSON, status=status)
def execute(self): """ Parallelize on the riskinputs and returns a dictionary of results. Require a `.core_task` to be defined with signature (riskinputs, riskmodel, rlzs_assoc, monitor). """ if not hasattr(self, 'riskinputs'): # in the reportwriter return res = Starmap.apply( self.core_task.__func__, (self.riskinputs, self.riskmodel, self.param, self.monitor()), concurrent_tasks=self.oqparam.concurrent_tasks or 1, weight=get_weight ).reduce(self.combine) return res
def test(self): mon = Monitor() iterargs = ((i,) for i in range(10)) smap = Starmap(double, iterargs, mon, distribute='zmq') self.assertEqual(sum(res for res in smap), 90)
def tearDown(cls): Starmap.shutdown()
def tearDown(self): Starmap.shutdown()