def execute(self): oq = self.oqparam self.set_param() self.offset = 0 srcfilter = self.src_filter(self.datastore.tempname) self.indices = AccumDict(accum=[]) # sid, idx -> indices if oq.hazard_calculation_id: # from ruptures self.datastore.parent = util.read(oq.hazard_calculation_id) self.init_logic_tree(self.datastore.parent['full_lt']) else: # from sources self.build_events_from_sources(srcfilter) if (oq.ground_motion_fields is False and oq.hazard_curves_from_gmfs is False): return {} if not oq.imtls: raise InvalidFile('There are no intensity measure types in %s' % oq.inputs['job_ini']) N = len(self.sitecol.complete) if oq.ground_motion_fields: nrups = len(self.datastore['ruptures']) self.datastore.create_dset('gmf_data/data', oq.gmf_data_dt()) self.datastore.create_dset('gmf_data/sigma_epsilon', sig_eps_dt(oq.imtls)) self.datastore.create_dset( 'gmf_data/indices', hdf5.vuint32, shape=(N, 2), fillvalue=None) self.datastore.create_dset('gmf_data/events_by_sid', U32, (N,)) self.datastore.create_dset('gmf_data/time_by_rup', time_dt, (nrups,), fillvalue=None) if oq.hazard_curves_from_gmfs: self.param['rlz_by_event'] = self.datastore['events']['rlz_id'] # compute_gmfs in parallel self.datastore.swmr_on() logging.info('Reading %d ruptures', len(self.datastore['ruptures'])) iterargs = ((rgetter, srcfilter, self.param) for rgetter in gen_rupture_getters( self.datastore, srcfilter)) acc = parallel.Starmap( self.core_task.__func__, iterargs, h5=self.datastore.hdf5, num_cores=oq.num_cores ).reduce(self.agg_dicts, self.acc0()) if self.indices: dset = self.datastore['gmf_data/indices'] num_evs = self.datastore['gmf_data/events_by_sid'] logging.info('Saving gmf_data/indices') with self.monitor('saving gmf_data/indices', measuremem=True): self.datastore['gmf_data/imts'] = ' '.join(oq.imtls) for sid in self.sitecol.complete.sids: start = numpy.array(self.indices[sid, 0]) stop = numpy.array(self.indices[sid, 1]) dset[sid, 0] = start dset[sid, 1] = stop num_evs[sid] = (stop - start).sum() avg_events_by_sid = num_evs[()].sum() / N logging.info('Found ~%d GMVs per site', avg_events_by_sid) elif oq.ground_motion_fields: raise RuntimeError('No GMFs were generated, perhaps they were ' 'all below the minimum_intensity threshold') return acc
def execute(self): oq = self.oqparam self.set_param() self.offset = 0 if oq.hazard_calculation_id: # from ruptures self.datastore.parent = datastore.read(oq.hazard_calculation_id) elif hasattr(self, 'csm'): # from sources self.build_events_from_sources() if (oq.ground_motion_fields is False and oq.hazard_curves_from_gmfs is False): return {} elif 'rupture_model' not in oq.inputs: logging.warning( 'There is no rupture_model, the calculator will just ' 'import data without performing any calculation') fake = logictree.FullLogicTree.fake() self.datastore['full_lt'] = fake # needed to expose the outputs self.datastore['weights'] = [1.] return {} else: # scenario self._read_scenario_ruptures() if (oq.ground_motion_fields is False and oq.hazard_curves_from_gmfs is False): return {} if oq.ground_motion_fields: imts = oq.get_primary_imtls() nrups = len(self.datastore['ruptures']) base.create_gmf_data(self.datastore, imts, oq.get_sec_imts()) self.datastore.create_dset('gmf_data/sigma_epsilon', sig_eps_dt(oq.imtls)) self.datastore.create_dset('gmf_data/time_by_rup', time_dt, (nrups, ), fillvalue=None) # compute_gmfs in parallel nr = len(self.datastore['ruptures']) logging.info('Reading {:_d} ruptures'.format(nr)) allargs = [(rgetter, self.param) for rgetter in gen_rupture_getters( self.datastore, oq.concurrent_tasks)] # reading the args is fast since we are not prefiltering the ruptures, # nor reading the geometries; using an iterator would cause the usual # damned h5py error, last seen on macos self.datastore.swmr_on() smap = parallel.Starmap(self.core_task.__func__, allargs, h5=self.datastore.hdf5) smap.monitor.save('srcfilter', self.srcfilter) acc = smap.reduce(self.agg_dicts, self.acc0()) if 'gmf_data' not in self.datastore: return acc if oq.ground_motion_fields: with self.monitor('saving avg_gmf', measuremem=True): self.save_avg_gmf() return acc
def import_gmfs(dstore, fname, sids): """ Import in the datastore a ground motion field CSV file. :param dstore: the datastore :param fname: the CSV file :param sids: the site IDs (complete) :returns: event_ids, num_rlzs """ array = hdf5.read_csv(fname, {'sid': U32, 'eid': U64, None: F32}).array names = array.dtype.names if names[0] == 'rlzi': # backward compatbility names = names[1:] imts = [name[4:] for name in names[2:]] gmf_data_dt = dstore['oqparam'].gmf_data_dt() arr = numpy.zeros(len(array), gmf_data_dt) col = 0 for name in names: if name.startswith('gmv_'): arr['gmv'][:, col] = array[name] col += 1 else: arr[name] = array[name] # store the events eids = numpy.unique(array['eid']) eids.sort() E = len(eids) eid2idx = dict(zip(eids, range(E))) events = numpy.zeros(E, rupture.events_dt) events['id'] = eids dstore['events'] = events # store the GMFs dic = general.group_array(arr, 'sid') lst = [] offset = 0 for sid in sids: n = len(dic.get(sid, [])) lst.append((offset, offset + n)) if n: offset += n gmvs = dic[sid] gmvs['eid'] = get_idxs(gmvs, eid2idx) dstore.extend('gmf_data/data', gmvs) dstore['gmf_data/indices'] = numpy.array(lst, U32) dstore['gmf_data/imts'] = ' '.join(imts) sig_eps = numpy.zeros(len(eids), getters.sig_eps_dt(imts)) sig_eps['eid'] = eids dstore['gmf_data/sigma_epsilon'] = sig_eps dstore['weights'] = numpy.ones(1) return eids
def pre_execute(self): """ Read the site collection and initialize GmfComputer and seeds """ oq = self.oqparam full_lt = logictree.FullLogicTree.fake(readinput.get_gsim_lt(oq)) self.realizations = full_lt.get_realizations() self.datastore['full_lt'] = full_lt if 'rupture_model' not in oq.inputs: logging.warning( 'There is no rupture_model, the calculator will just ' 'import data without performing any calculation') super().pre_execute() return self.rup = readinput.get_rupture(oq) self.gsims = readinput.get_gsims(oq) R = len(self.gsims) self.cmaker = ContextMaker( '*', self.gsims, { 'maximum_distance': oq.maximum_distance, 'filter_distance': oq.filter_distance }) super().pre_execute() self.datastore['oqparam'] = oq self.store_rlz_info({}) rlzs_by_gsim = full_lt.get_rlzs_by_gsim(0) E = oq.number_of_ground_motion_fields n_occ = numpy.array([E]) ebr = EBRupture(self.rup, 0, 0, n_occ) ebr.e0 = 0 dtlist = events_dt.descr + [('ses_id', U16), ('year', U16)] events = numpy.zeros(E * R, dtlist) for rlz, eids in ebr.get_eids_by_rlz(rlzs_by_gsim).items(): events[rlz * E:rlz * E + E]['id'] = eids events[rlz * E:rlz * E + E]['rlz_id'] = rlz self.datastore['events'] = self.events = events rupser = calc.RuptureSerializer(self.datastore) rup_array = get_rup_array([ebr], self.src_filter()) if len(rup_array) == 0: maxdist = oq.maximum_distance(self.rup.tectonic_region_type, self.rup.mag) raise RuntimeError('There are no sites within the maximum_distance' ' of %s km from the rupture' % maxdist) rupser.save(rup_array) rupser.close() self.computer = GmfComputer(ebr, self.sitecol, oq.imtls, self.cmaker, oq.truncation_level, oq.correl_model, self.amplifier) self.sig_eps_dt = getters.sig_eps_dt(self.oqparam.imtls)
def import_gmfs(dstore, fname, sids): """ Import in the datastore a ground motion field CSV file. :param dstore: the datastore :param fname: the CSV file :param sids: the site IDs (complete) :returns: event_ids, num_rlzs """ array = hdf5.read_csv(fname, {'sid': U32, 'eid': U64, None: F32}).array imts = [name[4:] for name in array.dtype.names[2:]] n_imts = len(imts) gmf_data_dt = numpy.dtype( [('rlzi', U16), ('sid', U32), ('eid', U64), ('gmv', (F32, (n_imts,)))]) arr = numpy.zeros(len(array), gmf_data_dt) col = 0 for name in array.dtype.names: if name.startswith('gmv_'): arr['gmv'][:, col] = array[name] col += 1 else: arr[name] = array[name] # store the events eids = numpy.unique(array['eid']) eids.sort() E = len(eids) eid2idx = dict(zip(eids, range(E))) events = numpy.zeros(E, rupture.events_dt) events['id'] = eids dstore['events'] = events # store the GMFs dic = general.group_array(arr, 'sid') lst = [] offset = 0 for sid in sids: n = len(dic.get(sid, [])) lst.append((offset, offset + n)) if n: offset += n gmvs = dic[sid] gmvs['eid'] = get_idxs(gmvs, eid2idx) dstore.extend('gmf_data/data', gmvs) dstore['gmf_data/indices'] = numpy.array(lst, U32) dstore['gmf_data/imts'] = ' '.join(imts) sig_eps = numpy.zeros(len(eids), getters.sig_eps_dt(imts)) sig_eps['eid'] = eids dstore['gmf_data/sigma_epsilon'] = sig_eps dstore['weights'] = numpy.ones(1) return eids
def execute(self): oq = self.oqparam self.set_param() self.offset = 0 if oq.hazard_calculation_id: # from ruptures self.datastore.parent = util.read(oq.hazard_calculation_id) elif hasattr(self, 'csm'): # from sources self.build_events_from_sources() if (oq.ground_motion_fields is False and oq.hazard_curves_from_gmfs is False): return {} elif 'rupture_model' not in oq.inputs: # download ShakeMap logging.warning( 'There is no rupture_model, the calculator will just ' 'import data without performing any calculation') fake = logictree.FullLogicTree.fake() self.datastore['full_lt'] = fake # needed to expose the outputs return {} else: # scenario self._read_scenario_ruptures() if (oq.ground_motion_fields is False and oq.hazard_curves_from_gmfs is False): return {} if not oq.imtls: raise InvalidFile('There are no intensity measure types in %s' % oq.inputs['job_ini']) N = len(self.sitecol.complete) if oq.ground_motion_fields: M = len(oq.imtls) nrups = len(self.datastore['ruptures']) base.create_gmf_data(self.datastore, M, self.param['sec_perils']) self.datastore.create_dset('gmf_data/sigma_epsilon', sig_eps_dt(oq.imtls)) self.datastore.create_dset('gmf_data/events_by_sid', U32, (N, )) self.datastore.create_dset('gmf_data/time_by_rup', time_dt, (nrups, ), fillvalue=None) # compute_gmfs in parallel nr = len(self.datastore['ruptures']) self.datastore.swmr_on() logging.info('Reading {:_d} ruptures'.format(nr)) iterargs = ((rgetter, self.param) for rgetter in gen_rupture_getters( self.datastore, oq.concurrent_tasks)) smap = parallel.Starmap(self.core_task.__func__, iterargs, h5=self.datastore.hdf5) smap.monitor.save('srcfilter', self.srcfilter) acc = smap.reduce(self.agg_dicts, self.acc0()) if 'gmf_data' not in self.datastore: return acc if oq.ground_motion_fields: eids = self.datastore['gmf_data/eid'][:] rel_events = numpy.unique(eids) e = len(rel_events) if e == 0: raise RuntimeError('No GMFs were generated, perhaps they were ' 'all below the minimum_intensity threshold') elif e < len(self.datastore['events']): self.datastore['relevant_events'] = rel_events logging.info('Stored %d relevant event IDs', e) return acc
def event_based(proxies, full_lt, oqparam, dstore, monitor): """ Compute GMFs and optionally hazard curves """ alldata = AccumDict(accum=[]) sig_eps = [] times = [] # rup_id, nsites, dt hcurves = {} # key -> poes trt_smr = proxies[0]['trt_smr'] fmon = monitor('filtering ruptures', measuremem=False) cmon = monitor('computing gmfs', measuremem=False) with dstore: trt = full_lt.trts[trt_smr // len(full_lt.sm_rlzs)] srcfilter = SourceFilter(dstore['sitecol'], oqparam.maximum_distance(trt)) rupgeoms = dstore['rupgeoms'] rlzs_by_gsim = full_lt._rlzs_by_gsim(trt_smr) param = vars(oqparam).copy() param['imtls'] = oqparam.imtls param['min_iml'] = oqparam.min_iml param['maximum_distance'] = oqparam.maximum_distance(trt) cmaker = ContextMaker(trt, rlzs_by_gsim, param) min_mag = getdefault(oqparam.minimum_magnitude, trt) for proxy in proxies: t0 = time.time() with fmon: if proxy['mag'] < min_mag: continue sids = srcfilter.close_sids(proxy, trt) if len(sids) == 0: # filtered away continue proxy.geom = rupgeoms[proxy['geom_id']] ebr = proxy.to_ebr(cmaker.trt) # after the geometry is set try: computer = GmfComputer(ebr, srcfilter.sitecol.filtered(sids), cmaker, oqparam.correl_model, oqparam.cross_correl, oqparam._amplifier, oqparam._sec_perils) except FarAwayRupture: continue with cmon: data = computer.compute_all(sig_eps) dt = time.time() - t0 times.append((computer.ebrupture.id, len(computer.ctx.sids), dt)) for key in data: alldata[key].extend(data[key]) for key, val in sorted(alldata.items()): if key in 'eid sid rlz': alldata[key] = U32(alldata[key]) else: alldata[key] = F32(alldata[key]) gmfdata = strip_zeros(pandas.DataFrame(alldata)) if len(gmfdata) and oqparam.hazard_curves_from_gmfs: hc_mon = monitor('building hazard curves', measuremem=False) for (sid, rlz), df in gmfdata.groupby(['sid', 'rlz']): with hc_mon: poes = calc.gmvs_to_poes(df, oqparam.imtls, oqparam.ses_per_logic_tree_path) for m, imt in enumerate(oqparam.imtls): hcurves[rsi2str(rlz, sid, imt)] = poes[m] times = numpy.array([tup + (monitor.task_no, ) for tup in times], time_dt) times.sort(order='rup_id') if not oqparam.ground_motion_fields: gmfdata = () return dict(gmfdata=gmfdata, hcurves=hcurves, times=times, sig_eps=numpy.array(sig_eps, sig_eps_dt(oqparam.imtls)))
def execute(self): oq = self.oqparam dstore = self.datastore if oq.ground_motion_fields and oq.min_iml.sum() == 0: logging.warning('The GMFs are not filtered: ' 'you may want to set a minimum_intensity') else: logging.info('minimum_intensity=%s', oq.minimum_intensity) self.offset = 0 if oq.hazard_calculation_id: # from ruptures dstore.parent = datastore.read(oq.hazard_calculation_id) elif hasattr(self, 'csm'): # from sources self.build_events_from_sources() if (oq.ground_motion_fields is False and oq.hazard_curves_from_gmfs is False): return {} elif 'rupture_model' not in oq.inputs: logging.warning( 'There is no rupture_model, the calculator will just ' 'import data without performing any calculation') fake = logictree.FullLogicTree.fake() dstore['full_lt'] = fake # needed to expose the outputs dstore['weights'] = [1.] return {} else: # scenario self._read_scenario_ruptures() if (oq.ground_motion_fields is False and oq.hazard_curves_from_gmfs is False): return {} if oq.ground_motion_fields: imts = oq.get_primary_imtls() nrups = len(dstore['ruptures']) base.create_gmf_data(dstore, imts, oq.get_sec_imts()) dstore.create_dset('gmf_data/sigma_epsilon', sig_eps_dt(oq.imtls)) dstore.create_dset('gmf_data/time_by_rup', time_dt, (nrups, ), fillvalue=None) # event_based in parallel nr = len(dstore['ruptures']) logging.info('Reading {:_d} ruptures'.format(nr)) scenario = 'scenario' in oq.calculation_mode proxies = [ RuptureProxy(rec, scenario) for rec in dstore['ruptures'][:] ] full_lt = self.datastore['full_lt'] dstore.swmr_on() # must come before the Starmap smap = parallel.Starmap.apply_split( self.core_task.__func__, (proxies, full_lt, oq, self.datastore), key=operator.itemgetter('trt_smr'), weight=operator.itemgetter('n_occ'), h5=dstore.hdf5, concurrent_tasks=oq.concurrent_tasks or 1, duration=oq.time_per_task, split_level=oq.split_level) acc = smap.reduce(self.agg_dicts, self.acc0()) if 'gmf_data' not in dstore: return acc if oq.ground_motion_fields: with self.monitor('saving avg_gmf', measuremem=True): self.save_avg_gmf() return acc
def execute(self): oq = self.oqparam self.set_param() self.offset = 0 if oq.hazard_calculation_id: # from ruptures self.datastore.parent = util.read(oq.hazard_calculation_id) elif hasattr(self, 'csm'): # from sources self.build_events_from_sources() if (oq.ground_motion_fields is False and oq.hazard_curves_from_gmfs is False): return {} elif 'rupture_model' not in oq.inputs: # download ShakeMap logging.warning( 'There is no rupture_model, the calculator will just ' 'import data without performing any calculation') fake = logictree.FullLogicTree.fake() self.datastore['full_lt'] = fake # needed to expose the outputs return {} else: # scenario self._read_scenario_ruptures() if (oq.ground_motion_fields is False and oq.hazard_curves_from_gmfs is False): return {} N = len(self.sitecol.complete) if oq.ground_motion_fields: M = len(oq.get_primary_imtls()) nrups = len(self.datastore['ruptures']) base.create_gmf_data(self.datastore, M, oq.get_sec_imts()) self.datastore.create_dset('gmf_data/sigma_epsilon', sig_eps_dt(oq.imtls)) self.datastore.create_dset('gmf_data/events_by_sid', U32, (N, )) self.datastore.create_dset('gmf_data/time_by_rup', time_dt, (nrups, ), fillvalue=None) # compute_gmfs in parallel nr = len(self.datastore['ruptures']) logging.info('Reading {:_d} ruptures'.format(nr)) allargs = [(rgetter, self.param) for rgetter in gen_rupture_getters( self.datastore, oq.concurrent_tasks)] # reading the args is fast since we are not prefiltering the ruptures, # nor reading the geometries; using an iterator would cause the usual # damned h5py error, last seen on macos self.datastore.swmr_on() smap = parallel.Starmap(self.core_task.__func__, allargs, h5=self.datastore.hdf5) smap.monitor.save('srcfilter', self.srcfilter) acc = smap.reduce(self.agg_dicts, self.acc0()) if 'gmf_data' not in self.datastore: return acc if oq.ground_motion_fields: with self.monitor('saving avg_gmf', measuremem=True): self.weights = self.datastore['weights'][:] self.rlzs = self.datastore['events']['rlz_id'] self.num_events = numpy.bincount(self.rlzs) # events by rlz avg_gmf = { imt: numpy.zeros(self.N, F32) for imt in oq.all_imts() } rel_events = self.save_avg_gmf(avg_gmf) self.datastore.create_dframe('avg_gmf', avg_gmf.items()) e = len(rel_events) if e == 0: raise RuntimeError('No GMFs were generated, perhaps they were ' 'all below the minimum_intensity threshold') elif e < len(self.datastore['events']): self.datastore['relevant_events'] = rel_events logging.info('Stored %d relevant event IDs', e) return acc
def run_calc(self): """ Run a calculation and return results (reinvented from openquake.calculators.base) """ with self.calculator._monitor: self.calculator._monitor.username = '' try: # Pre-execute setups self.calculator.pre_execute() #self.calculator.datastore.swmr_on() oq = self.calculator.oqparam dstore = self.calculator.datastore self.calculator.set_param() self.calculator.offset = 0 # Source model print('self.__dict__ = ') print(self.calculator.__dict__) if oq.hazard_calculation_id: # from ruptures dstore.parent = self.calculator.datastore.read( oq.hazard_calculation_id) elif hasattr(self.calculator, 'csm'): # from sources self.calculator_build_events_from_sources() #self.calculator.build_events_from_sources() if (oq.ground_motion_fields is False and oq.hazard_curves_from_gmfs is False): return {} elif 'rupture_model' not in oq.inputs: logging.warning( 'There is no rupture_model, the calculator will just ' 'import data without performing any calculation') fake = logictree.FullLogicTree.fake() dstore['full_lt'] = fake # needed to expose the outputs dstore['weights'] = [1.] return {} else: # scenario self.calculator._read_scenario_ruptures() if (oq.ground_motion_fields is False and oq.hazard_curves_from_gmfs is False): return {} # Intensity measure models if oq.ground_motion_fields: imts = oq.get_primary_imtls() nrups = len(dstore['ruptures']) base.create_gmf_data(dstore, imts, oq.get_sec_imts()) dstore.create_dset('gmf_data/sigma_epsilon', getters.sig_eps_dt(oq.imtls)) dstore.create_dset('gmf_data/time_by_rup', getters.time_dt, (nrups, ), fillvalue=None) # Prepare inputs for GmfGetter nr = len(dstore['ruptures']) logging.info('Reading {:_d} ruptures'.format(nr)) rgetters = getters.get_rupture_getters( dstore, oq.concurrent_tasks * 1.25, srcfilter=self.calculator.srcfilter) args = [(rgetter, self.calculator.param) for rgetter in rgetters] mon = performance.Monitor() mon.version = version mon.config = config rcvr = 'tcp://%s:%s' % (config.dbserver.listen, config.dbserver.receiver_ports) skt = zeromq.Socket(rcvr, zeromq.zmq.PULL, 'bind').__enter__() mon.backurl = 'tcp://%s:%s' % (config.dbserver.host, skt.port) mon = mon.new(operation='total ' + self.calculator.core_task.__func__.__name__, measuremem=True) mon.weight = getattr(args[0], 'weight', 1.) # used in task_info mon.task_no = 1 # initialize the task number args += (mon, ) self.args = args self.mon = mon self.dstore = dstore finally: print('FetchOpenQuake: OpenQuake Hazard Calculator defined.')