def execute(self):
oq = self.oqparam
self.set_param()
self.offset = 0
srcfilter = self.src_filter(self.datastore.tempname)
self.indices = AccumDict(accum=[]) # sid, idx -> indices
if oq.hazard_calculation_id: # from ruptures
self.datastore.parent = util.read(oq.hazard_calculation_id)
self.init_logic_tree(self.datastore.parent['full_lt'])
else: # from sources
self.build_events_from_sources(srcfilter)
if (oq.ground_motion_fields is False and
oq.hazard_curves_from_gmfs is False):
return {}
if not oq.imtls:
raise InvalidFile('There are no intensity measure types in %s' %
oq.inputs['job_ini'])
N = len(self.sitecol.complete)
if oq.ground_motion_fields:
nrups = len(self.datastore['ruptures'])
self.datastore.create_dset('gmf_data/data', oq.gmf_data_dt())
self.datastore.create_dset('gmf_data/sigma_epsilon',
sig_eps_dt(oq.imtls))
self.datastore.create_dset(
'gmf_data/indices', hdf5.vuint32, shape=(N, 2), fillvalue=None)
self.datastore.create_dset('gmf_data/events_by_sid', U32, (N,))
self.datastore.create_dset('gmf_data/time_by_rup',
time_dt, (nrups,), fillvalue=None)
if oq.hazard_curves_from_gmfs:
self.param['rlz_by_event'] = self.datastore['events']['rlz_id']
# compute_gmfs in parallel
self.datastore.swmr_on()
logging.info('Reading %d ruptures', len(self.datastore['ruptures']))
iterargs = ((rgetter, srcfilter, self.param)
for rgetter in gen_rupture_getters(
self.datastore, srcfilter))
acc = parallel.Starmap(
self.core_task.__func__, iterargs, h5=self.datastore.hdf5,
num_cores=oq.num_cores
).reduce(self.agg_dicts, self.acc0())
if self.indices:
dset = self.datastore['gmf_data/indices']
num_evs = self.datastore['gmf_data/events_by_sid']
logging.info('Saving gmf_data/indices')
with self.monitor('saving gmf_data/indices', measuremem=True):
self.datastore['gmf_data/imts'] = ' '.join(oq.imtls)
for sid in self.sitecol.complete.sids:
start = numpy.array(self.indices[sid, 0])
stop = numpy.array(self.indices[sid, 1])
dset[sid, 0] = start
dset[sid, 1] = stop
num_evs[sid] = (stop - start).sum()
avg_events_by_sid = num_evs[()].sum() / N
logging.info('Found ~%d GMVs per site', avg_events_by_sid)
elif oq.ground_motion_fields:
raise RuntimeError('No GMFs were generated, perhaps they were '
'all below the minimum_intensity threshold')
return acc
def execute(self):
oq = self.oqparam
self.set_param()
self.offset = 0
if oq.hazard_calculation_id: # from ruptures
self.datastore.parent = datastore.read(oq.hazard_calculation_id)
elif hasattr(self, 'csm'): # from sources
self.build_events_from_sources()
if (oq.ground_motion_fields is False
and oq.hazard_curves_from_gmfs is False):
return {}
elif 'rupture_model' not in oq.inputs:
logging.warning(
'There is no rupture_model, the calculator will just '
'import data without performing any calculation')
fake = logictree.FullLogicTree.fake()
self.datastore['full_lt'] = fake # needed to expose the outputs
self.datastore['weights'] = [1.]
return {}
else: # scenario
self._read_scenario_ruptures()
if (oq.ground_motion_fields is False
and oq.hazard_curves_from_gmfs is False):
return {}
if oq.ground_motion_fields:
imts = oq.get_primary_imtls()
nrups = len(self.datastore['ruptures'])
base.create_gmf_data(self.datastore, imts, oq.get_sec_imts())
self.datastore.create_dset('gmf_data/sigma_epsilon',
sig_eps_dt(oq.imtls))
self.datastore.create_dset('gmf_data/time_by_rup',
time_dt, (nrups, ),
fillvalue=None)
# compute_gmfs in parallel
nr = len(self.datastore['ruptures'])
logging.info('Reading {:_d} ruptures'.format(nr))
allargs = [(rgetter, self.param) for rgetter in gen_rupture_getters(
self.datastore, oq.concurrent_tasks)]
# reading the args is fast since we are not prefiltering the ruptures,
# nor reading the geometries; using an iterator would cause the usual
# damned h5py error, last seen on macos
self.datastore.swmr_on()
smap = parallel.Starmap(self.core_task.__func__,
allargs,
h5=self.datastore.hdf5)
smap.monitor.save('srcfilter', self.srcfilter)
acc = smap.reduce(self.agg_dicts, self.acc0())
if 'gmf_data' not in self.datastore:
return acc
if oq.ground_motion_fields:
with self.monitor('saving avg_gmf', measuremem=True):
self.save_avg_gmf()
return acc
def import_gmfs(dstore, fname, sids):
"""
Import in the datastore a ground motion field CSV file.
:param dstore: the datastore
:param fname: the CSV file
:param sids: the site IDs (complete)
:returns: event_ids, num_rlzs
"""
array = hdf5.read_csv(fname, {'sid': U32, 'eid': U64, None: F32}).array
names = array.dtype.names
if names[0] == 'rlzi': # backward compatbility
names = names[1:]
imts = [name[4:] for name in names[2:]]
gmf_data_dt = dstore['oqparam'].gmf_data_dt()
arr = numpy.zeros(len(array), gmf_data_dt)
col = 0
for name in names:
if name.startswith('gmv_'):
arr['gmv'][:, col] = array[name]
col += 1
else:
arr[name] = array[name]
# store the events
eids = numpy.unique(array['eid'])
eids.sort()
E = len(eids)
eid2idx = dict(zip(eids, range(E)))
events = numpy.zeros(E, rupture.events_dt)
events['id'] = eids
dstore['events'] = events
# store the GMFs
dic = general.group_array(arr, 'sid')
lst = []
offset = 0
for sid in sids:
n = len(dic.get(sid, []))
lst.append((offset, offset + n))
if n:
offset += n
gmvs = dic[sid]
gmvs['eid'] = get_idxs(gmvs, eid2idx)
dstore.extend('gmf_data/data', gmvs)
dstore['gmf_data/indices'] = numpy.array(lst, U32)
dstore['gmf_data/imts'] = ' '.join(imts)
sig_eps = numpy.zeros(len(eids), getters.sig_eps_dt(imts))
sig_eps['eid'] = eids
dstore['gmf_data/sigma_epsilon'] = sig_eps
dstore['weights'] = numpy.ones(1)
return eids
def pre_execute(self):
"""
Read the site collection and initialize GmfComputer and seeds
"""
oq = self.oqparam
full_lt = logictree.FullLogicTree.fake(readinput.get_gsim_lt(oq))
self.realizations = full_lt.get_realizations()
self.datastore['full_lt'] = full_lt
if 'rupture_model' not in oq.inputs:
logging.warning(
'There is no rupture_model, the calculator will just '
'import data without performing any calculation')
super().pre_execute()
return
self.rup = readinput.get_rupture(oq)
self.gsims = readinput.get_gsims(oq)
R = len(self.gsims)
self.cmaker = ContextMaker(
'*', self.gsims, {
'maximum_distance': oq.maximum_distance,
'filter_distance': oq.filter_distance
})
super().pre_execute()
self.datastore['oqparam'] = oq
self.store_rlz_info({})
rlzs_by_gsim = full_lt.get_rlzs_by_gsim(0)
E = oq.number_of_ground_motion_fields
n_occ = numpy.array([E])
ebr = EBRupture(self.rup, 0, 0, n_occ)
ebr.e0 = 0
dtlist = events_dt.descr + [('ses_id', U16), ('year', U16)]
events = numpy.zeros(E * R, dtlist)
for rlz, eids in ebr.get_eids_by_rlz(rlzs_by_gsim).items():
events[rlz * E:rlz * E + E]['id'] = eids
events[rlz * E:rlz * E + E]['rlz_id'] = rlz
self.datastore['events'] = self.events = events
rupser = calc.RuptureSerializer(self.datastore)
rup_array = get_rup_array([ebr], self.src_filter())
if len(rup_array) == 0:
maxdist = oq.maximum_distance(self.rup.tectonic_region_type,
self.rup.mag)
raise RuntimeError('There are no sites within the maximum_distance'
' of %s km from the rupture' % maxdist)
rupser.save(rup_array)
rupser.close()
self.computer = GmfComputer(ebr, self.sitecol, oq.imtls, self.cmaker,
oq.truncation_level, oq.correl_model,
self.amplifier)
self.sig_eps_dt = getters.sig_eps_dt(self.oqparam.imtls)
def import_gmfs(dstore, fname, sids):
"""
Import in the datastore a ground motion field CSV file.
:param dstore: the datastore
:param fname: the CSV file
:param sids: the site IDs (complete)
:returns: event_ids, num_rlzs
"""
array = hdf5.read_csv(fname, {'sid': U32, 'eid': U64, None: F32}).array
imts = [name[4:] for name in array.dtype.names[2:]]
n_imts = len(imts)
gmf_data_dt = numpy.dtype(
[('rlzi', U16), ('sid', U32), ('eid', U64), ('gmv', (F32, (n_imts,)))])
arr = numpy.zeros(len(array), gmf_data_dt)
col = 0
for name in array.dtype.names:
if name.startswith('gmv_'):
arr['gmv'][:, col] = array[name]
col += 1
else:
arr[name] = array[name]
# store the events
eids = numpy.unique(array['eid'])
eids.sort()
E = len(eids)
eid2idx = dict(zip(eids, range(E)))
events = numpy.zeros(E, rupture.events_dt)
events['id'] = eids
dstore['events'] = events
# store the GMFs
dic = general.group_array(arr, 'sid')
lst = []
offset = 0
for sid in sids:
n = len(dic.get(sid, []))
lst.append((offset, offset + n))
if n:
offset += n
gmvs = dic[sid]
gmvs['eid'] = get_idxs(gmvs, eid2idx)
dstore.extend('gmf_data/data', gmvs)
dstore['gmf_data/indices'] = numpy.array(lst, U32)
dstore['gmf_data/imts'] = ' '.join(imts)
sig_eps = numpy.zeros(len(eids), getters.sig_eps_dt(imts))
sig_eps['eid'] = eids
dstore['gmf_data/sigma_epsilon'] = sig_eps
dstore['weights'] = numpy.ones(1)
return eids
def execute(self):
oq = self.oqparam
self.set_param()
self.offset = 0
if oq.hazard_calculation_id: # from ruptures
self.datastore.parent = util.read(oq.hazard_calculation_id)
elif hasattr(self, 'csm'): # from sources
self.build_events_from_sources()
if (oq.ground_motion_fields is False
and oq.hazard_curves_from_gmfs is False):
return {}
elif 'rupture_model' not in oq.inputs: # download ShakeMap
logging.warning(
'There is no rupture_model, the calculator will just '
'import data without performing any calculation')
fake = logictree.FullLogicTree.fake()
self.datastore['full_lt'] = fake # needed to expose the outputs
return {}
else: # scenario
self._read_scenario_ruptures()
if (oq.ground_motion_fields is False
and oq.hazard_curves_from_gmfs is False):
return {}
if not oq.imtls:
raise InvalidFile('There are no intensity measure types in %s' %
oq.inputs['job_ini'])
N = len(self.sitecol.complete)
if oq.ground_motion_fields:
M = len(oq.imtls)
nrups = len(self.datastore['ruptures'])
base.create_gmf_data(self.datastore, M, self.param['sec_perils'])
self.datastore.create_dset('gmf_data/sigma_epsilon',
sig_eps_dt(oq.imtls))
self.datastore.create_dset('gmf_data/events_by_sid', U32, (N, ))
self.datastore.create_dset('gmf_data/time_by_rup',
time_dt, (nrups, ),
fillvalue=None)
# compute_gmfs in parallel
nr = len(self.datastore['ruptures'])
self.datastore.swmr_on()
logging.info('Reading {:_d} ruptures'.format(nr))
iterargs = ((rgetter, self.param) for rgetter in gen_rupture_getters(
self.datastore, oq.concurrent_tasks))
smap = parallel.Starmap(self.core_task.__func__,
iterargs,
h5=self.datastore.hdf5)
smap.monitor.save('srcfilter', self.srcfilter)
acc = smap.reduce(self.agg_dicts, self.acc0())
if 'gmf_data' not in self.datastore:
return acc
if oq.ground_motion_fields:
eids = self.datastore['gmf_data/eid'][:]
rel_events = numpy.unique(eids)
e = len(rel_events)
if e == 0:
raise RuntimeError('No GMFs were generated, perhaps they were '
'all below the minimum_intensity threshold')
elif e < len(self.datastore['events']):
self.datastore['relevant_events'] = rel_events
logging.info('Stored %d relevant event IDs', e)
return acc
def event_based(proxies, full_lt, oqparam, dstore, monitor):
"""
Compute GMFs and optionally hazard curves
"""
alldata = AccumDict(accum=[])
sig_eps = []
times = [] # rup_id, nsites, dt
hcurves = {} # key -> poes
trt_smr = proxies[0]['trt_smr']
fmon = monitor('filtering ruptures', measuremem=False)
cmon = monitor('computing gmfs', measuremem=False)
with dstore:
trt = full_lt.trts[trt_smr // len(full_lt.sm_rlzs)]
srcfilter = SourceFilter(dstore['sitecol'],
oqparam.maximum_distance(trt))
rupgeoms = dstore['rupgeoms']
rlzs_by_gsim = full_lt._rlzs_by_gsim(trt_smr)
param = vars(oqparam).copy()
param['imtls'] = oqparam.imtls
param['min_iml'] = oqparam.min_iml
param['maximum_distance'] = oqparam.maximum_distance(trt)
cmaker = ContextMaker(trt, rlzs_by_gsim, param)
min_mag = getdefault(oqparam.minimum_magnitude, trt)
for proxy in proxies:
t0 = time.time()
with fmon:
if proxy['mag'] < min_mag:
continue
sids = srcfilter.close_sids(proxy, trt)
if len(sids) == 0: # filtered away
continue
proxy.geom = rupgeoms[proxy['geom_id']]
ebr = proxy.to_ebr(cmaker.trt) # after the geometry is set
try:
computer = GmfComputer(ebr,
srcfilter.sitecol.filtered(sids),
cmaker, oqparam.correl_model,
oqparam.cross_correl,
oqparam._amplifier,
oqparam._sec_perils)
except FarAwayRupture:
continue
with cmon:
data = computer.compute_all(sig_eps)
dt = time.time() - t0
times.append((computer.ebrupture.id, len(computer.ctx.sids), dt))
for key in data:
alldata[key].extend(data[key])
for key, val in sorted(alldata.items()):
if key in 'eid sid rlz':
alldata[key] = U32(alldata[key])
else:
alldata[key] = F32(alldata[key])
gmfdata = strip_zeros(pandas.DataFrame(alldata))
if len(gmfdata) and oqparam.hazard_curves_from_gmfs:
hc_mon = monitor('building hazard curves', measuremem=False)
for (sid, rlz), df in gmfdata.groupby(['sid', 'rlz']):
with hc_mon:
poes = calc.gmvs_to_poes(df, oqparam.imtls,
oqparam.ses_per_logic_tree_path)
for m, imt in enumerate(oqparam.imtls):
hcurves[rsi2str(rlz, sid, imt)] = poes[m]
times = numpy.array([tup + (monitor.task_no, ) for tup in times], time_dt)
times.sort(order='rup_id')
if not oqparam.ground_motion_fields:
gmfdata = ()
return dict(gmfdata=gmfdata,
hcurves=hcurves,
times=times,
sig_eps=numpy.array(sig_eps, sig_eps_dt(oqparam.imtls)))
def execute(self):
oq = self.oqparam
dstore = self.datastore
if oq.ground_motion_fields and oq.min_iml.sum() == 0:
logging.warning('The GMFs are not filtered: '
'you may want to set a minimum_intensity')
else:
logging.info('minimum_intensity=%s', oq.minimum_intensity)
self.offset = 0
if oq.hazard_calculation_id: # from ruptures
dstore.parent = datastore.read(oq.hazard_calculation_id)
elif hasattr(self, 'csm'): # from sources
self.build_events_from_sources()
if (oq.ground_motion_fields is False
and oq.hazard_curves_from_gmfs is False):
return {}
elif 'rupture_model' not in oq.inputs:
logging.warning(
'There is no rupture_model, the calculator will just '
'import data without performing any calculation')
fake = logictree.FullLogicTree.fake()
dstore['full_lt'] = fake # needed to expose the outputs
dstore['weights'] = [1.]
return {}
else: # scenario
self._read_scenario_ruptures()
if (oq.ground_motion_fields is False
and oq.hazard_curves_from_gmfs is False):
return {}
if oq.ground_motion_fields:
imts = oq.get_primary_imtls()
nrups = len(dstore['ruptures'])
base.create_gmf_data(dstore, imts, oq.get_sec_imts())
dstore.create_dset('gmf_data/sigma_epsilon', sig_eps_dt(oq.imtls))
dstore.create_dset('gmf_data/time_by_rup',
time_dt, (nrups, ),
fillvalue=None)
# event_based in parallel
nr = len(dstore['ruptures'])
logging.info('Reading {:_d} ruptures'.format(nr))
scenario = 'scenario' in oq.calculation_mode
proxies = [
RuptureProxy(rec, scenario) for rec in dstore['ruptures'][:]
]
full_lt = self.datastore['full_lt']
dstore.swmr_on() # must come before the Starmap
smap = parallel.Starmap.apply_split(
self.core_task.__func__, (proxies, full_lt, oq, self.datastore),
key=operator.itemgetter('trt_smr'),
weight=operator.itemgetter('n_occ'),
h5=dstore.hdf5,
concurrent_tasks=oq.concurrent_tasks or 1,
duration=oq.time_per_task,
split_level=oq.split_level)
acc = smap.reduce(self.agg_dicts, self.acc0())
if 'gmf_data' not in dstore:
return acc
if oq.ground_motion_fields:
with self.monitor('saving avg_gmf', measuremem=True):
self.save_avg_gmf()
return acc
def execute(self):
oq = self.oqparam
self.set_param()
self.offset = 0
if oq.hazard_calculation_id: # from ruptures
self.datastore.parent = util.read(oq.hazard_calculation_id)
elif hasattr(self, 'csm'): # from sources
self.build_events_from_sources()
if (oq.ground_motion_fields is False
and oq.hazard_curves_from_gmfs is False):
return {}
elif 'rupture_model' not in oq.inputs: # download ShakeMap
logging.warning(
'There is no rupture_model, the calculator will just '
'import data without performing any calculation')
fake = logictree.FullLogicTree.fake()
self.datastore['full_lt'] = fake # needed to expose the outputs
return {}
else: # scenario
self._read_scenario_ruptures()
if (oq.ground_motion_fields is False
and oq.hazard_curves_from_gmfs is False):
return {}
N = len(self.sitecol.complete)
if oq.ground_motion_fields:
M = len(oq.get_primary_imtls())
nrups = len(self.datastore['ruptures'])
base.create_gmf_data(self.datastore, M, oq.get_sec_imts())
self.datastore.create_dset('gmf_data/sigma_epsilon',
sig_eps_dt(oq.imtls))
self.datastore.create_dset('gmf_data/events_by_sid', U32, (N, ))
self.datastore.create_dset('gmf_data/time_by_rup',
time_dt, (nrups, ),
fillvalue=None)
# compute_gmfs in parallel
nr = len(self.datastore['ruptures'])
logging.info('Reading {:_d} ruptures'.format(nr))
allargs = [(rgetter, self.param) for rgetter in gen_rupture_getters(
self.datastore, oq.concurrent_tasks)]
# reading the args is fast since we are not prefiltering the ruptures,
# nor reading the geometries; using an iterator would cause the usual
# damned h5py error, last seen on macos
self.datastore.swmr_on()
smap = parallel.Starmap(self.core_task.__func__,
allargs,
h5=self.datastore.hdf5)
smap.monitor.save('srcfilter', self.srcfilter)
acc = smap.reduce(self.agg_dicts, self.acc0())
if 'gmf_data' not in self.datastore:
return acc
if oq.ground_motion_fields:
with self.monitor('saving avg_gmf', measuremem=True):
self.weights = self.datastore['weights'][:]
self.rlzs = self.datastore['events']['rlz_id']
self.num_events = numpy.bincount(self.rlzs) # events by rlz
avg_gmf = {
imt: numpy.zeros(self.N, F32)
for imt in oq.all_imts()
}
rel_events = self.save_avg_gmf(avg_gmf)
self.datastore.create_dframe('avg_gmf', avg_gmf.items())
e = len(rel_events)
if e == 0:
raise RuntimeError('No GMFs were generated, perhaps they were '
'all below the minimum_intensity threshold')
elif e < len(self.datastore['events']):
self.datastore['relevant_events'] = rel_events
logging.info('Stored %d relevant event IDs', e)
return acc
def run_calc(self):
"""
Run a calculation and return results (reinvented from openquake.calculators.base)
"""
with self.calculator._monitor:
self.calculator._monitor.username = ''
try:
# Pre-execute setups
self.calculator.pre_execute()
#self.calculator.datastore.swmr_on()
oq = self.calculator.oqparam
dstore = self.calculator.datastore
self.calculator.set_param()
self.calculator.offset = 0
# Source model
print('self.__dict__ = ')
print(self.calculator.__dict__)
if oq.hazard_calculation_id: # from ruptures
dstore.parent = self.calculator.datastore.read(
oq.hazard_calculation_id)
elif hasattr(self.calculator, 'csm'): # from sources
self.calculator_build_events_from_sources()
#self.calculator.build_events_from_sources()
if (oq.ground_motion_fields is False
and oq.hazard_curves_from_gmfs is False):
return {}
elif 'rupture_model' not in oq.inputs:
logging.warning(
'There is no rupture_model, the calculator will just '
'import data without performing any calculation')
fake = logictree.FullLogicTree.fake()
dstore['full_lt'] = fake # needed to expose the outputs
dstore['weights'] = [1.]
return {}
else: # scenario
self.calculator._read_scenario_ruptures()
if (oq.ground_motion_fields is False
and oq.hazard_curves_from_gmfs is False):
return {}
# Intensity measure models
if oq.ground_motion_fields:
imts = oq.get_primary_imtls()
nrups = len(dstore['ruptures'])
base.create_gmf_data(dstore, imts, oq.get_sec_imts())
dstore.create_dset('gmf_data/sigma_epsilon',
getters.sig_eps_dt(oq.imtls))
dstore.create_dset('gmf_data/time_by_rup',
getters.time_dt, (nrups, ),
fillvalue=None)
# Prepare inputs for GmfGetter
nr = len(dstore['ruptures'])
logging.info('Reading {:_d} ruptures'.format(nr))
rgetters = getters.get_rupture_getters(
dstore,
oq.concurrent_tasks * 1.25,
srcfilter=self.calculator.srcfilter)
args = [(rgetter, self.calculator.param)
for rgetter in rgetters]
mon = performance.Monitor()
mon.version = version
mon.config = config
rcvr = 'tcp://%s:%s' % (config.dbserver.listen,
config.dbserver.receiver_ports)
skt = zeromq.Socket(rcvr, zeromq.zmq.PULL, 'bind').__enter__()
mon.backurl = 'tcp://%s:%s' % (config.dbserver.host, skt.port)
mon = mon.new(operation='total ' +
self.calculator.core_task.__func__.__name__,
measuremem=True)
mon.weight = getattr(args[0], 'weight',
1.) # used in task_info
mon.task_no = 1 # initialize the task number
args += (mon, )
self.args = args
self.mon = mon
self.dstore = dstore
finally:
print('FetchOpenQuake: OpenQuake Hazard Calculator defined.')
