def execute(self):
self.datastore.flush() # just to be sure
oq = self.oqparam
parent = self.datastore.parent
csm_info = parent['csm_info'] if parent else self.csm_info
self.init_logic_tree(csm_info)
self.set_param(
hdf5path=self.datastore.filename,
tempname=cache_epsilons(
self.datastore, oq, self.assetcol, self.crmodel, self.E))
srcfilter = self.src_filter(self.datastore.tempname)
maxw = self.E / (oq.concurrent_tasks or 1)
logging.info('Reading %d ruptures', len(self.datastore['ruptures']))
allargs = ((rgetter, srcfilter, self.param)
for rgetter in getters.gen_rupture_getters(
self.datastore, maxweight=maxw))
self.events_per_sid = []
self.lossbytes = 0
self.datastore.swmr_on()
smap = parallel.Starmap(
self.core_task.__func__, allargs, h5=self.datastore.hdf5)
smap.reduce(self.agg_dicts)
gmf_bytes = self.datastore['gmf_info']['gmfbytes'].sum()
logging.info(
'Produced %s of GMFs', general.humansize(gmf_bytes))
logging.info(
'Produced %s of losses', general.humansize(self.lossbytes))
return 1
def extract_rupture_info(dstore, what):
"""
Extract some information about the ruptures, including the boundary.
Example:
http://127.0.0.1:8800/v1/calc/30/extract/rupture_info?min_mag=6
"""
qdict = parse(what)
if 'min_mag' in qdict:
[min_mag] = qdict['min_mag']
else:
min_mag = 0
oq = dstore['oqparam']
dtlist = [('rup_id', U32), ('multiplicity', U16), ('mag', F32),
('centroid_lon', F32), ('centroid_lat', F32),
('centroid_depth', F32), ('trt', '<S50'), ('strike', F32),
('dip', F32), ('rake', F32)]
rows = []
boundaries = []
for rgetter in getters.gen_rupture_getters(dstore):
rups = rgetter.get_ruptures(min_mag)
rup_data = RuptureData(rgetter.trt, rgetter.rlzs_by_gsim)
for r, rup in zip(rup_data.to_array(rups), rups):
coords = ['%.5f %.5f' % xyz[:2] for xyz in zip(*r['boundaries'])]
boundaries.append('POLYGON((%s))' % ', '.join(coords))
rows.append(
(r['rup_id'], r['multiplicity'], r['mag'], r['lon'], r['lat'],
r['depth'], rgetter.trt, r['strike'], r['dip'], r['rake']))
arr = numpy.array(rows, dtlist)
arr.sort(order='rup_id')
geoms = gzip.compress('\n'.join(boundaries).encode('utf-8'))
return ArrayWrapper(
arr, dict(investigation_time=oq.investigation_time, boundaries=geoms))
def export_ruptures_csv(ekey, dstore):
"""
:param ekey: export key, i.e. a pair (datastore key, fmt)
:param dstore: datastore object
"""
oq = dstore['oqparam']
if 'scenario' in oq.calculation_mode:
return []
dest = dstore.export_path('ruptures.csv')
header = ('rupid multiplicity mag centroid_lon centroid_lat '
'centroid_depth trt strike dip rake boundary').split()
rows = []
for rgetter in gen_rupture_getters(dstore):
rups = rgetter.get_ruptures()
rup_data = calc.RuptureData(rgetter.trt, rgetter.rlzs_by_gsim)
for r in rup_data.to_array(rups):
rows.append((r['rup_id'], r['multiplicity'], r['mag'], r['lon'],
r['lat'], r['depth'], rgetter.trt, r['strike'],
r['dip'], r['rake'], r['boundary']))
rows.sort() # by rupture serial
comment = dstore.metadata
comment.update(investigation_time=oq.investigation_time,
ses_per_logic_tree_path=oq.ses_per_logic_tree_path)
writers.write_csv(dest, rows, header=header, sep='\t', comment=comment)
return [dest]
def execute(self):
"""
Compute risk from GMFs or ruptures depending on what is stored
"""
if 'gmf_data' not in self.datastore: # start from ruptures
smap = parallel.Starmap(start_ebrisk, h5=self.datastore.hdf5)
smap.monitor.save('srcfilter', self.src_filter())
smap.monitor.save('crmodel', self.crmodel)
smap.monitor.save('rlz_id', self.rlzs)
for rg in getters.gen_rupture_getters(
self.datastore, self.oqparam.concurrent_tasks):
smap.submit((rg, self.param))
smap.reduce(self.agg_dicts)
gmf_bytes = self.datastore['gmf_info']['gmfbytes']
if len(gmf_bytes) == 0:
raise RuntimeError(
'No GMFs were generated, perhaps they were '
'all below the minimum_intensity threshold')
logging.info(
'Produced %s of GMFs', general.humansize(gmf_bytes.sum()))
else: # start from GMFs
smap = parallel.Starmap(
event_based_risk, self.gen_args(), h5=self.datastore.hdf5)
smap.monitor.save('assets', self.assetcol.to_dframe())
smap.monitor.save('crmodel', self.crmodel)
smap.monitor.save('rlz_id', self.rlzs)
smap.reduce(self.agg_dicts)
return 1
def execute(self):
self.datastore.flush() # just to be sure
oq = self.oqparam
parent = self.datastore.parent
csm_info = parent['csm_info'] if parent else self.csm_info
self.init_logic_tree(csm_info)
self.set_param(hdf5path=self.datastore.filename,
tempname=cache_epsilons(self.datastore, oq,
self.assetcol, self.crmodel,
self.E))
srcfilter = self.src_filter(self.datastore.tempname)
logging.info('Sending %d ruptures', len(self.datastore['ruptures']))
self.events_per_sid = []
self.numlosses = 0
self.datastore.swmr_on()
self.indices = general.AccumDict(accum=[]) # rlzi -> [(start, stop)]
self.offset = 0
smap = parallel.Starmap(self.core_task.__func__,
h5=self.datastore.hdf5)
for rgetter in getters.gen_rupture_getters(self.datastore,
srcfilter=srcfilter):
smap.submit((rgetter, srcfilter, self.param))
smap.reduce(self.agg_dicts)
if self.indices:
self.datastore['asset_loss_table/indices'] = self.indices
gmf_bytes = self.datastore['gmf_info']['gmfbytes'].sum()
logging.info('Produced %s of GMFs', general.humansize(gmf_bytes))
logging.info('Stored {:_d} / {:_d} losses'.format(*self.numlosses))
return 1
def execute(self):
oq = self.oqparam
self.set_param()
self.offset = 0
srcfilter = self.src_filter(self.datastore.tempname)
self.indices = AccumDict(accum=[]) # sid, idx -> indices
if oq.hazard_calculation_id: # from ruptures
self.datastore.parent = util.read(oq.hazard_calculation_id)
self.init_logic_tree(self.datastore.parent['full_lt'])
else: # from sources
self.build_events_from_sources(srcfilter)
if (oq.ground_motion_fields is False and
oq.hazard_curves_from_gmfs is False):
return {}
if not oq.imtls:
raise InvalidFile('There are no intensity measure types in %s' %
oq.inputs['job_ini'])
N = len(self.sitecol.complete)
if oq.ground_motion_fields:
nrups = len(self.datastore['ruptures'])
self.datastore.create_dset('gmf_data/data', oq.gmf_data_dt())
self.datastore.create_dset('gmf_data/sigma_epsilon',
sig_eps_dt(oq.imtls))
self.datastore.create_dset(
'gmf_data/indices', hdf5.vuint32, shape=(N, 2), fillvalue=None)
self.datastore.create_dset('gmf_data/events_by_sid', U32, (N,))
self.datastore.create_dset('gmf_data/time_by_rup',
time_dt, (nrups,), fillvalue=None)
if oq.hazard_curves_from_gmfs:
self.param['rlz_by_event'] = self.datastore['events']['rlz_id']
# compute_gmfs in parallel
self.datastore.swmr_on()
logging.info('Reading %d ruptures', len(self.datastore['ruptures']))
iterargs = ((rgetter, srcfilter, self.param)
for rgetter in gen_rupture_getters(
self.datastore, srcfilter))
acc = parallel.Starmap(
self.core_task.__func__, iterargs, h5=self.datastore.hdf5,
num_cores=oq.num_cores
).reduce(self.agg_dicts, self.acc0())
if self.indices:
dset = self.datastore['gmf_data/indices']
num_evs = self.datastore['gmf_data/events_by_sid']
logging.info('Saving gmf_data/indices')
with self.monitor('saving gmf_data/indices', measuremem=True):
self.datastore['gmf_data/imts'] = ' '.join(oq.imtls)
for sid in self.sitecol.complete.sids:
start = numpy.array(self.indices[sid, 0])
stop = numpy.array(self.indices[sid, 1])
dset[sid, 0] = start
dset[sid, 1] = stop
num_evs[sid] = (stop - start).sum()
avg_events_by_sid = num_evs[()].sum() / N
logging.info('Found ~%d GMVs per site', avg_events_by_sid)
elif oq.ground_motion_fields:
raise RuntimeError('No GMFs were generated, perhaps they were '
'all below the minimum_intensity threshold')
return acc
def extract_ruptures(dstore, what):
"""
Extract some information about the ruptures, including the boundary.
Example:
http://127.0.0.1:8800/v1/calc/30/extract/ruptures?min_mag=6
"""
qdict = parse(what)
if 'min_mag' in qdict:
[min_mag] = qdict['min_mag']
else:
min_mag = 0
bio = io.StringIO()
first = True
trts = list(dstore.getitem('full_lt').attrs['trts'])
for rgetter in getters.gen_rupture_getters(dstore):
rups = [rupture._get_rupture(proxy.rec, proxy.geom, rgetter.trt)
for proxy in rgetter.get_proxies(min_mag)]
arr = rupture.to_csv_array(rups)
if first:
header = None
comment = dict(trts=trts)
first = False
else:
header = 'no-header'
comment = None
writers.write_csv(bio, arr, header=header, comment=comment)
return bio.getvalue()
def execute(self):
self.datastore.flush() # just to be sure
oq = self.oqparam
self.set_param(hdf5path=self.datastore.filename,
tempname=cache_epsilons(self.datastore, oq,
self.assetcol, self.crmodel,
self.E))
srcfilter = self.src_filter()
logging.info('Sending {:_d} ruptures'.format(
len(self.datastore['ruptures'])))
self.events_per_sid = []
self.numlosses = 0
self.datastore.swmr_on()
self.indices = general.AccumDict(accum=[]) # rlzi -> [(start, stop)]
smap = parallel.Starmap(start_ebrisk, h5=self.datastore.hdf5)
smap.monitor.save('srcfilter', srcfilter)
smap.monitor.save('crmodel', self.crmodel)
for rg in getters.gen_rupture_getters(self.datastore,
oq.concurrent_tasks):
smap.submit((rg, self.param))
smap.reduce(self.agg_dicts)
if self.indices:
self.datastore['event_loss_table/indices'] = self.indices
gmf_bytes = self.datastore['gmf_info']['gmfbytes'].sum()
logging.info('Produced %s of GMFs', general.humansize(gmf_bytes))
logging.info('Considered {:_d} / {:_d} losses'.format(*self.numlosses))
return 1
def execute(self):
self.datastore.flush() # just to be sure
oq = self.oqparam
self.set_param(hdf5path=self.datastore.filename,
tempname=cache_epsilons(self.datastore, oq,
self.assetcol, self.crmodel,
self.E))
srcfilter = self.src_filter()
logging.info('Sending {:_d} ruptures'.format(
len(self.datastore['ruptures'])))
self.events_per_sid = []
self.datastore.swmr_on()
self.avg_gmf = general.AccumDict(accum=numpy.zeros(self.N,
F32)) # imt -> gmvs
smap = parallel.Starmap(start_ebrisk, h5=self.datastore.hdf5)
smap.monitor.save('srcfilter', srcfilter)
smap.monitor.save('crmodel', self.crmodel)
for rg in getters.gen_rupture_getters(self.datastore,
oq.concurrent_tasks):
smap.submit((rg, self.param))
smap.reduce(self.agg_dicts)
gmf_bytes = self.datastore['gmf_info']['gmfbytes'].sum()
logging.info('Produced %s of GMFs', general.humansize(gmf_bytes))
size = general.humansize(self.datastore.getsize('agg_loss_table'))
logging.info('Stored %s in the agg_loss_table', size)
return 1
def export_ruptures_csv(ekey, dstore):
"""
:param ekey: export key, i.e. a pair (datastore key, fmt)
:param dstore: datastore object
"""
oq = dstore['oqparam']
if 'scenario' in oq.calculation_mode:
return []
dest = dstore.export_path('ruptures.csv')
header = ('rupid multiplicity mag centroid_lon centroid_lat '
'centroid_depth trt strike dip rake boundary').split()
rows = []
for rgetter in gen_rupture_getters(dstore):
rups = rgetter.get_ruptures()
rup_data = calc.RuptureData(rgetter.trt, rgetter.rlzs_by_gsim)
for r in rup_data.to_array(rups):
rows.append(
(r['rup_id'], r['multiplicity'], r['mag'],
r['lon'], r['lat'], r['depth'],
rgetter.trt, r['strike'], r['dip'], r['rake'],
r['boundary']))
rows.sort() # by rupture serial
comment = dstore.metadata
comment.update(investigation_time=oq.investigation_time,
ses_per_logic_tree_path=oq.ses_per_logic_tree_path)
writers.write_csv(dest, rows, header=header, sep='\t', comment=comment)
return [dest]
def extract_rupture(dstore, rup_id):
"""
Extract information about the given event index.
Example:
http://127.0.0.1:8800/v1/calc/30/extract/rupture/1066
"""
ridx = list(dstore['ruptures']['rup_id']).index(int(rup_id))
[getter] = getters.gen_rupture_getters(dstore, slice(ridx, ridx + 1))
yield from getter.get_rupdict().items()
def extract_rupture(dstore, serial):
"""
Extract information about the given event index.
Example:
http://127.0.0.1:8800/v1/calc/30/extract/rupture/1066
"""
ridx = list(dstore['ruptures']['serial']).index(int(serial))
[getter] = getters.gen_rupture_getters(dstore, slice(ridx, ridx + 1))
yield from getter.get_rupdict().items()
def extract_rupture(dstore, rup_id):
"""
Extract information about the given event index.
Example:
http://127.0.0.1:8800/v1/calc/30/extract/rupture/1066
"""
ridx = list(dstore['ruptures']['id']).index(int(rup_id))
[getter] = getters.gen_rupture_getters(dstore, slice(ridx, ridx + 1))
[ebr] = getter.get_ruptures()
return ArrayWrapper((), ebr.rupture.todict())
def gen_rupture_getters(self):
"""
:returns: a list of RuptureGetters
"""
dstore = (self.datastore.parent
if self.datastore.parent else self.datastore)
yield from gen_rupture_getters(
dstore, concurrent_tasks=self.oqparam.concurrent_tasks or 1)
if self.datastore.parent:
self.datastore.parent.close()
def export_agg_losses_ebr(ekey, dstore):
"""
:param ekey: export key, i.e. a pair (datastore key, fmt)
:param dstore: datastore object
"""
if 'ruptures' not in dstore:
logging.warning('There are no ruptures in the datastore')
return []
name, ext = export.keyfunc(ekey)
agg_losses = dstore['losses_by_event']
has_rup_data = 'ruptures' in dstore
extra_list = [('magnitude', F32),
('centroid_lon', F32),
('centroid_lat', F32),
('centroid_depth', F32)] if has_rup_data else []
oq = dstore['oqparam']
lti = oq.lti
dtlist = ([('event_id', U64), ('rup_id', U32), ('year', U32),
('rlzi', U16)] + extra_list + oq.loss_dt_list())
elt_dt = numpy.dtype(dtlist)
elt = numpy.zeros(len(agg_losses), elt_dt)
writer = writers.CsvWriter(fmt=writers.FIVEDIGITS)
events = dstore['events'].value
events_by_rupid = collections.defaultdict(list)
for event in events:
rupid = event['eid'] // TWO32
events_by_rupid[rupid].append(event)
year_of = year_dict(events['eid'], oq.investigation_time, oq.ses_seed)
rup_data = {}
event_by_eid = {} # eid -> event
# populate rup_data and event_by_eid
# TODO: avoid reading the events twice
for rgetter in getters.gen_rupture_getters(dstore):
ruptures = rgetter.get_ruptures()
for ebr in ruptures:
for event in events_by_rupid[ebr.serial]:
event_by_eid[event['eid']] = event
if has_rup_data:
rup_data.update(get_rup_data(ruptures))
for r, row in enumerate(agg_losses):
rec = elt[r]
event = event_by_eid[row['eid']]
rec['event_id'] = eid = event['eid']
rec['year'] = year_of[eid]
rec['rlzi'] = row['rlzi']
if rup_data:
rec['rup_id'] = rup_id = event['eid'] // TWO32
(rec['magnitude'], rec['centroid_lon'], rec['centroid_lat'],
rec['centroid_depth']) = rup_data[rup_id]
for lt, i in lti.items():
rec[lt] = row['loss'][i]
elt.sort(order=['year', 'event_id', 'rlzi'])
dest = dstore.build_fname('agg_losses', 'all', 'csv')
writer.save(elt, dest)
return writer.getsaved()
def execute(self):
oq = self.oqparam
self.set_param()
self.offset = 0
if oq.hazard_calculation_id: # from ruptures
self.datastore.parent = datastore.read(oq.hazard_calculation_id)
elif hasattr(self, 'csm'): # from sources
self.build_events_from_sources()
if (oq.ground_motion_fields is False
and oq.hazard_curves_from_gmfs is False):
return {}
elif 'rupture_model' not in oq.inputs:
logging.warning(
'There is no rupture_model, the calculator will just '
'import data without performing any calculation')
fake = logictree.FullLogicTree.fake()
self.datastore['full_lt'] = fake # needed to expose the outputs
self.datastore['weights'] = [1.]
return {}
else: # scenario
self._read_scenario_ruptures()
if (oq.ground_motion_fields is False
and oq.hazard_curves_from_gmfs is False):
return {}
if oq.ground_motion_fields:
imts = oq.get_primary_imtls()
nrups = len(self.datastore['ruptures'])
base.create_gmf_data(self.datastore, imts, oq.get_sec_imts())
self.datastore.create_dset('gmf_data/sigma_epsilon',
sig_eps_dt(oq.imtls))
self.datastore.create_dset('gmf_data/time_by_rup',
time_dt, (nrups, ),
fillvalue=None)
# compute_gmfs in parallel
nr = len(self.datastore['ruptures'])
logging.info('Reading {:_d} ruptures'.format(nr))
allargs = [(rgetter, self.param) for rgetter in gen_rupture_getters(
self.datastore, oq.concurrent_tasks)]
# reading the args is fast since we are not prefiltering the ruptures,
# nor reading the geometries; using an iterator would cause the usual
# damned h5py error, last seen on macos
self.datastore.swmr_on()
smap = parallel.Starmap(self.core_task.__func__,
allargs,
h5=self.datastore.hdf5)
smap.monitor.save('srcfilter', self.srcfilter)
acc = smap.reduce(self.agg_dicts, self.acc0())
if 'gmf_data' not in self.datastore:
return acc
if oq.ground_motion_fields:
with self.monitor('saving avg_gmf', measuremem=True):
self.save_avg_gmf()
return acc
def export_agg_losses_ebr(ekey, dstore):
"""
:param ekey: export key, i.e. a pair (datastore key, fmt)
:param dstore: datastore object
"""
if 'ruptures' not in dstore:
logging.warning('There are no ruptures in the datastore')
return []
name, ext = export.keyfunc(ekey)
agg_losses = dstore['losses_by_event']
has_rup_data = 'ruptures' in dstore
extra_list = [('magnitude', F32),
('centroid_lon', F32),
('centroid_lat', F32),
('centroid_depth', F32)] if has_rup_data else []
oq = dstore['oqparam']
lti = oq.lti
dtlist = ([('event_id', U64), ('rup_id', U32), ('year', U32)]
+ extra_list + oq.loss_dt_list())
elt_dt = numpy.dtype(dtlist)
elt = numpy.zeros(len(agg_losses), elt_dt)
writer = writers.CsvWriter(fmt=writers.FIVEDIGITS)
events = dstore['events'].value
events_by_rupid = collections.defaultdict(list)
for event in events:
rupid = event['id'] // TWO32
events_by_rupid[rupid].append(event)
year_of = year_dict(events['id'], oq.investigation_time, oq.ses_seed)
rup_data = {}
event_by_eid = {} # eid -> event
# populate rup_data and event_by_eid
# TODO: avoid reading the events twice
for rgetter in getters.gen_rupture_getters(dstore):
ruptures = rgetter.get_ruptures()
for ebr in ruptures:
for event in events_by_rupid[ebr.serial]:
event_by_eid[event['id']] = event
if has_rup_data:
rup_data.update(get_rup_data(ruptures))
for r, row in enumerate(agg_losses):
rec = elt[r]
event = event_by_eid[row['eid']]
rec['event_id'] = eid = event['id']
rec['year'] = year_of[eid]
if rup_data:
rec['rup_id'] = rup_id = event['id'] // TWO32
(rec['magnitude'], rec['centroid_lon'], rec['centroid_lat'],
rec['centroid_depth']) = rup_data[rup_id]
for lt, i in lti.items():
rec[lt] = row['loss'][i]
elt.sort(order=['year', 'event_id'])
dest = dstore.build_fname('elt', '', 'csv')
writer.save(elt, dest)
return writer.getsaved()
def save_events(self, rup_array):
"""
:param rup_array: an array of ruptures with fields et_id
:returns: a list of RuptureGetters
"""
from openquake.calculators.getters import (get_eid_rlz,
gen_rupture_getters)
# this is very fast compared to saving the ruptures
E = rup_array['n_occ'].sum()
self.check_overflow(E) # check the number of events
events = numpy.zeros(E, rupture.events_dt)
# when computing the events all ruptures must be considered,
# including the ones far away that will be discarded later on
rgetters = gen_rupture_getters(self.datastore,
self.oqparam.concurrent_tasks)
# build the associations eid -> rlz sequentially or in parallel
# this is very fast: I saw 30 million events associated in 1 minute!
logging.info('Associating event_id -> rlz_id for {:_d} events '
'and {:_d} ruptures'.format(len(events), len(rup_array)))
iterargs = ((rg.proxies, rg.rlzs_by_gsim) for rg in rgetters)
if len(events) < 1E5:
it = itertools.starmap(get_eid_rlz, iterargs)
else:
it = parallel.Starmap(get_eid_rlz,
iterargs,
progress=logging.debug,
h5=self.datastore.hdf5)
i = 0
for eid_rlz in it:
for er in eid_rlz:
events[i] = er
i += 1
if i >= TWO32:
raise ValueError('There are more than %d events!' % i)
events.sort(order='rup_id') # fast too
# sanity check
n_unique_events = len(numpy.unique(events[['id', 'rup_id']]))
assert n_unique_events == len(events), (n_unique_events, len(events))
events['id'] = numpy.arange(len(events))
# set event year and event ses starting from 1
nses = self.oqparam.ses_per_logic_tree_path
extra = numpy.zeros(len(events), [('year', U32), ('ses_id', U32)])
numpy.random.seed(self.oqparam.ses_seed)
if self.oqparam.investigation_time:
itime = int(self.oqparam.investigation_time)
extra['year'] = numpy.random.choice(itime, len(events)) + 1
extra['ses_id'] = numpy.random.choice(nses, len(events)) + 1
self.datastore['events'] = util.compose_arrays(events, extra)
eindices = get_indices(events['rup_id'])
arr = numpy.array(list(eindices.values()))[:, 0, :]
self.datastore['ruptures']['e0'] = arr[:, 0]
self.datastore['ruptures']['e1'] = arr[:, 1]
def gen_rupture_getters(self, num_events=0):
"""
:returns: a list of RuptureGetters
"""
oq = self.oqparam
dstore = (self.datastore.parent
if self.datastore.parent else self.datastore)
E = num_events or len(dstore['events'])
yield from gen_rupture_getters(dstore,
maxweight=E /
(oq.concurrent_tasks or 1))
if self.datastore.parent:
self.datastore.parent.close()
def export_gmf_scenario_csv(ekey, dstore):
what = ekey[0].split('/')
if len(what) == 1:
raise ValueError(r'Missing "/rup-\d+"')
oq = dstore['oqparam']
csm_info = dstore['csm_info']
rlzs_assoc = csm_info.get_rlzs_assoc()
num_ruptures = len(dstore['ruptures'])
imts = list(oq.imtls)
mo = re.match(r'rup-(\d+)$', what[1])
if mo is None:
raise ValueError(
r"Invalid format: %r does not match 'rup-(\d+)$'" % what[1])
ridx = int(mo.group(1))
assert 0 <= ridx < num_ruptures, ridx
# for scenario there is an unique grp_id=0
[rgetter] = gen_rupture_getters(dstore, slice(ridx, ridx + 1))
[ebr] = rgetter.get_ruptures()
sitecol = dstore['sitecol'].complete
srcfilter = filters.SourceFilter(sitecol, oq.maximum_distance)
getter = GmfGetter(rgetter, srcfilter, oq)
getter.init()
eids = rgetter.get_eid_rlz()['eid']
sids = getter.computers[0].sids
rlzs = rlzs_assoc.realizations
hazardr = [collections.defaultdict(list) for rlz in rlzs]
for sid, haz in getter.get_hazard().items():
for rec in haz:
hazardr[rec['rlzi']][sid].append(rec)
fields = ['eid-%03d' % eid for eid in eids]
dt = numpy.dtype([(f, F32) for f in fields])
mesh = numpy.zeros(len(sids), [('lon', F64), ('lat', F64)])
mesh['lon'] = sitecol.lons[sids]
mesh['lat'] = sitecol.lats[sids]
writer = writers.CsvWriter(fmt='%.5f')
for rlzi in range(len(rlzs)):
hazard = hazardr[rlzi]
for imti, imt in enumerate(imts):
gmfs = numpy.zeros(len(sids), dt)
for s, sid in enumerate(sids):
for rec in hazard[sid]:
event = 'eid-%03d' % rec['eid']
gmfs[s][event] = rec['gmv'][imti]
dest = dstore.build_fname(
'gmf', 'rup-%s-rlz-%s-%s' % (ebr.serial, rlzi, imt), 'csv')
data = util.compose_arrays(mesh, gmfs)
writer.save(data, dest)
return writer.getsaved()
def export_gmf_scenario_csv(ekey, dstore):
what = ekey[0].split('/')
if len(what) == 1:
raise ValueError(r'Missing "/rup-\d+"')
oq = dstore['oqparam']
csm_info = dstore['csm_info']
rlzs_assoc = csm_info.get_rlzs_assoc()
num_ruptures = len(dstore['ruptures'])
imts = list(oq.imtls)
mo = re.match(r'rup-(\d+)$', what[1])
if mo is None:
raise ValueError(r"Invalid format: %r does not match 'rup-(\d+)$'" %
what[1])
ridx = int(mo.group(1))
assert 0 <= ridx < num_ruptures, ridx
# for scenario there is an unique grp_id=0
[rgetter] = gen_rupture_getters(dstore, slice(ridx, ridx + 1))
[ebr] = rgetter.get_ruptures()
sitecol = dstore['sitecol'].complete
srcfilter = filters.SourceFilter(sitecol, oq.maximum_distance)
getter = GmfGetter(rgetter, srcfilter, oq)
getter.init()
eids = rgetter.get_eid_rlz()['eid']
sids = getter.computers[0].sids
rlzs = rlzs_assoc.realizations
hazardr = [collections.defaultdict(list) for rlz in rlzs]
for sid, haz in getter.get_hazard().items():
for rec in haz:
hazardr[rec['rlzi']][sid].append(rec)
fields = ['eid-%03d' % eid for eid in eids]
dt = numpy.dtype([(f, F32) for f in fields])
mesh = numpy.zeros(len(sids), [('lon', F64), ('lat', F64)])
mesh['lon'] = sitecol.lons[sids]
mesh['lat'] = sitecol.lats[sids]
writer = writers.CsvWriter(fmt='%.5f')
for rlzi in range(len(rlzs)):
hazard = hazardr[rlzi]
for imti, imt in enumerate(imts):
gmfs = numpy.zeros(len(sids), dt)
for s, sid in enumerate(sids):
for rec in hazard[sid]:
event = 'eid-%03d' % rec['eid']
gmfs[s][event] = rec['gmv'][imti]
dest = dstore.build_fname(
'gmf', 'rup-%s-rlz-%s-%s' % (ebr.serial, rlzi, imt), 'csv')
data = util.compose_arrays(mesh, gmfs)
writer.save(data, dest)
return writer.getsaved()
def export_ruptures_xml(ekey, dstore):
"""
:param ekey: export key, i.e. a pair (datastore key, fmt)
:param dstore: datastore object
"""
fmt = ekey[-1]
oq = dstore['oqparam']
num_ses = oq.ses_per_logic_tree_path
ruptures_by_grp = AccumDict(accum=[])
for rgetter in gen_rupture_getters(dstore):
ebrs = [ebr.export(rgetter.rlzs_by_gsim, num_ses)
for ebr in rgetter.get_ruptures()]
ruptures_by_grp[rgetter.grp_id].extend(ebrs)
dest = dstore.export_path('ses.' + fmt)
writer = hazard_writers.SESXMLWriter(dest)
writer.serialize(ruptures_by_grp, oq.investigation_time)
return [dest]
def extract_event_info(dstore, eidx):
"""
Extract information about the given event index.
Example:
http://127.0.0.1:8800/v1/calc/30/extract/event_info/0
"""
event = dstore['events'][int(eidx)]
ridx = event['rup_id']
[getter] = getters.gen_rupture_getters(dstore, slice(ridx, ridx + 1))
rupdict = getter.get_rupdict()
rlzi = event['rlz_id']
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
gsim = rlzs_assoc.gsim_by_trt[rlzi][rupdict['trt']]
for key, val in rupdict.items():
yield key, val
yield 'rlzi', rlzi
yield 'gsim', repr(gsim)
def extract_event_info(dstore, eidx):
"""
Extract information about the given event index.
Example:
http://127.0.0.1:8800/v1/calc/30/extract/event_info/0
"""
event = dstore['events'][int(eidx)]
serial = int(event['id'] // TWO32)
ridx = list(dstore['ruptures']['serial']).index(serial)
[getter] = getters.gen_rupture_getters(dstore, slice(ridx, ridx + 1))
rupdict = getter.get_rupdict()
rlzi = event['rlz']
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
gsim = rlzs_assoc.gsim_by_trt[rlzi][rupdict['trt']]
for key, val in rupdict.items():
yield key, val
yield 'rlzi', rlzi
yield 'gsim', repr(gsim)
def gen_rupture_getters(self):
"""
:returns: a list of RuptureGetters
"""
dstore = (self.datastore.parent if self.datastore.parent
else self.datastore)
hdf5cache = dstore.hdf5cache()
mode = 'r+' if os.path.exists(hdf5cache) else 'w'
with hdf5.File(hdf5cache, mode) as cache:
if 'ruptures' not in cache:
dstore.hdf5.copy('ruptures', cache)
if 'rupgeoms' not in cache:
dstore.hdf5.copy('rupgeoms', cache)
yield from gen_rupture_getters(
dstore, concurrent_tasks=self.oqparam.concurrent_tasks or 1,
hdf5cache=hdf5cache)
if self.datastore.parent:
self.datastore.parent.close()
def gen_rupture_getters(self):
"""
:returns: a list of RuptureGetters
"""
dstore = (self.datastore.parent if self.datastore.parent
else self.datastore)
hdf5cache = dstore.hdf5cache()
mode = 'r+' if os.path.exists(hdf5cache) else 'w'
with hdf5.File(hdf5cache, mode) as cache:
if 'ruptures' not in cache:
dstore.hdf5.copy('ruptures', cache)
if 'rupgeoms' not in cache:
dstore.hdf5.copy('rupgeoms', cache)
yield from gen_rupture_getters(
dstore, concurrent_tasks=self.oqparam.concurrent_tasks or 1,
hdf5cache=hdf5cache)
if self.datastore.parent:
self.datastore.parent.close()
def export_ruptures_xml(ekey, dstore):
"""
:param ekey: export key, i.e. a pair (datastore key, fmt)
:param dstore: datastore object
"""
fmt = ekey[-1]
oq = dstore['oqparam']
num_ses = oq.ses_per_logic_tree_path
mesh = get_mesh(dstore['sitecol'])
ruptures_by_grp = {}
for rgetter in gen_rupture_getters(dstore):
ebrs = [ebr.export(mesh, rgetter.rlzs_by_gsim, num_ses)
for ebr in rgetter.get_ruptures()]
if ebrs:
ruptures_by_grp[rgetter.grp_id] = ebrs
dest = dstore.export_path('ses.' + fmt)
writer = hazard_writers.SESXMLWriter(dest)
writer.serialize(ruptures_by_grp, oq.investigation_time)
return [dest]
def export_ruptures_xml(ekey, dstore):
"""
:param ekey: export key, i.e. a pair (datastore key, fmt)
:param dstore: datastore object
"""
fmt = ekey[-1]
oq = dstore['oqparam']
sf = filters.SourceFilter(dstore['sitecol'], oq.maximum_distance)
num_ses = oq.ses_per_logic_tree_path
ruptures_by_grp = {}
for rgetter in gen_rupture_getters(dstore):
ebrs = [ebr.export(rgetter.rlzs_by_gsim, num_ses)
for ebr in rgetter.get_ruptures(sf)]
if ebrs:
ruptures_by_grp[rgetter.grp_id] = ebrs
dest = dstore.export_path('ses.' + fmt)
writer = hazard_writers.SESXMLWriter(dest)
writer.serialize(ruptures_by_grp, oq.investigation_time)
return [dest]
def export_ruptures_xml(ekey, dstore):
"""
:param ekey: export key, i.e. a pair (datastore key, fmt)
:param dstore: datastore object
"""
fmt = ekey[-1]
oq = dstore['oqparam']
events = group_array(dstore['events'][()], 'rup_id')
ruptures_by_grp = AccumDict(accum=[])
for rgetter in gen_rupture_getters(dstore):
ebrs = []
for proxy in rgetter.get_proxies():
events_by_ses = group_array(events[proxy['id']], 'ses_id')
ebr = proxy.to_ebr(rgetter.trt)
ebrs.append(ebr.export(events_by_ses))
ruptures_by_grp[rgetter.et_id].extend(ebrs)
dest = dstore.export_path('ses.' + fmt)
writer = hazard_writers.SESXMLWriter(dest)
writer.serialize(ruptures_by_grp, oq.investigation_time)
return [dest]
def save_events(self, rup_array):
"""
:param rup_array: an array of ruptures with fields grp_id
:returns: a list of RuptureGetters
"""
# this is very fast compared to saving the ruptures
eids = rupture.get_eids(rup_array, self.samples_by_grp,
self.num_rlzs_by_grp)
self.check_overflow() # check the number of events
events = numpy.zeros(len(eids), rupture.events_dt)
# when computing the events all ruptures must be considered,
# including the ones far away that will be discarded later on
rgetters = gen_rupture_getters(self.datastore)
# build the associations eid -> rlz sequentially or in parallel
# this is very fast: I saw 30 million events associated in 1 minute!
logging.info('Building assocs event_id -> rlz_id for {:_d} events'
' and {:_d} ruptures'.format(len(events), len(rup_array)))
if len(events) < 1E5:
it = map(RuptureGetter.get_eid_rlz, rgetters)
else:
it = parallel.Starmap(RuptureGetter.get_eid_rlz,
((rgetter, ) for rgetter in rgetters),
progress=logging.debug,
h5=self.datastore.hdf5)
i = 0
for eid_rlz in it:
for er in eid_rlz:
events[i] = er
i += 1
if i >= TWO32:
raise ValueError('There are more than %d events!' % i)
events.sort(order='rup_id') # fast too
# sanity check
n_unique_events = len(numpy.unique(events[['id', 'rup_id']]))
assert n_unique_events == len(events), (n_unique_events, len(events))
events['id'] = numpy.arange(len(events))
self.datastore['events'] = events
eindices = get_indices(events['rup_id'])
arr = numpy.array(list(eindices.values()))[:, 0, :]
self.datastore['eslices'] = arr # shape (U, 2)
def execute(self):
oq = self.oqparam
self.set_param()
self.offset = 0
if oq.hazard_calculation_id: # from ruptures
self.datastore.parent = util.read(oq.hazard_calculation_id)
elif hasattr(self, 'csm'): # from sources
self.build_events_from_sources()
if (oq.ground_motion_fields is False
and oq.hazard_curves_from_gmfs is False):
return {}
elif 'rupture_model' not in oq.inputs: # download ShakeMap
logging.warning(
'There is no rupture_model, the calculator will just '
'import data without performing any calculation')
fake = logictree.FullLogicTree.fake()
self.datastore['full_lt'] = fake # needed to expose the outputs
return {}
else: # scenario
self._read_scenario_ruptures()
if (oq.ground_motion_fields is False
and oq.hazard_curves_from_gmfs is False):
return {}
N = len(self.sitecol.complete)
if oq.ground_motion_fields:
M = len(oq.get_primary_imtls())
nrups = len(self.datastore['ruptures'])
base.create_gmf_data(self.datastore, M, oq.get_sec_imts())
self.datastore.create_dset('gmf_data/sigma_epsilon',
sig_eps_dt(oq.imtls))
self.datastore.create_dset('gmf_data/events_by_sid', U32, (N, ))
self.datastore.create_dset('gmf_data/time_by_rup',
time_dt, (nrups, ),
fillvalue=None)
# compute_gmfs in parallel
nr = len(self.datastore['ruptures'])
logging.info('Reading {:_d} ruptures'.format(nr))
allargs = [(rgetter, self.param) for rgetter in gen_rupture_getters(
self.datastore, oq.concurrent_tasks)]
# reading the args is fast since we are not prefiltering the ruptures,
# nor reading the geometries; using an iterator would cause the usual
# damned h5py error, last seen on macos
self.datastore.swmr_on()
smap = parallel.Starmap(self.core_task.__func__,
allargs,
h5=self.datastore.hdf5)
smap.monitor.save('srcfilter', self.srcfilter)
acc = smap.reduce(self.agg_dicts, self.acc0())
if 'gmf_data' not in self.datastore:
return acc
if oq.ground_motion_fields:
with self.monitor('saving avg_gmf', measuremem=True):
self.weights = self.datastore['weights'][:]
self.rlzs = self.datastore['events']['rlz_id']
self.num_events = numpy.bincount(self.rlzs) # events by rlz
avg_gmf = {
imt: numpy.zeros(self.N, F32)
for imt in oq.all_imts()
}
rel_events = self.save_avg_gmf(avg_gmf)
self.datastore.create_dframe('avg_gmf', avg_gmf.items())
e = len(rel_events)
if e == 0:
raise RuntimeError('No GMFs were generated, perhaps they were '
'all below the minimum_intensity threshold')
elif e < len(self.datastore['events']):
self.datastore['relevant_events'] = rel_events
logging.info('Stored %d relevant event IDs', e)
return acc
def execute(self):
oq = self.oqparam
self.set_param()
self.offset = 0
if oq.hazard_calculation_id: # from ruptures
self.datastore.parent = util.read(oq.hazard_calculation_id)
elif hasattr(self, 'csm'): # from sources
self.build_events_from_sources()
if (oq.ground_motion_fields is False
and oq.hazard_curves_from_gmfs is False):
return {}
elif 'rupture_model' not in oq.inputs: # download ShakeMap
logging.warning(
'There is no rupture_model, the calculator will just '
'import data without performing any calculation')
fake = logictree.FullLogicTree.fake()
self.datastore['full_lt'] = fake # needed to expose the outputs
return {}
else: # scenario
self._read_scenario_ruptures()
if (oq.ground_motion_fields is False
and oq.hazard_curves_from_gmfs is False):
return {}
if not oq.imtls:
raise InvalidFile('There are no intensity measure types in %s' %
oq.inputs['job_ini'])
N = len(self.sitecol.complete)
if oq.ground_motion_fields:
M = len(oq.imtls)
nrups = len(self.datastore['ruptures'])
base.create_gmf_data(self.datastore, M, self.param['sec_perils'])
self.datastore.create_dset('gmf_data/sigma_epsilon',
sig_eps_dt(oq.imtls))
self.datastore.create_dset('gmf_data/events_by_sid', U32, (N, ))
self.datastore.create_dset('gmf_data/time_by_rup',
time_dt, (nrups, ),
fillvalue=None)
# compute_gmfs in parallel
nr = len(self.datastore['ruptures'])
self.datastore.swmr_on()
logging.info('Reading {:_d} ruptures'.format(nr))
iterargs = ((rgetter, self.param) for rgetter in gen_rupture_getters(
self.datastore, oq.concurrent_tasks))
smap = parallel.Starmap(self.core_task.__func__,
iterargs,
h5=self.datastore.hdf5)
smap.monitor.save('srcfilter', self.srcfilter)
acc = smap.reduce(self.agg_dicts, self.acc0())
if 'gmf_data' not in self.datastore:
return acc
if oq.ground_motion_fields:
eids = self.datastore['gmf_data/eid'][:]
rel_events = numpy.unique(eids)
e = len(rel_events)
if e == 0:
raise RuntimeError('No GMFs were generated, perhaps they were '
'all below the minimum_intensity threshold')
elif e < len(self.datastore['events']):
self.datastore['relevant_events'] = rel_events
logging.info('Stored %d relevant event IDs', e)
return acc