def execute(self):
"""
Compute risk from GMFs or ruptures depending on what is stored
"""
if 'gmf_data' not in self.datastore: # start from ruptures
srcfilter = self.src_filter()
smap = parallel.Starmap(start_ebrisk, h5=self.datastore.hdf5)
smap.monitor.save('srcfilter', srcfilter)
smap.monitor.save('crmodel', self.crmodel)
smap.monitor.save('rlz_id', self.rlzs)
for rg in getters.get_rupture_getters(
self.datastore, self.oqparam.concurrent_tasks,
srcfilter=srcfilter):
smap.submit((rg, self.param))
smap.reduce(self.agg_dicts)
gmf_bytes = self.datastore['gmf_info']['gmfbytes']
if len(gmf_bytes) == 0:
raise RuntimeError(
'No GMFs were generated, perhaps they were '
'all below the minimum_intensity threshold')
logging.info(
'Produced %s of GMFs', general.humansize(gmf_bytes.sum()))
else: # start from GMFs
eids = self.datastore['gmf_data/eid'][:]
logging.info('Processing {:_d} rows of gmf_data'.format(len(eids)))
self.datastore.swmr_on() # crucial!
smap = parallel.Starmap(
event_based_risk, self.gen_args(eids), h5=self.datastore.hdf5)
smap.monitor.save('assets', self.assetcol.to_dframe())
smap.monitor.save('crmodel', self.crmodel)
smap.monitor.save('rlz_id', self.rlzs)
smap.reduce(self.agg_dicts)
return 1
def export_ruptures_csv(ekey, dstore):
"""
:param ekey: export key, i.e. a pair (datastore key, fmt)
:param dstore: datastore object
"""
oq = dstore['oqparam']
if 'scenario' in oq.calculation_mode:
return []
dest = dstore.export_path('ruptures.csv')
header = ('rupid multiplicity mag centroid_lon centroid_lat '
'centroid_depth trt strike dip rake boundary').split()
rows = []
for rgetter in get_rupture_getters(dstore):
rups = list(rgetter)
rup_data = calc.RuptureData(rgetter.trt, rgetter.rlzs_by_gsim)
for r in rup_data.to_array(rups):
rows.append(
(r['rup_id'], r['multiplicity'], r['mag'],
r['lon'], r['lat'], r['depth'],
rgetter.trt, r['strike'], r['dip'], r['rake'],
r['boundary']))
rows.sort() # by rupture serial
comment = 'investigation_time=%s, ses_per_logic_tree_path=%s' % (
oq.investigation_time, oq.ses_per_logic_tree_path)
writers.write_csv(dest, rows, header=header, sep='\t', comment=comment)
return [dest]
def view_rupture(token, dstore):
"""
Show a rupture with its geometry
"""
rup_id = int(token.split(':')[1])
slc = slice(rup_id, rup_id + 1)
dicts = []
for rgetter in get_rupture_getters(dstore, slc=slc):
dicts.append(rgetter.get_rupdict())
return str(dicts)
def execute(self):
oq = self.oqparam
dstore = self.datastore
self.set_param()
self.offset = 0
if oq.hazard_calculation_id: # from ruptures
dstore.parent = datastore.read(oq.hazard_calculation_id)
elif hasattr(self, 'csm'): # from sources
self.build_events_from_sources()
if (oq.ground_motion_fields is False
and oq.hazard_curves_from_gmfs is False):
return {}
elif 'rupture_model' not in oq.inputs:
logging.warning(
'There is no rupture_model, the calculator will just '
'import data without performing any calculation')
fake = logictree.FullLogicTree.fake()
dstore['full_lt'] = fake # needed to expose the outputs
dstore['weights'] = [1.]
return {}
else: # scenario
self._read_scenario_ruptures()
if (oq.ground_motion_fields is False
and oq.hazard_curves_from_gmfs is False):
return {}
if oq.ground_motion_fields:
imts = oq.get_primary_imtls()
nrups = len(dstore['ruptures'])
base.create_gmf_data(dstore, imts, oq.get_sec_imts())
dstore.create_dset('gmf_data/sigma_epsilon', sig_eps_dt(oq.imtls))
dstore.create_dset('gmf_data/time_by_rup',
time_dt, (nrups, ),
fillvalue=None)
# compute_gmfs in parallel
nr = len(dstore['ruptures'])
logging.info('Reading {:_d} ruptures'.format(nr))
rgetters = get_rupture_getters(dstore,
oq.concurrent_tasks * 1.25,
srcfilter=self.srcfilter)
allargs = [(rgetter, self.param) for rgetter in rgetters]
dstore.swmr_on()
smap = parallel.Starmap(self.core_task.__func__,
allargs,
h5=dstore.hdf5)
smap.monitor.save('srcfilter', self.srcfilter)
acc = smap.reduce(self.agg_dicts, self.acc0())
if 'gmf_data' not in dstore:
return acc
if oq.ground_motion_fields:
with self.monitor('saving avg_gmf', measuremem=True):
self.save_avg_gmf()
return acc
def export_agg_losses_ebr(ekey, dstore):
"""
:param ekey: export key, i.e. a pair (datastore key, fmt)
:param dstore: datastore object
"""
if 'ruptures' not in dstore:
logging.warn('There are no ruptures in the datastore')
return []
name, ext = export.keyfunc(ekey)
agg_losses = dstore['losses_by_event']
has_rup_data = 'ruptures' in dstore
extra_list = [('magnitude', F32), ('centroid_lon', F32),
('centroid_lat', F32),
('centroid_depth', F32)] if has_rup_data else []
oq = dstore['oqparam']
lti = oq.lti
dtlist = ([('event_id', U64), ('rup_id', U32), ('year', U32),
('rlzi', U16)] + extra_list + oq.loss_dt_list())
elt_dt = numpy.dtype(dtlist)
elt = numpy.zeros(len(agg_losses), elt_dt)
writer = writers.CsvWriter(fmt=writers.FIVEDIGITS)
events = dstore['events'].value
events_by_rupid = collections.defaultdict(list)
for event in events:
rupid = event['eid'] // TWO32
events_by_rupid[rupid].append(event)
year_of = year_dict(events['eid'], oq.investigation_time, oq.ses_seed)
rup_data = {}
event_by_eid = {} # eid -> event
# populate rup_data and event_by_eid
# TODO: avoid reading the events twice
for rgetter in getters.get_rupture_getters(dstore):
for ebr in rgetter:
for event in events_by_rupid[ebr.serial]:
event_by_eid[event['eid']] = event
if has_rup_data:
rup_data.update(get_rup_data(rgetter))
for r, row in enumerate(agg_losses):
rec = elt[r]
event = event_by_eid[row['eid']]
rec['event_id'] = eid = event['eid']
rec['year'] = year_of[eid]
rec['rlzi'] = row['rlzi']
if rup_data:
rec['rup_id'] = rup_id = event['eid'] // TWO32
(rec['magnitude'], rec['centroid_lon'], rec['centroid_lat'],
rec['centroid_depth']) = rup_data[rup_id]
for lt, i in lti.items():
rec[lt] = row['loss'][i]
elt.sort(order=['year', 'event_id', 'rlzi'])
dest = dstore.build_fname('agg_losses', 'all', 'csv')
writer.save(elt, dest)
return writer.getsaved()
def export_gmf_scenario_csv(ekey, dstore):
what = ekey[0].split('/')
if len(what) == 1:
raise ValueError('Missing "/rup-\d+"')
oq = dstore['oqparam']
csm_info = dstore['csm_info']
rlzs_assoc = csm_info.get_rlzs_assoc()
num_ruptures = len(dstore['ruptures'])
imts = list(oq.imtls)
mo = re.match('rup-(\d+)$', what[1])
if mo is None:
raise ValueError(
"Invalid format: %r does not match 'rup-(\d+)$'" % what[1])
ridx = int(mo.group(1))
assert 0 <= ridx < num_ruptures, ridx
# for scenario there is an unique grp_id=0
[[ebr]] = get_rupture_getters(dstore, slice(ridx, ridx + 1))
rlzs_by_gsim = rlzs_assoc.get_rlzs_by_gsim(0)
min_iml = calc.fix_minimum_intensity(oq.minimum_intensity, imts)
sitecol = dstore['sitecol'].complete
getter = GmfGetter(rlzs_by_gsim, [ebr], sitecol, oq, min_iml)
getter.init()
eids = (numpy.concatenate([
eids for eids in ebr.get_eids_by_rlz(rlzs_by_gsim).values()]))
sids = getter.computers[0].sids
hazardr = getter.get_hazard()
rlzs = rlzs_assoc.realizations
fields = ['eid-%03d' % eid for eid in eids]
dt = numpy.dtype([(f, F32) for f in fields])
mesh = numpy.zeros(len(sids), [('lon', F64), ('lat', F64)])
mesh['lon'] = sitecol.lons[sids]
mesh['lat'] = sitecol.lats[sids]
writer = writers.CsvWriter(fmt='%.5f')
for rlzi in range(len(rlzs)):
hazard = hazardr[rlzi]
for imti, imt in enumerate(imts):
gmfs = numpy.zeros(len(sids), dt)
for s, sid in enumerate(sids):
for rec in hazard[sid]:
event = 'eid-%03d' % rec['eid']
gmfs[s][event] = rec['gmv'][imti]
dest = dstore.build_fname(
'gmf', 'rup-%s-rlz-%s-%s' % (ebr.serial, rlzi, imt), 'csv')
data = util.compose_arrays(mesh, gmfs)
writer.save(data, dest)
return writer.getsaved()
def export_ruptures_xml(ekey, dstore):
"""
:param ekey: export key, i.e. a pair (datastore key, fmt)
:param dstore: datastore object
"""
fmt = ekey[-1]
oq = dstore['oqparam']
num_ses = oq.ses_per_logic_tree_path
mesh = get_mesh(dstore['sitecol'])
ruptures_by_grp = {}
for rgetter in get_rupture_getters(dstore):
ebrs = [ebr.export(mesh, rgetter.rlzs_by_gsim, num_ses)
for ebr in rgetter]
if ebrs:
ruptures_by_grp[rgetter.grp_id] = ebrs
dest = dstore.export_path('ses.' + fmt)
writer = hazard_writers.SESXMLWriter(dest)
writer.serialize(ruptures_by_grp, oq.investigation_time)
return [dest]
def export_ruptures_xml(ekey, dstore):
"""
:param ekey: export key, i.e. a pair (datastore key, fmt)
:param dstore: datastore object
"""
fmt = ekey[-1]
oq = dstore['oqparam']
events = group_array(dstore['events'][()], 'rup_id')
ruptures_by_grp = AccumDict(accum=[])
for rgetter in get_rupture_getters(dstore):
ebrs = []
for proxy in rgetter.get_proxies():
events_by_ses = group_array(events[proxy['id']], 'ses_id')
ebr = proxy.to_ebr(rgetter.trt)
ebrs.append(ebr.export(events_by_ses))
ruptures_by_grp[rgetter.trt_smr].extend(ebrs)
dest = dstore.export_path('ses.' + fmt)
writer = hazard_writers.SESXMLWriter(dest)
writer.serialize(ruptures_by_grp, oq.investigation_time)
return [dest]
def get_rupture_getters(self):
"""
:returns: a list of RuptureGetters
"""
dstore = (self.datastore.parent
if self.datastore.parent else self.datastore)
hdf5cache = dstore.hdf5cache()
with hdf5.File(hdf5cache, 'r+') as cache:
if 'rupgeoms' not in cache:
dstore.hdf5.copy('rupgeoms', cache)
rgetters = get_rupture_getters(dstore,
split=self.oqparam.concurrent_tasks,
hdf5cache=hdf5cache)
num_events = self.E if hasattr(self, 'E') else len(dstore['events'])
num_ruptures = len(dstore['ruptures'])
logging.info('Found {:,d} ruptures and {:,d} events'.format(
num_ruptures, num_events))
if self.datastore.parent:
self.datastore.parent.close()
return rgetters
def run_calc(self):
"""
Run a calculation and return results (reinvented from openquake.calculators.base)
"""
with self.calculator._monitor:
self.calculator._monitor.username = ''
try:
# Pre-execute setups
self.calculator.pre_execute()
#self.calculator.datastore.swmr_on()
oq = self.calculator.oqparam
dstore = self.calculator.datastore
self.calculator.set_param()
self.calculator.offset = 0
# Source model
print('self.__dict__ = ')
print(self.calculator.__dict__)
if oq.hazard_calculation_id: # from ruptures
dstore.parent = self.calculator.datastore.read(
oq.hazard_calculation_id)
elif hasattr(self.calculator, 'csm'): # from sources
self.calculator_build_events_from_sources()
#self.calculator.build_events_from_sources()
if (oq.ground_motion_fields is False
and oq.hazard_curves_from_gmfs is False):
return {}
elif 'rupture_model' not in oq.inputs:
logging.warning(
'There is no rupture_model, the calculator will just '
'import data without performing any calculation')
fake = logictree.FullLogicTree.fake()
dstore['full_lt'] = fake # needed to expose the outputs
dstore['weights'] = [1.]
return {}
else: # scenario
self.calculator._read_scenario_ruptures()
if (oq.ground_motion_fields is False
and oq.hazard_curves_from_gmfs is False):
return {}
# Intensity measure models
if oq.ground_motion_fields:
imts = oq.get_primary_imtls()
nrups = len(dstore['ruptures'])
base.create_gmf_data(dstore, imts, oq.get_sec_imts())
dstore.create_dset('gmf_data/sigma_epsilon',
getters.sig_eps_dt(oq.imtls))
dstore.create_dset('gmf_data/time_by_rup',
getters.time_dt, (nrups, ),
fillvalue=None)
# Prepare inputs for GmfGetter
nr = len(dstore['ruptures'])
logging.info('Reading {:_d} ruptures'.format(nr))
rgetters = getters.get_rupture_getters(
dstore,
oq.concurrent_tasks * 1.25,
srcfilter=self.calculator.srcfilter)
args = [(rgetter, self.calculator.param)
for rgetter in rgetters]
mon = performance.Monitor()
mon.version = version
mon.config = config
rcvr = 'tcp://%s:%s' % (config.dbserver.listen,
config.dbserver.receiver_ports)
skt = zeromq.Socket(rcvr, zeromq.zmq.PULL, 'bind').__enter__()
mon.backurl = 'tcp://%s:%s' % (config.dbserver.host, skt.port)
mon = mon.new(operation='total ' +
self.calculator.core_task.__func__.__name__,
measuremem=True)
mon.weight = getattr(args[0], 'weight',
1.) # used in task_info
mon.task_no = 1 # initialize the task number
args += (mon, )
self.args = args
self.mon = mon
self.dstore = dstore
finally:
print('FetchOpenQuake: OpenQuake Hazard Calculator defined.')