def test_missing_line(self):
self.oqparam.inputs['gmfs'] = general.writetmp('''\
col=00|ses=0001|src=test|rup=001-00,0 1,3.05128000E-01 6.04032000E-01
col=00|ses=0001|src=test|rup=001-01,0 1,2.67031000E-01 3.34878000E-01
''')
with self.assertRaises(readinput.InvalidFile):
readinput.get_gmfs(self.oqparam, self.sitecol)
def test_negative_indices(self):
self.oqparam.inputs['gmfs'] = general.writetmp('''\
col=00|ses=0001|src=test|rup=001-00,0 -1,1.59434000E-01 3.92602000E-01
col=00|ses=0001|src=test|rup=001-01,0 1,3.05128000E-01 6.04032000E-01
col=00|ses=0001|src=test|rup=001-02,0 1,2.67031000E-01 3.34878000E-01
''')
with self.assertRaises(readinput.InvalidFile):
readinput.get_gmfs(self.oqparam, self.sitecol)
def save_gmfs(calculator):
"""
:param calculator: a scenario_risk/damage or event_based_risk calculator
:returns: a pair (eids, R) where R is the number of realizations
"""
dstore = calculator.datastore
oq = calculator.oqparam
logging.info('Reading gmfs from file')
if oq.inputs['gmfs'].endswith('.csv'):
eids, num_rlzs, calculator.gmdata = import_gmfs(
dstore, oq.inputs['gmfs'], calculator.sitecol.complete.sids)
save_gmdata(calculator, calculator.R)
else: # XML
eids, gmfs = readinput.get_gmfs(oq)
E = len(eids)
calculator.eids = eids
if hasattr(oq, 'number_of_ground_motion_fields'):
if oq.number_of_ground_motion_fields != E:
raise RuntimeError('Expected %d ground motion fields, found %d' %
(oq.number_of_ground_motion_fields, E))
else: # set the number of GMFs from the file
oq.number_of_ground_motion_fields = E
# NB: save_gmfs redefine oq.sites in case of GMFs from XML or CSV
if oq.inputs['gmfs'].endswith('.xml'):
haz_sitecol = readinput.get_site_collection(oq)
R, N, E, I = gmfs.shape
idx = (slice(None)
if haz_sitecol.indices is None else haz_sitecol.indices)
save_gmf_data(dstore, haz_sitecol, gmfs[:, idx], eids)
def test_missing_indices_are_ok(self):
self.oqparam.inputs['gmfs'] = general.writetmp('''\
col=00|ses=0001|src=test|rup=001-00,,1.59434000E-01 3.92602000E-01
col=00|ses=0001|src=test|rup=001-01,0 1,3.05128000E-01 6.04032000E-01
col=00|ses=0001|src=test|rup=001-02,0,2.67031000E-01
''')
gmfs = readinput.get_gmfs(self.oqparam, self.sitecol)
gmvs1, gmvs2 = gmfs['PGA']
assert_allclose(gmvs1, [0.159434, 0.305128, 0.267031])
assert_allclose(gmvs2, [0.392602, 0.604032, 0.])
def get_gmfs(calculator):
"""
:param calculator: a scenario_risk/damage or gmf_ebrisk calculator
:returns: a pair (eids, R) where R is the number of realizations
"""
dstore = calculator.datastore
oq = calculator.oqparam
sitecol = calculator.sitecol
if dstore.parent:
haz_sitecol = dstore.parent['sitecol'] # S sites
else:
haz_sitecol = sitecol # N sites
N = len(haz_sitecol.complete)
I = len(oq.imtls)
if 'gmfs' in oq.inputs: # from file
logging.info('Reading gmfs from file')
eids, gmfs = readinput.get_gmfs(oq)
E = len(eids)
if hasattr(oq, 'number_of_ground_motion_fields'):
if oq.number_of_ground_motion_fields != E:
raise RuntimeError(
'Expected %d ground motion fields, found %d' %
(oq.number_of_ground_motion_fields, E))
else: # set the number of GMFs from the file
oq.number_of_ground_motion_fields = E
# NB: get_gmfs redefine oq.sites in case of GMFs from XML or CSV
haz_sitecol = readinput.get_site_collection(oq) or haz_sitecol
calculator.assoc_assets(haz_sitecol)
R, N, E, I = gmfs.shape
idx = (slice(None)
if haz_sitecol.indices is None else haz_sitecol.indices)
save_gmf_data(dstore, haz_sitecol, gmfs[:, idx])
# store the events, useful when read the GMFs from a file
events = numpy.zeros(E, readinput.stored_event_dt)
events['eid'] = eids
dstore['events'] = events
return eids, len(gmfs)
elif calculator.precalc: # from previous step
num_assocs = dstore['csm_info'].get_num_rlzs()
E = oq.number_of_ground_motion_fields
eids = numpy.arange(E)
gmfs = numpy.zeros((num_assocs, N, E, I))
for g, gsim in enumerate(calculator.precalc.gsims):
gmfs[g, sitecol.sids] = calculator.precalc.gmfa[gsim]
return eids, len(gmfs)
else: # with --hc option
return (calculator.datastore['events']['eid'],
calculator.datastore['csm_info'].get_num_rlzs())
def get_gmfs(dstore, precalc=None):
"""
:param dstore: a datastore
:param precalc: a scenario calculator with attribute .gmfa
:returns: a dictionary grp_id, gsid -> gmfa
"""
oq = dstore['oqparam']
if 'gmfs' in oq.inputs: # from file
logging.info('Reading gmfs from file')
sitecol, etags, gmfs_by_imt = readinput.get_gmfs(oq)
# reduce the gmfs matrices to the filtered sites
for imt in oq.imtls:
gmfs_by_imt[imt] = gmfs_by_imt[imt][sitecol.indices]
logging.info('Preparing the risk input')
return etags, [gmfs_by_imt]
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
rlzs = rlzs_assoc.realizations
sitecol = dstore['sitecol']
# NB: if the hazard site collection has N sites, the hazard
# filtered site collection for the nonzero GMFs has N' <= N sites
# whereas the risk site collection associated to the assets
# has N'' <= N' sites
if dstore.parent:
haz_sitecol = dstore.parent['sitecol'] # N' values
else:
haz_sitecol = sitecol
risk_indices = set(sitecol.indices) # N'' values
N = len(haz_sitecol.complete)
imt_dt = numpy.dtype([(str(imt), F32) for imt in oq.imtls])
E = oq.number_of_ground_motion_fields
etags = numpy.array(sorted(b'scenario-%010d~ses=1' % i for i in range(E)))
gmfs = numpy.zeros((len(rlzs_assoc), N, E), imt_dt)
if precalc:
for i, gsim in enumerate(precalc.gsims):
for imti, imt in enumerate(oq.imtls):
gmfs[imt][i, sitecol.sids] = precalc.gmfa[gsim][imti]
return etags, gmfs
# else read from the datastore
for i, rlz in enumerate(rlzs):
data = group_array(dstore['gmf_data/sm-0000/%04d' % i], 'sid')
for sid, array in data.items():
if sid in risk_indices:
for imti, imt in enumerate(oq.imtls):
a = get_array(array, imti=imti)
gmfs[imt][i, sid, a['eid']] = a['gmv']
return etags, gmfs
def get_gmfs(dstore):
"""
:param dstore: a datastore
:returns: a dictionary grp_id, gsid -> gmfa
"""
oq = dstore['oqparam']
if 'gmfs' in oq.inputs: # from file
logging.info('Reading gmfs from file')
sitecol, etags, gmfs_by_imt = readinput.get_gmfs(oq)
# reduce the gmfs matrices to the filtered sites
for imt in oq.imtls:
gmfs_by_imt[imt] = gmfs_by_imt[imt][sitecol.indices]
logging.info('Preparing the risk input')
return etags, {(0, 'FromFile'): gmfs_by_imt}
# else from datastore
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
rlzs = rlzs_assoc.realizations
sitecol = dstore['sitecol']
# NB: if the hazard site collection has N sites, the hazard
# filtered site collection for the nonzero GMFs has N' <= N sites
# whereas the risk site collection associated to the assets
# has N'' <= N' sites
if dstore.parent:
haz_sitecol = dstore.parent['sitecol'] # N' values
else:
haz_sitecol = sitecol
risk_indices = set(sitecol.indices) # N'' values
N = len(haz_sitecol.complete)
imt_dt = dtype((imt, F32) for imt in oq.imtls)
E = oq.number_of_ground_motion_fields
# build a matrix N x E for each GSIM realization
gmfs = {(grp_id, gsim): numpy.zeros((N, E), imt_dt)
for grp_id, gsim in rlzs_assoc}
for i, rlz in enumerate(rlzs):
data = group_array(dstore['gmf_data/%04d' % i], 'sid')
for sid, array in data.items():
if sid in risk_indices:
for imti, imt in enumerate(oq.imtls):
a = get_array(array, imti=imti)
gs = str(rlz.gsim_rlz)
gmfs[0, gs][imt][sid, a['eid']] = a['gmv']
etags = numpy.array(
sorted([b'scenario-%010d~ses=1' % i
for i in range(oq.number_of_ground_motion_fields)]))
return etags, gmfs
def get_gmfs(calc):
"""
:param calc: a ScenarioDamage or ScenarioRisk calculator
:returns: a dictionary of gmfs
"""
if 'gmfs' in calc.oqparam.inputs: # from file
logging.info('Reading gmfs from file')
sitecol, calc.tags, gmfs_by_imt = readinput.get_gmfs(calc.oqparam)
calc.save_params() # save number_of_ground_motion_fields and sites
# reduce the gmfs matrices to the filtered sites
for imt in calc.oqparam.imtls:
gmfs_by_imt[imt] = gmfs_by_imt[imt][sitecol.indices]
logging.info('Preparing the risk input')
fake_rlz = logictree.Realization(
value=('FromFile',), weight=1, lt_path=('',),
ordinal=0, lt_uid=('*',))
calc.rlzs_assoc = logictree.RlzsAssoc([fake_rlz])
return sitecol, {(0, 'FromFile'): gmfs_by_imt}
# else from rupture
gmf = calc.datastore['gmfs/col00'].value
# NB: if the hazard site collection has N sites, the hazard
# filtered site collection for the nonzero GMFs has N' <= N sites
# whereas the risk site collection associated to the assets
# has N'' <= N' sites
if calc.datastore.parent:
haz_sitecol = calc.datastore.parent['sitecol'] # N' values
else:
haz_sitecol = calc.sitecol
risk_indices = set(calc.sitecol.indices) # N'' values
N = len(haz_sitecol.complete)
imt_dt = numpy.dtype([(imt, float) for imt in calc.oqparam.imtls])
gmf_by_idx = general.groupby(gmf, lambda row: row['idx'])
R = len(gmf_by_idx)
# build a matrix N x R for each GSIM realization
gmfs = {(trt_id, gsim): numpy.zeros((N, R), imt_dt)
for trt_id, gsim in calc.rlzs_assoc}
for rupid, rows in sorted(gmf_by_idx.items()):
assert len(haz_sitecol.indices) == len(rows), (
len(haz_sitecol.indices), len(rows))
for sid, gmv in zip(haz_sitecol.indices, rows):
if sid in risk_indices:
for trt_id, gsim in gmfs:
gmfs[trt_id, gsim][sid, rupid] = gmv[gsim]
return haz_sitecol, gmfs
def get_gmfs(dstore):
"""
:param dstore: a datastore
:returns: a dictionary trt_id, gsid -> gmfa
"""
oq = dstore['oqparam']
if 'gmfs' in oq.inputs: # from file
logging.info('Reading gmfs from file')
sitecol, etags, gmfs_by_imt = readinput.get_gmfs(oq)
# reduce the gmfs matrices to the filtered sites
for imt in oq.imtls:
gmfs_by_imt[imt] = gmfs_by_imt[imt][sitecol.indices]
logging.info('Preparing the risk input')
return etags, {(0, 'FromFile'): gmfs_by_imt}
# else from datastore
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
rlzs = rlzs_assoc.realizations
sitecol = dstore['sitecol']
# NB: if the hazard site collection has N sites, the hazard
# filtered site collection for the nonzero GMFs has N' <= N sites
# whereas the risk site collection associated to the assets
# has N'' <= N' sites
if dstore.parent:
haz_sitecol = dstore.parent['sitecol'] # N' values
else:
haz_sitecol = sitecol
risk_indices = set(sitecol.indices) # N'' values
N = len(haz_sitecol.complete)
imt_dt = numpy.dtype([(bytes(imt), F32) for imt in oq.imtls])
E = oq.number_of_ground_motion_fields
# build a matrix N x E for each GSIM realization
gmfs = {(trt_id, gsim): numpy.zeros((N, E), imt_dt)
for trt_id, gsim in rlzs_assoc}
for i, rlz in enumerate(rlzs):
data = group_array(dstore['gmf_data/%04d' % i], 'sid')
for sid, array in data.items():
if sid in risk_indices:
for imti, imt in enumerate(oq.imtls):
a = get_array(array, imti=imti)
gs = str(rlz.gsim_rlz)
gmfs[0, gs][imt][sid, a['eid']] = a['gmv']
return dstore['etags'].value, gmfs
def get_gmfs(dstore):
"""
:param dstore: a datastore
:returns: a dictionary trt_id, gsid -> gmfa
"""
oq = dstore["oqparam"]
if "gmfs" in oq.inputs: # from file
logging.info("Reading gmfs from file")
sitecol, etags, gmfs_by_imt = readinput.get_gmfs(oq)
# reduce the gmfs matrices to the filtered sites
for imt in oq.imtls:
gmfs_by_imt[imt] = gmfs_by_imt[imt][sitecol.indices]
logging.info("Preparing the risk input")
return etags, {(0, "FromFile"): gmfs_by_imt}
# else from datastore
rlzs_assoc = dstore["csm_info"].get_rlzs_assoc()
rlzs = rlzs_assoc.realizations
sitecol = dstore["sitecol"]
# NB: if the hazard site collection has N sites, the hazard
# filtered site collection for the nonzero GMFs has N' <= N sites
# whereas the risk site collection associated to the assets
# has N'' <= N' sites
if dstore.parent:
haz_sitecol = dstore.parent["sitecol"] # N' values
else:
haz_sitecol = sitecol
risk_indices = set(sitecol.indices) # N'' values
N = len(haz_sitecol.complete)
imt_dt = numpy.dtype([(bytes(imt), F32) for imt in oq.imtls])
E = oq.number_of_ground_motion_fields
# build a matrix N x E for each GSIM realization
gmfs = {(trt_id, gsim): numpy.zeros((N, E), imt_dt) for trt_id, gsim in rlzs_assoc}
for i, rlz in enumerate(rlzs):
data = general.group_array(dstore["gmf_data/%04d" % i], "sid")
for sid, array in data.items():
if sid in risk_indices:
for imti, imt in enumerate(oq.imtls):
a = general.get_array(array, imti=imti)
gs = str(rlz.gsim_rlz)
gmfs[0, gs][imt][sid, a["eid"]] = a["gmv"]
return dstore["etags"].value, gmfs
def get_gmfs(dstore, precalc=None):
"""
:param dstore: a datastore
:param precalc: a scenario calculator with attribute .gmfa
:returns: a pair (eids, gmfs) where gmfs is a matrix of shape (G, N, E, I)
"""
oq = dstore['oqparam']
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
sitecol = dstore['sitecol']
if dstore.parent:
haz_sitecol = dstore.parent['sitecol'] # S sites
else:
haz_sitecol = sitecol # N sites
N = len(haz_sitecol.complete)
I = len(oq.imtls)
E = oq.number_of_ground_motion_fields
eids = numpy.arange(E)
gmfs = numpy.zeros((len(rlzs_assoc), N, E, I))
if precalc:
for g, gsim in enumerate(precalc.gsims):
gmfs[g, sitecol.sids] = precalc.gmfa[gsim]
return eids, gmfs
if 'gmf_data/grp-00' in dstore:
# read from the datastore
dset = dstore['gmf_data/grp-00']
R = len(dstore['realizations'])
nrows = len(dset) // R
for r in range(R):
for s, sid in enumerate(haz_sitecol.sids):
start = r * nrows + E * s
array = dset[start:start + E] # shape (E, I)
if numpy.unique(array['sid']) != [sid]: # sanity check
raise ValueError('The GMFs have been stored incorrectly')
gmfs[r, sid] = array['gmv']
return eids, gmfs
elif 'gmfs' in oq.inputs: # from file
logging.info('Reading gmfs from file')
_sitecol, eids, gmfa = readinput.get_gmfs(oq)
dstore['gmf_data/grp-00'] = get_gmv_data(haz_sitecol.sids,
[gmfa[haz_sitecol.indices]])
return eids, [gmfa]
def get_gmfs(dstore, precalc=None):
"""
:param dstore: a datastore
:param precalc: a scenario calculator with attribute .gmfa
:returns: a dictionary grp_id, gsid -> gmfa
"""
oq = dstore['oqparam']
if 'gmfs' in oq.inputs: # from file
logging.info('Reading gmfs from file')
sitecol, etags, gmfa = readinput.get_gmfs(oq)
# reduce the gmfa matrix to the filtered sites
return etags, [gmfa[sitecol.indices]]
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
sitecol = dstore['sitecol']
if dstore.parent:
haz_sitecol = dstore.parent['sitecol'] # S sites
else:
haz_sitecol = sitecol # N sites
S = len(haz_sitecol)
N = len(haz_sitecol.complete)
I = len(oq.imtls)
E = oq.number_of_ground_motion_fields
etags = numpy.arange(E)
gmfs = numpy.zeros((len(rlzs_assoc), N, I, E))
if precalc:
for g, gsim in enumerate(precalc.gsims):
gmfs[g, sitecol.sids] = precalc.gmfa[gsim]
return etags, gmfs
# else read from the datastore
gsims = sorted(dstore['gmf_data/grp-00'])
imtis = range(len(oq.imtls))
for i, gsim in enumerate(gsims):
dset = dstore['gmf_data/grp-00/' + gsim]
for s, sid in enumerate(haz_sitecol.sids):
for imti in imtis:
idx = E * (S * imti + s)
array = dset[idx:idx + E]
if numpy.unique(array['sid']) != [sid]: # sanity check
raise ValueError('The GMFs have been stored incorrectly')
gmfs[i, sid, imti] = array['gmv']
return etags, gmfs
def get_gmfs(dstore):
"""
:param dstore: a datastore
:returns: a dictionary of gmfs
"""
oq = dstore['oqparam']
if 'gmfs' in oq.inputs: # from file
logging.info('Reading gmfs from file')
sitecol, etags, gmfs_by_imt = readinput.get_gmfs(oq)
# reduce the gmfs matrices to the filtered sites
for imt in oq.imtls:
gmfs_by_imt[imt] = gmfs_by_imt[imt][sitecol.indices]
logging.info('Preparing the risk input')
return etags, {(0, 'FromFile'): gmfs_by_imt}
# else from rupture
sitecol = dstore['sitecol']
gmfa = dstore['gmf_data/1'].value
# NB: if the hazard site collection has N sites, the hazard
# filtered site collection for the nonzero GMFs has N' <= N sites
# whereas the risk site collection associated to the assets
# has N'' <= N' sites
if dstore.parent:
haz_sitecol = dstore.parent['sitecol'] # N' values
else:
haz_sitecol = sitecol
risk_indices = set(sitecol.indices) # N'' values
N = len(haz_sitecol.complete)
imt_dt = numpy.dtype([(bytes(imt), F32) for imt in oq.imtls])
E = gmfa.shape[0]
# build a matrix N x E for each GSIM realization
gmfs = {(trt_id, gsim): numpy.zeros((N, E), imt_dt)
for trt_id, gsim in dstore['rlzs_assoc']}
for eid, gmf in enumerate(gmfa):
assert len(haz_sitecol.indices) == len(gmf), (
len(haz_sitecol.indices), len(gmf))
for sid, gmv in zip(haz_sitecol.indices, gmf):
if sid in risk_indices:
for trt_id, gsim in gmfs:
gmfs[trt_id, gsim][sid, eid] = gmv[gsim]
return dstore['etags'].value, gmfs
def read_gmfs_from_csv(calc):
"""
:param calc: a ScenarioDamage or ScenarioRisk calculator
:returns: riskinputs
"""
logging.info('Reading hazard curves from CSV')
gmfs_by_imt = readinput.get_gmfs(calc.oqparam, calc.sitecol.complete)
# reduce the gmfs matrices to the filtered sites
for imt in calc.oqparam.imtls:
gmfs_by_imt[imt] = gmfs_by_imt[imt][calc.sitecol.indices]
num_assets = calc.count_assets()
num_sites = len(calc.sitecol)
logging.info('Associated %d assets to %d sites', num_assets, num_sites)
logging.info('Preparing the risk input')
fake_rlz = logictree.Realization(
value=('FromCsv',), weight=1, lt_path=('',),
ordinal=0, lt_uid=('*',))
calc.rlzs_assoc = logictree.RlzsAssoc([fake_rlz])
return {(0, 'FromCsv'): gmfs_by_imt}
def read_gmfs_from_csv(calc):
"""
:param calc: a ScenarioDamage or ScenarioRisk calculator
:returns: riskinputs
"""
logging.info('Reading hazard curves from CSV')
gmfs_by_imt = readinput.get_gmfs(calc.oqparam, calc.sitecol.complete)
# reduce the gmfs matrices to the filtered sites
for imt in calc.oqparam.imtls:
gmfs_by_imt[imt] = gmfs_by_imt[imt][calc.sitecol.indices]
num_assets = calc.count_assets()
num_sites = len(calc.sitecol)
logging.info('Associated %d assets to %d sites', num_assets, num_sites)
logging.info('Preparing the risk input')
fake_rlz = logictree.Realization(value=('FromCsv', ),
weight=1,
lt_path=('', ),
ordinal=0,
lt_uid=('*', ))
calc.rlzs_assoc = logictree.RlzsAssoc([fake_rlz])
return {(0, 'FromCsv'): gmfs_by_imt}
def read_gmfs_from_file(calc):
"""
:param calc: a ScenarioDamage or ScenarioRisk calculator
:returns: riskinputs
"""
logging.info('Reading gmfs from file')
try:
sitecol = calc.sitecol.complete
except KeyError:
sitecol = None
calc.sitecol, calc.tags, gmfs_by_imt = readinput.get_gmfs(
calc.oqparam, sitecol)
calc.save_params() # save number_of_ground_motion_fields and sites
# reduce the gmfs matrices to the filtered sites
for imt in calc.oqparam.imtls:
gmfs_by_imt[imt] = gmfs_by_imt[imt][calc.sitecol.indices]
logging.info('Preparing the risk input')
fake_rlz = logictree.Realization(
value=('FromFile',), weight=1, lt_path=('',),
ordinal=0, lt_uid=('*',))
calc.rlzs_assoc = logictree.RlzsAssoc([fake_rlz])
return {(0, 'FromFile'): gmfs_by_imt}
