def build_hazard(pgetter, N, hstats, individual_curves, max_sites_disagg,
amplifier, monitor):
"""
:param pgetter: an :class:`openquake.commonlib.getters.PmapGetter`
:param N: the total number of sites
:param hstats: a list of pairs (statname, statfunc)
:param individual_curves: if True, also build the individual curves
:param max_sites_disagg: if there are less sites than this, store rup info
:param amplifier: instance of Amplifier or None
:param monitor: instance of Monitor
:returns: a dictionary kind -> ProbabilityMap
The "kind" is a string of the form 'rlz-XXX' or 'mean' of 'quantile-XXX'
used to specify the kind of output.
"""
with monitor('read PoEs'):
pgetter.init()
if amplifier:
ampcode = pgetter.dstore['sitecol'].ampcode
imtls, poes, weights = pgetter.imtls, pgetter.poes, pgetter.weights
M = len(imtls)
P = len(poes)
L = len(imtls.array) if amplifier is None else len(amplifier.amplevels) * M
R = len(weights)
S = len(hstats)
pmap_by_kind = {}
if R > 1 and individual_curves or not hstats:
pmap_by_kind['hcurves-rlzs'] = [ProbabilityMap(L) for r in range(R)]
if hstats:
pmap_by_kind['hcurves-stats'] = [ProbabilityMap(L) for r in range(S)]
if poes:
pmap_by_kind['hmaps-stats'] = [
ProbabilityMap(M, P) for r in range(S)
]
combine_mon = monitor('combine pmaps', measuremem=False)
compute_mon = monitor('compute stats', measuremem=False)
for sid in pgetter.sids:
with combine_mon:
pcurves = pgetter.get_pcurves(sid)
if amplifier:
pcurves = amplifier.amplify(ampcode[sid], pcurves)
if sum(pc.array.sum() for pc in pcurves) == 0: # no data
continue
with compute_mon:
if hstats:
arr = numpy.array([pc.array for pc in pcurves])
for s, (statname, stat) in enumerate(hstats.items()):
pc = getters.build_stat_curve(arr, imtls, stat, weights)
pmap_by_kind['hcurves-stats'][s][sid] = pc
if poes:
hmap = calc.make_hmap(pc, pgetter.imtls, poes, sid)
pmap_by_kind['hmaps-stats'][s].update(hmap)
if R > 1 and individual_curves or not hstats:
for pmap, pc in zip(pmap_by_kind['hcurves-rlzs'], pcurves):
pmap[sid] = pc
if poes:
pmap_by_kind['hmaps-rlzs'] = [
calc.make_hmap(pc, imtls, poes, sid) for pc in pcurves
]
return pmap_by_kind
def build_hazard_stats(pgetter, N, hstats, individual_curves, monitor):
"""
:param pgetter: an :class:`openquake.commonlib.getters.PmapGetter`
:param N: the total number of sites
:param hstats: a list of pairs (statname, statfunc)
:param individual_curves: if True, also build the individual curves
:param monitor: instance of Monitor
:returns: a dictionary kind -> ProbabilityMap
The "kind" is a string of the form 'rlz-XXX' or 'mean' of 'quantile-XXX'
used to specify the kind of output.
"""
with monitor('read PoEs'):
pgetter.init()
imtls, poes, weights = pgetter.imtls, pgetter.poes, pgetter.weights
L = len(imtls.array)
R = len(weights)
S = len(hstats)
pmap_by_kind = {'rlz_by_sid': {}}
if R > 1 and individual_curves or not hstats:
pmap_by_kind['hcurves-rlzs'] = [ProbabilityMap(L) for r in range(R)]
if hstats:
pmap_by_kind['hcurves-stats'] = [ProbabilityMap(L) for r in range(S)]
if poes:
pmap_by_kind['hmaps-stats'] = [ProbabilityMap(L) for r in range(S)]
combine_mon = monitor('combine pmaps', measuremem=False)
compute_mon = monitor('compute stats', measuremem=False)
for sid in pgetter.sids:
with combine_mon:
pcurves = pgetter.get_pcurves(sid)
if sum(pc.array.sum() for pc in pcurves) == 0: # no data
continue
with compute_mon:
if hstats:
arr = numpy.array([pc.array for pc in pcurves])
for s, (statname, stat) in enumerate(hstats.items()):
pc = _build_stat_curve(arr, imtls, stat, weights)
pmap_by_kind['hcurves-stats'][s][sid] = pc
if poes:
hmap = calc.make_hmap(pc, pgetter.imtls, poes, sid)
pmap_by_kind['hmaps-stats'][s].update(hmap)
if statname == 'mean' and R > 1 and N <= FEWSITES:
rlz = pmap_by_kind['rlz_by_sid']
rlz[sid] = util.closest_to_ref(
[p.array for p in pcurves], pc.array)['rlz']
if R > 1 and individual_curves or not hstats:
for pmap, pc in zip(pmap_by_kind['hcurves-rlzs'], pcurves):
pmap[sid] = pc
if poes:
pmap_by_kind['hmaps-rlzs'] = [
calc.make_hmap(pc, imtls, poes, sid) for pc in pcurves
]
return pmap_by_kind
def build_hazard_stats(pgetter, N, hstats, individual_curves, monitor):
"""
:param pgetter: an :class:`openquake.commonlib.getters.PmapGetter`
:param N: the total number of sites
:param hstats: a list of pairs (statname, statfunc)
:param individual_curves: if True, also build the individual curves
:param monitor: instance of Monitor
:returns: a dictionary kind -> ProbabilityMap
The "kind" is a string of the form 'rlz-XXX' or 'mean' of 'quantile-XXX'
used to specify the kind of output.
"""
with monitor('combine pmaps'):
pgetter.init() # if not already initialized
try:
pmaps = pgetter.get_pmaps()
except IndexError: # no data
return {}
if sum(len(pmap) for pmap in pmaps) == 0: # no data
return {}
R = len(pmaps)
imtls, poes, weights = pgetter.imtls, pgetter.poes, pgetter.weights
pmap_by_kind = {}
hmaps_stats = []
hcurves_stats = []
with monitor('compute stats'):
for statname, stat in hstats.items():
pmap = compute_pmap_stats(pmaps, [stat], weights, imtls)
hcurves_stats.append(pmap)
if pgetter.poes:
hmaps_stats.append(
calc.make_hmap(pmap, pgetter.imtls, pgetter.poes))
if statname == 'mean' and R > 1 and N <= FEWSITES:
pmap_by_kind['rlz_by_sid'] = rlz = {}
for sid, pcurve in pmap.items():
rlz[sid] = util.closest_to_ref(
[pm.setdefault(sid, 0).array for pm in pmaps],
pcurve.array)['rlz']
if hcurves_stats:
pmap_by_kind['hcurves-stats'] = hcurves_stats
if hmaps_stats:
pmap_by_kind['hmaps-stats'] = hmaps_stats
if R > 1 and individual_curves or not hstats:
pmap_by_kind['hcurves-rlzs'] = pmaps
if pgetter.poes:
with monitor('build individual hmaps'):
pmap_by_kind['hmaps-rlzs'] = [
calc.make_hmap(pmap, imtls, poes) for pmap in pmaps
]
return pmap_by_kind
def build_hazard_stats(pgetter, N, hstats, individual_curves, monitor):
"""
:param pgetter: an :class:`openquake.commonlib.getters.PmapGetter`
:param N: the total number of sites
:param hstats: a list of pairs (statname, statfunc)
:param individual_curves: if True, also build the individual curves
:param monitor: instance of Monitor
:returns: a dictionary kind -> ProbabilityMap
The "kind" is a string of the form 'rlz-XXX' or 'mean' of 'quantile-XXX'
used to specify the kind of output.
"""
with monitor('combine pmaps'):
pgetter.init() # if not already initialized
try:
pmaps = pgetter.get_pmaps()
except IndexError: # no data
return {}
if sum(len(pmap) for pmap in pmaps) == 0: # no data
return {}
R = len(pmaps)
imtls, poes, weights = pgetter.imtls, pgetter.poes, pgetter.weights
pmap_by_kind = {}
hmaps_stats = []
hcurves_stats = []
with monitor('compute stats'):
for statname, stat in hstats.items():
pmap = compute_pmap_stats(pmaps, [stat], weights, imtls)
hcurves_stats.append(pmap)
if pgetter.poes:
hmaps_stats.append(
calc.make_hmap(pmap, pgetter.imtls, pgetter.poes))
if statname == 'mean' and R > 1 and N <= FEWSITES:
pmap_by_kind['rlz_by_sid'] = rlz = {}
for sid, pcurve in pmap.items():
rlz[sid] = util.closest_to_ref(
[pm.setdefault(sid, 0).array for pm in pmaps],
pcurve.array)['rlz']
if hcurves_stats:
pmap_by_kind['hcurves-stats'] = hcurves_stats
if hmaps_stats:
pmap_by_kind['hmaps-stats'] = hmaps_stats
if R > 1 and individual_curves or not hstats:
pmap_by_kind['hcurves-rlzs'] = pmaps
if pgetter.poes:
with monitor('build individual hmaps'):
pmap_by_kind['hmaps-rlzs'] = [
calc.make_hmap(pmap, imtls, poes) for pmap in pmaps]
return pmap_by_kind
def extract_hazard(dstore, what):
"""
Extracts hazard curves and possibly hazard maps and/or uniform hazard
spectra. Use it as /extract/hazard/mean or /extract/hazard/rlz-0, etc
"""
oq = dstore['oqparam']
sitecol = dstore['sitecol']
yield 'sitecol', sitecol
yield 'oqparam', oq
yield 'imtls', oq.imtls
yield 'realizations', dstore['csm_info'].rlzs
yield 'checksum32', dstore['/'].attrs['checksum32']
nsites = len(sitecol)
M = len(oq.imtls)
P = len(oq.poes)
for kind, pmap in getters.PmapGetter(dstore).items(what):
for imt in oq.imtls:
key = 'hcurves/%s/%s' % (imt, kind)
arr = numpy.zeros((nsites, len(oq.imtls[imt])))
for sid in pmap:
arr[sid] = pmap[sid].array[oq.imtls.slicedic[imt], 0]
logging.info('extracting %s', key)
yield key, arr
if oq.poes:
hmap = calc.make_hmap(pmap, oq.imtls, oq.poes)
for p, poe in enumerate(oq.poes):
key = 'hmaps/poe-%s/%s' % (poe, kind)
arr = numpy.zeros((nsites, M))
idx = [m * P + p for m in range(M)]
for sid in pmap:
arr[sid] = hmap[sid].array[idx, 0]
logging.info('extracting %s', key)
yield key, arr
def export_hmaps_xml_json(ekey, dstore):
key, kind, fmt = get_kkf(ekey)
oq = dstore['oqparam']
sitecol = dstore['sitecol']
sitemesh = get_mesh(sitecol)
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
fnames = []
writercls = hazard_writers.HazardMapXMLWriter
pdic = DictArray({imt: oq.poes for imt in oq.imtls})
nsites = len(sitemesh)
for kind, hcurves in PmapGetter(dstore).items():
hmaps = calc.make_hmap(
hcurves, oq.imtls, oq.poes).convert(pdic, nsites)
if kind.startswith('rlz-'):
rlz = rlzs_assoc.realizations[int(kind[4:])]
smlt_path = '_'.join(rlz.sm_lt_path)
gsimlt_path = rlz.gsim_rlz.uid
else:
smlt_path = ''
gsimlt_path = ''
for imt in oq.imtls:
for j, poe in enumerate(oq.poes):
suffix = '-%s-%s' % (poe, imt)
fname = hazard_curve_name(
dstore, ekey, kind + suffix, rlzs_assoc)
data = [HazardMap(site[0], site[1], _extract(hmap, imt, j))
for site, hmap in zip(sitemesh, hmaps)]
writer = writercls(
fname, investigation_time=oq.investigation_time,
imt=imt, poe=poe,
smlt_path=smlt_path, gsimlt_path=gsimlt_path)
writer.serialize(data)
fnames.append(fname)
return sorted(fnames)
def export_hcurves_csv(ekey, dstore):
"""
Exports the hazard curves into several .csv files
:param ekey: export key, i.e. a pair (datastore key, fmt)
:param dstore: datastore object
"""
oq = dstore['oqparam']
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
sitecol = dstore['sitecol']
sitemesh = get_mesh(sitecol)
key, kind, fmt = get_kkf(ekey)
fnames = []
if oq.poes:
pdic = DictArray({imt: oq.poes for imt in oq.imtls})
for kind, hcurves in calc.PmapGetter(dstore).items(kind):
fname = hazard_curve_name(dstore, (key, fmt), kind, rlzs_assoc)
comment = _comment(rlzs_assoc, kind, oq.investigation_time)
if key == 'uhs' and oq.poes and oq.uniform_hazard_spectra:
uhs_curves = calc.make_uhs(hcurves, oq.imtls, oq.poes,
len(sitemesh))
writers.write_csv(fname,
util.compose_arrays(sitemesh, uhs_curves),
comment=comment)
fnames.append(fname)
elif key == 'hmaps' and oq.poes and oq.hazard_maps:
hmap = calc.make_hmap(hcurves, oq.imtls, oq.poes)
fnames.extend(
export_hazard_csv(ekey, fname, sitemesh, hmap, pdic, comment))
elif key == 'hcurves':
fnames.extend(
export_hcurves_by_imt_csv(ekey, kind, rlzs_assoc, fname,
sitecol, hcurves, oq))
return sorted(fnames)
def extract_hazard(dstore, what):
"""
Extracts hazard curves and possibly hazard maps and/or uniform hazard
spectra. Use it as /extract/hazard/mean or /extract/hazard/rlz-0, etc
"""
oq = dstore['oqparam']
sitecol = dstore['sitecol']
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
yield 'sitecol', sitecol
yield 'oqparam', oq
yield 'imtls', oq.imtls
yield 'realizations', dstore['csm_info'].rlzs
yield 'checksum32', dstore['/'].attrs['checksum32']
nsites = len(sitecol)
M = len(oq.imtls)
for statname, pmap in getters.PmapGetter(dstore, rlzs_assoc).items(what):
for imt in oq.imtls:
key = 'hcurves/%s/%s' % (imt, statname)
arr = numpy.zeros((nsites, len(oq.imtls[imt])))
for sid in pmap:
arr[sid] = pmap[sid].array[oq.imtls(imt), 0]
logging.info('extracting %s', key)
yield key, arr
hmap = calc.make_hmap(pmap, oq.imtls, oq.poes)
for p, poe in enumerate(oq.poes):
key = 'hmaps/poe-%s/%s' % (poe, statname)
arr = numpy.zeros((nsites, M))
for sid in pmap:
arr[sid] = hmap[sid].array[:, p]
logging.info('extracting %s', key)
yield key, arr
def build_hazard_stats(pgetter, hstats, monitor):
"""
:param pgetter: an :class:`openquake.commonlib.getters.PmapGetter`
:param hstats: a list of pairs (statname, statfunc)
:param monitor: instance of Monitor
:returns: a dictionary kind -> ProbabilityMap
The "kind" is a string of the form 'rlz-XXX' or 'mean' of 'quantile-XXX'
used to specify the kind of output.
"""
with monitor('combine pmaps'):
pgetter.init() # if not already initialized
try:
pmaps = pgetter.get_pmaps(pgetter.sids)
except IndexError: # no data
return {}
if sum(len(pmap) for pmap in pmaps) == 0: # no data
return {}
pmap_by_kind = {}
for statname, stat in hstats:
with monitor('compute ' + statname):
pmap = compute_pmap_stats(pmaps, [stat], pgetter.weights)
pmap_by_kind['hcurves', statname] = pmap
if pgetter.poes:
pmap_by_kind['hmaps',
statname] = calc.make_hmap(pmap, pgetter.imtls,
pgetter.poes)
return pmap_by_kind
def export_hmaps_npz(ekey, dstore):
oq = dstore['oqparam']
mesh = get_mesh(dstore['sitecol'])
pdic = DictArray({imt: oq.poes for imt in oq.imtls})
fname = dstore.export_path('%s.%s' % ekey)
dic = {}
for kind, hcurves in calc.PmapGetter(dstore).items():
hmap = calc.make_hmap(hcurves, oq.imtls, oq.poes)
dic[kind] = convert_to_array(hmap, mesh, pdic)
savez(fname, **dic)
return [fname]
def export_hmaps_hdf5(ekey, dstore):
oq = dstore['oqparam']
mesh = get_mesh(dstore['sitecol'])
pdic = DictArray({imt: oq.poes for imt in oq.imtls})
fname = dstore.export_path('%s.%s' % ekey)
with hdf5.File(fname, 'w') as f:
for dskey in dstore['hcurves']:
hcurves = dstore['hcurves/%s' % dskey]
hmap = calc.make_hmap(hcurves, oq.imtls, oq.poes)
f['hmaps/%s' % dskey] = convert_to_array(hmap, mesh, pdic)
return [fname]
def export_hmaps_npz(ekey, dstore):
oq = dstore['oqparam']
mesh = get_mesh(dstore['sitecol'])
pdic = DictArray({imt: oq.poes for imt in oq.imtls})
fname = dstore.export_path('%s.%s' % ekey)
dic = {}
for dskey in dstore['hcurves']:
hcurves = dstore['hcurves/%s' % dskey]
hmap = calc.make_hmap(hcurves, oq.imtls, oq.poes)
dic[dskey] = convert_to_array(hmap, mesh, pdic)
savez(fname, **dic)
return [fname]
def plot_hmaps(calc_id):
"""
Mean hazard maps plotter.
"""
dstore = datastore.read(calc_id)
oq = dstore['oqparam']
mean = calc.PmapGetter(dstore).get_mean()
hmaps = calc.make_hmap(mean, oq.imtls, oq.poes)
M, P = len(oq.imtls), len(oq.poes)
array = hmaps.array.reshape(len(hmaps.array), M, P)
plt = make_figure(dstore['sitecol'], oq.imtls, oq.poes, array)
plt.show()
def export_hmaps_np(ekey, dstore):
oq = dstore['oqparam']
sitecol = dstore['sitecol']
mesh = get_mesh(sitecol)
pdic = DictArray({imt: oq.poes for imt in oq.imtls})
fname = dstore.export_path('%s.%s' % ekey)
dic = {}
for kind, hcurves in PmapGetter(dstore).items():
hmap = calc.make_hmap(hcurves, oq.imtls, oq.poes)
dic[kind] = calc.convert_to_array(hmap, len(mesh), pdic)
save_np(fname, dic, mesh, ('vs30', F32, sitecol.vs30),
investigation_time=oq.investigation_time)
return [fname]
def plot_hmaps(calc_id):
"""
Mean hazard maps plotter.
"""
dstore = engine.read(calc_id)
oq = dstore['oqparam']
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
mean = getters.PmapGetter(dstore, rlzs_assoc).get_mean()
hmaps = calc.make_hmap(mean, oq.imtls, oq.poes)
M, P = len(oq.imtls), len(oq.poes)
array = hmaps.array.reshape(len(hmaps.array), M, P)
plt = make_figure(dstore['sitecol'], oq.imtls, oq.poes, array)
plt.show()
def extract_hmaps(dstore, what):
"""
Extracts hazard maps. Use it as /extract/hmaps/mean or
/extract/hmaps/rlz-0, etc
"""
oq = dstore['oqparam']
sitecol = dstore['sitecol']
mesh = get_mesh(sitecol)
pdic = DictArray({imt: oq.poes for imt in oq.imtls})
dic = {}
for kind, hcurves in getters.PmapGetter(dstore).items(what):
hmap = calc.make_hmap(hcurves, oq.imtls, oq.poes)
dic[kind] = calc.convert_to_array(hmap, len(mesh), pdic)
return hazard_items(dic, mesh, investigation_time=oq.investigation_time)
def view_flat_hmaps(token, dstore):
"""
Display the flat hazard maps for the calculation. They are
used for debugging purposes when comparing the results of two
calculations. They are the mean over the sites of the mean hazard
maps.
"""
oq = dstore['oqparam']
assert oq.poes
nsites = len(dstore['sitecol'])
pdic = DictArray({imt: oq.poes for imt in oq.imtls})
mean = getters.PmapGetter(dstore).get_mean()
hmaps = calc.make_hmap(mean, oq.imtls, oq.poes)
array = calc.convert_to_array(hmaps, nsites, pdic)
res = numpy.zeros(1, array.dtype)
for name in array.dtype.names:
res[name] = array[name].mean()
return rst_table(res)
def export_hcurves_csv(ekey, dstore):
"""
Exports the hazard curves into several .csv files
:param ekey: export key, i.e. a pair (datastore key, fmt)
:param dstore: datastore object
"""
oq = dstore['oqparam']
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
sitecol = dstore['sitecol']
sitemesh = get_mesh(sitecol)
key, fmt = ekey
fnames = []
if oq.poes:
pdic = DictArray({imt: oq.poes for imt in oq.imtls})
for kind in sorted(dstore['hcurves']):
hcurves = dstore['hcurves/' + kind]
fname = hazard_curve_name(dstore, ekey, kind, rlzs_assoc)
comment = _comment(rlzs_assoc, kind, oq.investigation_time)
if key == 'uhs':
uhs_curves = calc.make_uhs(
hcurves, oq.imtls, oq.poes, len(sitemesh))
writers.write_csv(
fname, util.compose_arrays(sitemesh, uhs_curves),
comment=comment)
fnames.append(fname)
elif key == 'hmaps':
hmap = calc.make_hmap(hcurves, oq.imtls, oq.poes)
fnames.extend(
export_hazard_csv(ekey, fname, sitemesh, hmap, pdic, comment))
else:
if export.from_db: # called by export_from_db
fnames.extend(
export_hcurves_by_imt_csv(
ekey, kind, rlzs_assoc, fname, sitecol, hcurves, oq))
else: # when exporting directly from the datastore
fnames.extend(
export_hazard_csv(
ekey, fname, sitemesh, hcurves, oq.imtls, comment))
return sorted(fnames)
def view_hmap(token, dstore):
"""
Display the highest 20 points of the mean hazard map. Called as
$ oq show hmap:0.1 # 10% PoE
"""
try:
poe = valid.probability(token.split(':')[1])
except IndexError:
poe = 0.1
try:
mean = dstore['hcurves/mean']
except KeyError: # there is a single realization
mean = dstore['hcurves/rlz-000']
oq = dstore['oqparam']
hmap = calc.make_hmap(mean, oq.imtls, [poe])
items = sorted([(hmap[sid].array.sum(), sid) for sid in hmap])[-20:]
dt = numpy.dtype([('sid', U32)] + [(imt, F32) for imt in oq.imtls])
array = numpy.zeros(len(items), dt)
for i, (maxvalue, sid) in enumerate(reversed(items)):
array[i] = (sid, ) + tuple(hmap[sid].array[:, 0])
return rst_table(array)
def extract_hazard_for_qgis(dstore, what):
"""
Extracts hazard curves and possibly hazard maps and/or uniform hazard
spectra. Use it as /extract/qgis-hazard/rlz-0, etc
"""
oq = dstore['oqparam']
sitecol = dstore['sitecol']
yield 'sitecol', sitecol
yield 'oqparam', oq
yield 'realizations', dstore['csm_info'].rlzs
yield 'checksum32', dstore['/'].attrs['checksum32']
N = len(sitecol)
if oq.poes:
pdic = {imt: oq.poes for imt in oq.imtls}
for kind, hcurves in getters.PmapGetter(dstore).items(what):
logging.info('extracting hazard/%s', kind)
yield 'hcurves-' + kind, calc.convert_to_array(hcurves, N, oq.imtls)
if oq.poes and oq.uniform_hazard_spectra:
yield 'uhs-' + kind, calc.make_uhs(hcurves, oq.imtls, oq.poes, N)
if oq.poes and oq.hazard_maps:
hmaps = calc.make_hmap(hcurves, oq.imtls, oq.poes)
yield 'hmaps-' + kind, calc.convert_to_array(hmaps, N, pdic)
def export_hmaps_xml_json(ekey, dstore):
export_type = ekey[1]
oq = dstore['oqparam']
sitemesh = get_mesh(dstore['sitecol'])
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
fnames = []
writercls = (hazard_writers.HazardMapGeoJSONWriter
if export_type == 'geojson' else
hazard_writers.HazardMapXMLWriter)
pdic = DictArray({imt: oq.poes for imt in oq.imtls})
nsites = len(sitemesh)
for kind in dstore['hcurves']:
hcurves = dstore['hcurves/' + kind]
hmaps = calc.make_hmap(
hcurves, oq.imtls, oq.poes).convert(pdic, nsites)
if kind.startswith('rlz-'):
rlz = rlzs_assoc.realizations[int(kind[4:])]
smlt_path = '_'.join(rlz.sm_lt_path)
gsimlt_path = rlz.gsim_rlz.uid
else:
smlt_path = ''
gsimlt_path = ''
for imt in oq.imtls:
for j, poe in enumerate(oq.poes):
suffix = '-%s-%s' % (poe, imt)
fname = hazard_curve_name(
dstore, ekey, kind + suffix, rlzs_assoc)
data = [HazardMap(site[0], site[1], _extract(hmap, imt, j))
for site, hmap in zip(sitemesh, hmaps)]
writer = writercls(
fname, investigation_time=oq.investigation_time,
imt=imt, poe=poe,
smlt_path=smlt_path, gsimlt_path=gsimlt_path)
writer.serialize(data)
fnames.append(fname)
return sorted(fnames)
def post_execute(self, result):
oq = self.oqparam
if not oq.ground_motion_fields and not oq.hazard_curves_from_gmfs:
return
N = len(self.sitecol.complete)
M = len(oq.imtls)
L = len(oq.imtls.array)
L1 = L // M
if result and oq.hazard_curves_from_gmfs:
rlzs = self.datastore['full_lt'].get_realizations()
# compute and save statistics; this is done in process and can
# be very slow if there are thousands of realizations
weights = [rlz.weight for rlz in rlzs]
# NB: in the future we may want to save to individual hazard
# curves if oq.individual_curves is set; for the moment we
# save the statistical curves only
hstats = oq.hazard_stats()
S = len(hstats)
pmaps = list(result.values())
R = len(weights)
if len(pmaps) != R:
# this should never happen, unless I break the
# logic tree reduction mechanism during refactoring
raise AssertionError('Expected %d pmaps, got %d' %
(len(weights), len(pmaps)))
if oq.individual_curves:
logging.info('Saving individual hazard curves')
self.datastore.create_dset('hcurves-rlzs', F32, (N, R, M, L1))
self.datastore.set_shape_attrs('hcurves-rlzs',
site_id=N,
rlz_id=R,
imt=list(oq.imtls),
lvl=numpy.arange(L1))
if oq.poes:
P = len(oq.poes)
M = len(oq.imtls)
ds = self.datastore.create_dset('hmaps-rlzs', F32,
(N, R, M, P))
self.datastore.set_shape_attrs('hmaps-rlzs',
site_id=N,
rlz_id=R,
imt=list(oq.imtls),
poe=oq.poes)
for r, pmap in enumerate(pmaps):
arr = numpy.zeros((N, M, L1), F32)
for sid in pmap:
arr[sid] = pmap[sid].array.reshape(M, L1)
self.datastore['hcurves-rlzs'][:, r] = arr
if oq.poes:
hmap = calc.make_hmap(pmap, oq.imtls, oq.poes)
for sid in hmap:
ds[sid, r] = hmap[sid].array
if S:
logging.info('Computing statistical hazard curves')
self.datastore.create_dset('hcurves-stats', F32, (N, S, M, L1))
self.datastore.set_shape_attrs('hcurves-stats',
site_id=N,
stat=list(hstats),
imt=list(oq.imtls),
lvl=numpy.arange(L1))
if oq.poes:
P = len(oq.poes)
M = len(oq.imtls)
ds = self.datastore.create_dset('hmaps-stats', F32,
(N, S, M, P))
self.datastore.set_shape_attrs('hmaps-stats',
site_id=N,
stat=list(hstats),
imt=list(oq.imtls),
poes=oq.poes)
for s, stat in enumerate(hstats):
pmap = compute_pmap_stats(pmaps, [hstats[stat]], weights,
oq.imtls)
arr = numpy.zeros((N, M, L1), F32)
for sid in pmap:
arr[sid] = pmap[sid].array.reshape(M, L1)
self.datastore['hcurves-stats'][:, s] = arr
if oq.poes:
hmap = calc.make_hmap(pmap, oq.imtls, oq.poes)
for sid in hmap:
ds[sid, s] = hmap[sid].array
if self.datastore.parent:
self.datastore.parent.open('r')
if oq.compare_with_classical: # compute classical curves
export_dir = os.path.join(oq.export_dir, 'cl')
if not os.path.exists(export_dir):
os.makedirs(export_dir)
oq.export_dir = export_dir
job_id = logs.init('job')
oq.calculation_mode = 'classical'
self.cl = ClassicalCalculator(oq, job_id)
# TODO: perhaps it is possible to avoid reprocessing the source
# model, however usually this is quite fast and do not dominate
# the computation
self.cl.run()
engine.expose_outputs(self.datastore)
for imt in oq.imtls:
cl_mean_curves = get_mean_curves(self.datastore, imt)
eb_mean_curves = get_mean_curves(self.datastore, imt)
self.rdiff, index = util.max_rel_diff_index(
cl_mean_curves, eb_mean_curves)
logging.warning(
'Relative difference with the classical '
'mean curves: %d%% at site index %d, imt=%s',
self.rdiff * 100, index, imt)
def post_execute(self, result):
"""
Save the SES collection
"""
oq = self.oqparam
if 'ucerf' in oq.calculation_mode:
self.rupser.close()
self.csm.info.update_eff_ruptures(self.csm.get_num_ruptures())
self.setting_events()
N = len(self.sitecol.complete)
L = len(oq.imtls.array)
if result and oq.hazard_curves_from_gmfs:
rlzs = self.csm_info.get_rlzs_assoc().realizations
# compute and save statistics; this is done in process and can
# be very slow if there are thousands of realizations
weights = [rlz.weight for rlz in rlzs]
# NB: in the future we may want to save to individual hazard
# curves if oq.individual_curves is set; for the moment we
# save the statistical curves only
hstats = oq.hazard_stats()
if len(hstats):
logging.info('Computing statistical hazard curves')
for statname, stat in hstats:
pmap = compute_pmap_stats(result.values(), [stat], weights)
arr = numpy.zeros((N, L), F32)
for sid in pmap:
arr[sid] = pmap[sid].array[:, 0]
self.datastore['hcurves/' + statname] = arr
if oq.poes:
P = len(oq.poes)
I = len(oq.imtls)
self.datastore.create_dset(
'hmaps/' + statname, F32, (N, P * I))
self.datastore.set_attrs(
'hmaps/' + statname, nbytes=N * P * I * 4)
hmap = calc.make_hmap(pmap, oq.imtls, oq.poes)
ds = self.datastore['hmaps/' + statname]
for sid in hmap:
ds[sid] = hmap[sid].array[:, 0]
if self.datastore.parent:
self.datastore.parent.open('r')
if 'gmf_data' in self.datastore:
self.save_gmf_bytes()
if oq.compare_with_classical: # compute classical curves
export_dir = os.path.join(oq.export_dir, 'cl')
if not os.path.exists(export_dir):
os.makedirs(export_dir)
oq.export_dir = export_dir
# one could also set oq.number_of_logic_tree_samples = 0
self.cl = ClassicalCalculator(oq)
# TODO: perhaps it is possible to avoid reprocessing the source
# model, however usually this is quite fast and do not dominate
# the computation
self.cl.run(close=False)
cl_mean_curves = get_mean_curves(self.cl.datastore)
eb_mean_curves = get_mean_curves(self.datastore)
rdiff, index = util.max_rel_diff_index(
cl_mean_curves, eb_mean_curves)
logging.warn('Relative difference with the classical '
'mean curves: %d%% at site index %d',
rdiff * 100, index)
def post_execute(self, result):
oq = self.oqparam
if not oq.ground_motion_fields:
return
N = len(self.sitecol.complete)
L = len(oq.imtls.array)
if result and oq.hazard_curves_from_gmfs:
rlzs = self.rlzs_assoc.realizations
# compute and save statistics; this is done in process and can
# be very slow if there are thousands of realizations
weights = [rlz.weight for rlz in rlzs]
# NB: in the future we may want to save to individual hazard
# curves if oq.individual_curves is set; for the moment we
# save the statistical curves only
hstats = oq.hazard_stats()
S = len(hstats)
pmaps = list(result.values())
R = len(weights)
if len(pmaps) != R:
# this should never happen, unless I break the
# logic tree reduction mechanism during refactoring
raise AssertionError('Expected %d pmaps, got %d' %
(len(weights), len(pmaps)))
if oq.individual_curves:
logging.info('Saving individual hazard curves')
self.datastore.create_dset('hcurves-rlzs', F32, (N, R, L))
self.datastore.set_attrs('hcurves-rlzs', nbytes=N * R * L * 4)
if oq.poes:
P = len(oq.poes)
M = len(oq.imtls)
ds = self.datastore.create_dset(
'hmaps-rlzs', F32, (N, R, M, P))
self.datastore.set_attrs(
'hmaps-rlzs', nbytes=N * R * P * M * 4)
for r, pmap in enumerate(pmaps):
arr = numpy.zeros((N, L), F32)
for sid in pmap:
arr[sid] = pmap[sid].array[:, 0]
self.datastore['hcurves-rlzs'][:, r] = arr
if oq.poes:
hmap = calc.make_hmap(pmap, oq.imtls, oq.poes)
for sid in hmap:
ds[sid, r] = hmap[sid].array
if S:
logging.info('Computing statistical hazard curves')
self.datastore.create_dset('hcurves-stats', F32, (N, S, L))
self.datastore.set_attrs('hcurves-stats', nbytes=N * S * L * 4)
if oq.poes:
P = len(oq.poes)
M = len(oq.imtls)
ds = self.datastore.create_dset(
'hmaps-stats', F32, (N, S, M, P))
self.datastore.set_attrs(
'hmaps-stats', nbytes=N * S * P * M * 4)
for s, stat in enumerate(hstats):
pmap = compute_pmap_stats(
pmaps, [hstats[stat]], weights, oq.imtls)
arr = numpy.zeros((N, L), F32)
for sid in pmap:
arr[sid] = pmap[sid].array[:, 0]
self.datastore['hcurves-stats'][:, s] = arr
if oq.poes:
hmap = calc.make_hmap(pmap, oq.imtls, oq.poes)
for sid in hmap:
ds[sid, s] = hmap[sid].array
if self.datastore.parent:
self.datastore.parent.open('r')
if oq.compare_with_classical: # compute classical curves
export_dir = os.path.join(oq.export_dir, 'cl')
if not os.path.exists(export_dir):
os.makedirs(export_dir)
oq.export_dir = export_dir
job_id = logs.init('job')
self.cl = ClassicalCalculator(oq, job_id)
# TODO: perhaps it is possible to avoid reprocessing the source
# model, however usually this is quite fast and do not dominate
# the computation
self.cl.run(close=False)
engine.expose_outputs(self.cl.datastore)
cl_mean_curves = get_mean_curves(self.cl.datastore)
eb_mean_curves = get_mean_curves(self.datastore)
self.rdiff, index = util.max_rel_diff_index(
cl_mean_curves, eb_mean_curves)
logging.warning('Relative difference with the classical '
'mean curves: %d%% at site index %d',
self.rdiff * 100, index)
def post_execute(self, result):
oq = self.oqparam
if not oq.ground_motion_fields:
return
N = len(self.sitecol.complete)
L = len(oq.imtls.array)
if result and oq.hazard_curves_from_gmfs:
rlzs = self.rlzs_assoc.realizations
# compute and save statistics; this is done in process and can
# be very slow if there are thousands of realizations
weights = [rlz.weight for rlz in rlzs]
# NB: in the future we may want to save to individual hazard
# curves if oq.individual_curves is set; for the moment we
# save the statistical curves only
hstats = oq.hazard_stats()
pmaps = list(result.values())
if len(hstats):
logging.info('Computing statistical hazard curves')
if len(weights) != len(pmaps):
# this should never happen, unless I break the
# logic tree reduction mechanism during refactoring
raise AssertionError('Expected %d pmaps, got %d' %
(len(weights), len(pmaps)))
for statname, stat in hstats:
pmap = compute_pmap_stats(pmaps, [stat], weights, oq.imtls)
arr = numpy.zeros((N, L), F32)
for sid in pmap:
arr[sid] = pmap[sid].array[:, 0]
self.datastore['hcurves/' + statname] = arr
if oq.poes:
P = len(oq.poes)
M = len(oq.imtls)
self.datastore.create_dset(
'hmaps/' + statname, F32, (N, M, P))
self.datastore.set_attrs(
'hmaps/' + statname, nbytes=N * P * M * 4)
hmap = calc.make_hmap(pmap, oq.imtls, oq.poes)
ds = self.datastore['hmaps/' + statname]
for sid in hmap:
ds[sid] = hmap[sid].array
if self.datastore.parent:
self.datastore.parent.open('r')
if oq.compare_with_classical: # compute classical curves
export_dir = os.path.join(oq.export_dir, 'cl')
if not os.path.exists(export_dir):
os.makedirs(export_dir)
oq.export_dir = export_dir
job_id = logs.init('job')
self.cl = ClassicalCalculator(oq, job_id)
# TODO: perhaps it is possible to avoid reprocessing the source
# model, however usually this is quite fast and do not dominate
# the computation
self.cl.run(close=False)
cl_mean_curves = get_mean_curves(self.cl.datastore)
eb_mean_curves = get_mean_curves(self.datastore)
rdiff, index = util.max_rel_diff_index(
cl_mean_curves, eb_mean_curves)
logging.warning('Relative difference with the classical '
'mean curves: %d%% at site index %d',
rdiff * 100, index)
def postclassical(pgetter, N, hstats, individual_rlzs, max_sites_disagg,
amplifier, monitor):
"""
:param pgetter: an :class:`openquake.commonlib.getters.PmapGetter`
:param N: the total number of sites
:param hstats: a list of pairs (statname, statfunc)
:param individual_rlzs: if True, also build the individual curves
:param max_sites_disagg: if there are less sites than this, store rup info
:param amplifier: instance of Amplifier or None
:param monitor: instance of Monitor
:returns: a dictionary kind -> ProbabilityMap
The "kind" is a string of the form 'rlz-XXX' or 'mean' of 'quantile-XXX'
used to specify the kind of output.
"""
with monitor('read PoEs', measuremem=True):
pgetter.init()
if amplifier:
with hdf5.File(pgetter.filename, 'r') as f:
ampcode = f['sitecol'].ampcode
imtls = DictArray({imt: amplifier.amplevels for imt in pgetter.imtls})
else:
imtls = pgetter.imtls
poes, weights = pgetter.poes, pgetter.weights
M = len(imtls)
P = len(poes)
L = imtls.size
R = len(weights)
S = len(hstats)
pmap_by_kind = {}
if R > 1 and individual_rlzs or not hstats:
pmap_by_kind['hcurves-rlzs'] = [ProbabilityMap(L) for r in range(R)]
if poes:
pmap_by_kind['hmaps-rlzs'] = [
ProbabilityMap(M, P) for r in range(R)
]
if hstats:
pmap_by_kind['hcurves-stats'] = [ProbabilityMap(L) for r in range(S)]
if poes:
pmap_by_kind['hmaps-stats'] = [
ProbabilityMap(M, P) for r in range(S)
]
combine_mon = monitor('combine pmaps', measuremem=False)
compute_mon = monitor('compute stats', measuremem=False)
for sid in pgetter.sids:
with combine_mon:
pc = pgetter.get_pcurve(sid) # shape (L, R)
if amplifier:
pc = amplifier.amplify(ampcode[sid], pc)
# NB: the pcurve have soil levels != IMT levels
if pc.array.sum() == 0: # no data
continue
with compute_mon:
if hstats:
for s, (statname, stat) in enumerate(hstats.items()):
sc = getters.build_stat_curve(pc, imtls, stat, weights)
pmap_by_kind['hcurves-stats'][s][sid] = sc
if poes:
hmap = calc.make_hmap(sc, imtls, poes, sid)
pmap_by_kind['hmaps-stats'][s].update(hmap)
if R > 1 and individual_rlzs or not hstats:
for r, pmap in enumerate(pmap_by_kind['hcurves-rlzs']):
pmap[sid] = pc.extract(r)
if poes:
for r in range(R):
hmap = calc.make_hmap(pc.extract(r), imtls, poes, sid)
pmap_by_kind['hmaps-rlzs'][r].update(hmap)
return pmap_by_kind