def build_hazard(pgetter, N, hstats, individual_curves, max_sites_disagg, amplifier, monitor): """ :param pgetter: an :class:`openquake.commonlib.getters.PmapGetter` :param N: the total number of sites :param hstats: a list of pairs (statname, statfunc) :param individual_curves: if True, also build the individual curves :param max_sites_disagg: if there are less sites than this, store rup info :param amplifier: instance of Amplifier or None :param monitor: instance of Monitor :returns: a dictionary kind -> ProbabilityMap The "kind" is a string of the form 'rlz-XXX' or 'mean' of 'quantile-XXX' used to specify the kind of output. """ with monitor('read PoEs'): pgetter.init() if amplifier: ampcode = pgetter.dstore['sitecol'].ampcode imtls, poes, weights = pgetter.imtls, pgetter.poes, pgetter.weights M = len(imtls) P = len(poes) L = len(imtls.array) if amplifier is None else len(amplifier.amplevels) * M R = len(weights) S = len(hstats) pmap_by_kind = {} if R > 1 and individual_curves or not hstats: pmap_by_kind['hcurves-rlzs'] = [ProbabilityMap(L) for r in range(R)] if hstats: pmap_by_kind['hcurves-stats'] = [ProbabilityMap(L) for r in range(S)] if poes: pmap_by_kind['hmaps-stats'] = [ ProbabilityMap(M, P) for r in range(S) ] combine_mon = monitor('combine pmaps', measuremem=False) compute_mon = monitor('compute stats', measuremem=False) for sid in pgetter.sids: with combine_mon: pcurves = pgetter.get_pcurves(sid) if amplifier: pcurves = amplifier.amplify(ampcode[sid], pcurves) if sum(pc.array.sum() for pc in pcurves) == 0: # no data continue with compute_mon: if hstats: arr = numpy.array([pc.array for pc in pcurves]) for s, (statname, stat) in enumerate(hstats.items()): pc = getters.build_stat_curve(arr, imtls, stat, weights) pmap_by_kind['hcurves-stats'][s][sid] = pc if poes: hmap = calc.make_hmap(pc, pgetter.imtls, poes, sid) pmap_by_kind['hmaps-stats'][s].update(hmap) if R > 1 and individual_curves or not hstats: for pmap, pc in zip(pmap_by_kind['hcurves-rlzs'], pcurves): pmap[sid] = pc if poes: pmap_by_kind['hmaps-rlzs'] = [ calc.make_hmap(pc, imtls, poes, sid) for pc in pcurves ] return pmap_by_kind
def build_hazard_stats(pgetter, N, hstats, individual_curves, monitor): """ :param pgetter: an :class:`openquake.commonlib.getters.PmapGetter` :param N: the total number of sites :param hstats: a list of pairs (statname, statfunc) :param individual_curves: if True, also build the individual curves :param monitor: instance of Monitor :returns: a dictionary kind -> ProbabilityMap The "kind" is a string of the form 'rlz-XXX' or 'mean' of 'quantile-XXX' used to specify the kind of output. """ with monitor('read PoEs'): pgetter.init() imtls, poes, weights = pgetter.imtls, pgetter.poes, pgetter.weights L = len(imtls.array) R = len(weights) S = len(hstats) pmap_by_kind = {'rlz_by_sid': {}} if R > 1 and individual_curves or not hstats: pmap_by_kind['hcurves-rlzs'] = [ProbabilityMap(L) for r in range(R)] if hstats: pmap_by_kind['hcurves-stats'] = [ProbabilityMap(L) for r in range(S)] if poes: pmap_by_kind['hmaps-stats'] = [ProbabilityMap(L) for r in range(S)] combine_mon = monitor('combine pmaps', measuremem=False) compute_mon = monitor('compute stats', measuremem=False) for sid in pgetter.sids: with combine_mon: pcurves = pgetter.get_pcurves(sid) if sum(pc.array.sum() for pc in pcurves) == 0: # no data continue with compute_mon: if hstats: arr = numpy.array([pc.array for pc in pcurves]) for s, (statname, stat) in enumerate(hstats.items()): pc = _build_stat_curve(arr, imtls, stat, weights) pmap_by_kind['hcurves-stats'][s][sid] = pc if poes: hmap = calc.make_hmap(pc, pgetter.imtls, poes, sid) pmap_by_kind['hmaps-stats'][s].update(hmap) if statname == 'mean' and R > 1 and N <= FEWSITES: rlz = pmap_by_kind['rlz_by_sid'] rlz[sid] = util.closest_to_ref( [p.array for p in pcurves], pc.array)['rlz'] if R > 1 and individual_curves or not hstats: for pmap, pc in zip(pmap_by_kind['hcurves-rlzs'], pcurves): pmap[sid] = pc if poes: pmap_by_kind['hmaps-rlzs'] = [ calc.make_hmap(pc, imtls, poes, sid) for pc in pcurves ] return pmap_by_kind
def build_hazard_stats(pgetter, N, hstats, individual_curves, monitor): """ :param pgetter: an :class:`openquake.commonlib.getters.PmapGetter` :param N: the total number of sites :param hstats: a list of pairs (statname, statfunc) :param individual_curves: if True, also build the individual curves :param monitor: instance of Monitor :returns: a dictionary kind -> ProbabilityMap The "kind" is a string of the form 'rlz-XXX' or 'mean' of 'quantile-XXX' used to specify the kind of output. """ with monitor('combine pmaps'): pgetter.init() # if not already initialized try: pmaps = pgetter.get_pmaps() except IndexError: # no data return {} if sum(len(pmap) for pmap in pmaps) == 0: # no data return {} R = len(pmaps) imtls, poes, weights = pgetter.imtls, pgetter.poes, pgetter.weights pmap_by_kind = {} hmaps_stats = [] hcurves_stats = [] with monitor('compute stats'): for statname, stat in hstats.items(): pmap = compute_pmap_stats(pmaps, [stat], weights, imtls) hcurves_stats.append(pmap) if pgetter.poes: hmaps_stats.append( calc.make_hmap(pmap, pgetter.imtls, pgetter.poes)) if statname == 'mean' and R > 1 and N <= FEWSITES: pmap_by_kind['rlz_by_sid'] = rlz = {} for sid, pcurve in pmap.items(): rlz[sid] = util.closest_to_ref( [pm.setdefault(sid, 0).array for pm in pmaps], pcurve.array)['rlz'] if hcurves_stats: pmap_by_kind['hcurves-stats'] = hcurves_stats if hmaps_stats: pmap_by_kind['hmaps-stats'] = hmaps_stats if R > 1 and individual_curves or not hstats: pmap_by_kind['hcurves-rlzs'] = pmaps if pgetter.poes: with monitor('build individual hmaps'): pmap_by_kind['hmaps-rlzs'] = [ calc.make_hmap(pmap, imtls, poes) for pmap in pmaps ] return pmap_by_kind
def build_hazard_stats(pgetter, N, hstats, individual_curves, monitor): """ :param pgetter: an :class:`openquake.commonlib.getters.PmapGetter` :param N: the total number of sites :param hstats: a list of pairs (statname, statfunc) :param individual_curves: if True, also build the individual curves :param monitor: instance of Monitor :returns: a dictionary kind -> ProbabilityMap The "kind" is a string of the form 'rlz-XXX' or 'mean' of 'quantile-XXX' used to specify the kind of output. """ with monitor('combine pmaps'): pgetter.init() # if not already initialized try: pmaps = pgetter.get_pmaps() except IndexError: # no data return {} if sum(len(pmap) for pmap in pmaps) == 0: # no data return {} R = len(pmaps) imtls, poes, weights = pgetter.imtls, pgetter.poes, pgetter.weights pmap_by_kind = {} hmaps_stats = [] hcurves_stats = [] with monitor('compute stats'): for statname, stat in hstats.items(): pmap = compute_pmap_stats(pmaps, [stat], weights, imtls) hcurves_stats.append(pmap) if pgetter.poes: hmaps_stats.append( calc.make_hmap(pmap, pgetter.imtls, pgetter.poes)) if statname == 'mean' and R > 1 and N <= FEWSITES: pmap_by_kind['rlz_by_sid'] = rlz = {} for sid, pcurve in pmap.items(): rlz[sid] = util.closest_to_ref( [pm.setdefault(sid, 0).array for pm in pmaps], pcurve.array)['rlz'] if hcurves_stats: pmap_by_kind['hcurves-stats'] = hcurves_stats if hmaps_stats: pmap_by_kind['hmaps-stats'] = hmaps_stats if R > 1 and individual_curves or not hstats: pmap_by_kind['hcurves-rlzs'] = pmaps if pgetter.poes: with monitor('build individual hmaps'): pmap_by_kind['hmaps-rlzs'] = [ calc.make_hmap(pmap, imtls, poes) for pmap in pmaps] return pmap_by_kind
def extract_hazard(dstore, what): """ Extracts hazard curves and possibly hazard maps and/or uniform hazard spectra. Use it as /extract/hazard/mean or /extract/hazard/rlz-0, etc """ oq = dstore['oqparam'] sitecol = dstore['sitecol'] yield 'sitecol', sitecol yield 'oqparam', oq yield 'imtls', oq.imtls yield 'realizations', dstore['csm_info'].rlzs yield 'checksum32', dstore['/'].attrs['checksum32'] nsites = len(sitecol) M = len(oq.imtls) P = len(oq.poes) for kind, pmap in getters.PmapGetter(dstore).items(what): for imt in oq.imtls: key = 'hcurves/%s/%s' % (imt, kind) arr = numpy.zeros((nsites, len(oq.imtls[imt]))) for sid in pmap: arr[sid] = pmap[sid].array[oq.imtls.slicedic[imt], 0] logging.info('extracting %s', key) yield key, arr if oq.poes: hmap = calc.make_hmap(pmap, oq.imtls, oq.poes) for p, poe in enumerate(oq.poes): key = 'hmaps/poe-%s/%s' % (poe, kind) arr = numpy.zeros((nsites, M)) idx = [m * P + p for m in range(M)] for sid in pmap: arr[sid] = hmap[sid].array[idx, 0] logging.info('extracting %s', key) yield key, arr
def export_hmaps_xml_json(ekey, dstore): key, kind, fmt = get_kkf(ekey) oq = dstore['oqparam'] sitecol = dstore['sitecol'] sitemesh = get_mesh(sitecol) rlzs_assoc = dstore['csm_info'].get_rlzs_assoc() fnames = [] writercls = hazard_writers.HazardMapXMLWriter pdic = DictArray({imt: oq.poes for imt in oq.imtls}) nsites = len(sitemesh) for kind, hcurves in PmapGetter(dstore).items(): hmaps = calc.make_hmap( hcurves, oq.imtls, oq.poes).convert(pdic, nsites) if kind.startswith('rlz-'): rlz = rlzs_assoc.realizations[int(kind[4:])] smlt_path = '_'.join(rlz.sm_lt_path) gsimlt_path = rlz.gsim_rlz.uid else: smlt_path = '' gsimlt_path = '' for imt in oq.imtls: for j, poe in enumerate(oq.poes): suffix = '-%s-%s' % (poe, imt) fname = hazard_curve_name( dstore, ekey, kind + suffix, rlzs_assoc) data = [HazardMap(site[0], site[1], _extract(hmap, imt, j)) for site, hmap in zip(sitemesh, hmaps)] writer = writercls( fname, investigation_time=oq.investigation_time, imt=imt, poe=poe, smlt_path=smlt_path, gsimlt_path=gsimlt_path) writer.serialize(data) fnames.append(fname) return sorted(fnames)
def export_hcurves_csv(ekey, dstore): """ Exports the hazard curves into several .csv files :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ oq = dstore['oqparam'] rlzs_assoc = dstore['csm_info'].get_rlzs_assoc() sitecol = dstore['sitecol'] sitemesh = get_mesh(sitecol) key, kind, fmt = get_kkf(ekey) fnames = [] if oq.poes: pdic = DictArray({imt: oq.poes for imt in oq.imtls}) for kind, hcurves in calc.PmapGetter(dstore).items(kind): fname = hazard_curve_name(dstore, (key, fmt), kind, rlzs_assoc) comment = _comment(rlzs_assoc, kind, oq.investigation_time) if key == 'uhs' and oq.poes and oq.uniform_hazard_spectra: uhs_curves = calc.make_uhs(hcurves, oq.imtls, oq.poes, len(sitemesh)) writers.write_csv(fname, util.compose_arrays(sitemesh, uhs_curves), comment=comment) fnames.append(fname) elif key == 'hmaps' and oq.poes and oq.hazard_maps: hmap = calc.make_hmap(hcurves, oq.imtls, oq.poes) fnames.extend( export_hazard_csv(ekey, fname, sitemesh, hmap, pdic, comment)) elif key == 'hcurves': fnames.extend( export_hcurves_by_imt_csv(ekey, kind, rlzs_assoc, fname, sitecol, hcurves, oq)) return sorted(fnames)
def extract_hazard(dstore, what): """ Extracts hazard curves and possibly hazard maps and/or uniform hazard spectra. Use it as /extract/hazard/mean or /extract/hazard/rlz-0, etc """ oq = dstore['oqparam'] sitecol = dstore['sitecol'] rlzs_assoc = dstore['csm_info'].get_rlzs_assoc() yield 'sitecol', sitecol yield 'oqparam', oq yield 'imtls', oq.imtls yield 'realizations', dstore['csm_info'].rlzs yield 'checksum32', dstore['/'].attrs['checksum32'] nsites = len(sitecol) M = len(oq.imtls) for statname, pmap in getters.PmapGetter(dstore, rlzs_assoc).items(what): for imt in oq.imtls: key = 'hcurves/%s/%s' % (imt, statname) arr = numpy.zeros((nsites, len(oq.imtls[imt]))) for sid in pmap: arr[sid] = pmap[sid].array[oq.imtls(imt), 0] logging.info('extracting %s', key) yield key, arr hmap = calc.make_hmap(pmap, oq.imtls, oq.poes) for p, poe in enumerate(oq.poes): key = 'hmaps/poe-%s/%s' % (poe, statname) arr = numpy.zeros((nsites, M)) for sid in pmap: arr[sid] = hmap[sid].array[:, p] logging.info('extracting %s', key) yield key, arr
def build_hazard_stats(pgetter, hstats, monitor): """ :param pgetter: an :class:`openquake.commonlib.getters.PmapGetter` :param hstats: a list of pairs (statname, statfunc) :param monitor: instance of Monitor :returns: a dictionary kind -> ProbabilityMap The "kind" is a string of the form 'rlz-XXX' or 'mean' of 'quantile-XXX' used to specify the kind of output. """ with monitor('combine pmaps'): pgetter.init() # if not already initialized try: pmaps = pgetter.get_pmaps(pgetter.sids) except IndexError: # no data return {} if sum(len(pmap) for pmap in pmaps) == 0: # no data return {} pmap_by_kind = {} for statname, stat in hstats: with monitor('compute ' + statname): pmap = compute_pmap_stats(pmaps, [stat], pgetter.weights) pmap_by_kind['hcurves', statname] = pmap if pgetter.poes: pmap_by_kind['hmaps', statname] = calc.make_hmap(pmap, pgetter.imtls, pgetter.poes) return pmap_by_kind
def export_hmaps_npz(ekey, dstore): oq = dstore['oqparam'] mesh = get_mesh(dstore['sitecol']) pdic = DictArray({imt: oq.poes for imt in oq.imtls}) fname = dstore.export_path('%s.%s' % ekey) dic = {} for kind, hcurves in calc.PmapGetter(dstore).items(): hmap = calc.make_hmap(hcurves, oq.imtls, oq.poes) dic[kind] = convert_to_array(hmap, mesh, pdic) savez(fname, **dic) return [fname]
def export_hmaps_hdf5(ekey, dstore): oq = dstore['oqparam'] mesh = get_mesh(dstore['sitecol']) pdic = DictArray({imt: oq.poes for imt in oq.imtls}) fname = dstore.export_path('%s.%s' % ekey) with hdf5.File(fname, 'w') as f: for dskey in dstore['hcurves']: hcurves = dstore['hcurves/%s' % dskey] hmap = calc.make_hmap(hcurves, oq.imtls, oq.poes) f['hmaps/%s' % dskey] = convert_to_array(hmap, mesh, pdic) return [fname]
def export_hmaps_npz(ekey, dstore): oq = dstore['oqparam'] mesh = get_mesh(dstore['sitecol']) pdic = DictArray({imt: oq.poes for imt in oq.imtls}) fname = dstore.export_path('%s.%s' % ekey) dic = {} for dskey in dstore['hcurves']: hcurves = dstore['hcurves/%s' % dskey] hmap = calc.make_hmap(hcurves, oq.imtls, oq.poes) dic[dskey] = convert_to_array(hmap, mesh, pdic) savez(fname, **dic) return [fname]
def plot_hmaps(calc_id): """ Mean hazard maps plotter. """ dstore = datastore.read(calc_id) oq = dstore['oqparam'] mean = calc.PmapGetter(dstore).get_mean() hmaps = calc.make_hmap(mean, oq.imtls, oq.poes) M, P = len(oq.imtls), len(oq.poes) array = hmaps.array.reshape(len(hmaps.array), M, P) plt = make_figure(dstore['sitecol'], oq.imtls, oq.poes, array) plt.show()
def export_hmaps_np(ekey, dstore): oq = dstore['oqparam'] sitecol = dstore['sitecol'] mesh = get_mesh(sitecol) pdic = DictArray({imt: oq.poes for imt in oq.imtls}) fname = dstore.export_path('%s.%s' % ekey) dic = {} for kind, hcurves in PmapGetter(dstore).items(): hmap = calc.make_hmap(hcurves, oq.imtls, oq.poes) dic[kind] = calc.convert_to_array(hmap, len(mesh), pdic) save_np(fname, dic, mesh, ('vs30', F32, sitecol.vs30), investigation_time=oq.investigation_time) return [fname]
def plot_hmaps(calc_id): """ Mean hazard maps plotter. """ dstore = engine.read(calc_id) oq = dstore['oqparam'] rlzs_assoc = dstore['csm_info'].get_rlzs_assoc() mean = getters.PmapGetter(dstore, rlzs_assoc).get_mean() hmaps = calc.make_hmap(mean, oq.imtls, oq.poes) M, P = len(oq.imtls), len(oq.poes) array = hmaps.array.reshape(len(hmaps.array), M, P) plt = make_figure(dstore['sitecol'], oq.imtls, oq.poes, array) plt.show()
def extract_hmaps(dstore, what): """ Extracts hazard maps. Use it as /extract/hmaps/mean or /extract/hmaps/rlz-0, etc """ oq = dstore['oqparam'] sitecol = dstore['sitecol'] mesh = get_mesh(sitecol) pdic = DictArray({imt: oq.poes for imt in oq.imtls}) dic = {} for kind, hcurves in getters.PmapGetter(dstore).items(what): hmap = calc.make_hmap(hcurves, oq.imtls, oq.poes) dic[kind] = calc.convert_to_array(hmap, len(mesh), pdic) return hazard_items(dic, mesh, investigation_time=oq.investigation_time)
def view_flat_hmaps(token, dstore): """ Display the flat hazard maps for the calculation. They are used for debugging purposes when comparing the results of two calculations. They are the mean over the sites of the mean hazard maps. """ oq = dstore['oqparam'] assert oq.poes nsites = len(dstore['sitecol']) pdic = DictArray({imt: oq.poes for imt in oq.imtls}) mean = getters.PmapGetter(dstore).get_mean() hmaps = calc.make_hmap(mean, oq.imtls, oq.poes) array = calc.convert_to_array(hmaps, nsites, pdic) res = numpy.zeros(1, array.dtype) for name in array.dtype.names: res[name] = array[name].mean() return rst_table(res)
def export_hcurves_csv(ekey, dstore): """ Exports the hazard curves into several .csv files :param ekey: export key, i.e. a pair (datastore key, fmt) :param dstore: datastore object """ oq = dstore['oqparam'] rlzs_assoc = dstore['csm_info'].get_rlzs_assoc() sitecol = dstore['sitecol'] sitemesh = get_mesh(sitecol) key, fmt = ekey fnames = [] if oq.poes: pdic = DictArray({imt: oq.poes for imt in oq.imtls}) for kind in sorted(dstore['hcurves']): hcurves = dstore['hcurves/' + kind] fname = hazard_curve_name(dstore, ekey, kind, rlzs_assoc) comment = _comment(rlzs_assoc, kind, oq.investigation_time) if key == 'uhs': uhs_curves = calc.make_uhs( hcurves, oq.imtls, oq.poes, len(sitemesh)) writers.write_csv( fname, util.compose_arrays(sitemesh, uhs_curves), comment=comment) fnames.append(fname) elif key == 'hmaps': hmap = calc.make_hmap(hcurves, oq.imtls, oq.poes) fnames.extend( export_hazard_csv(ekey, fname, sitemesh, hmap, pdic, comment)) else: if export.from_db: # called by export_from_db fnames.extend( export_hcurves_by_imt_csv( ekey, kind, rlzs_assoc, fname, sitecol, hcurves, oq)) else: # when exporting directly from the datastore fnames.extend( export_hazard_csv( ekey, fname, sitemesh, hcurves, oq.imtls, comment)) return sorted(fnames)
def view_hmap(token, dstore): """ Display the highest 20 points of the mean hazard map. Called as $ oq show hmap:0.1 # 10% PoE """ try: poe = valid.probability(token.split(':')[1]) except IndexError: poe = 0.1 try: mean = dstore['hcurves/mean'] except KeyError: # there is a single realization mean = dstore['hcurves/rlz-000'] oq = dstore['oqparam'] hmap = calc.make_hmap(mean, oq.imtls, [poe]) items = sorted([(hmap[sid].array.sum(), sid) for sid in hmap])[-20:] dt = numpy.dtype([('sid', U32)] + [(imt, F32) for imt in oq.imtls]) array = numpy.zeros(len(items), dt) for i, (maxvalue, sid) in enumerate(reversed(items)): array[i] = (sid, ) + tuple(hmap[sid].array[:, 0]) return rst_table(array)
def extract_hazard_for_qgis(dstore, what): """ Extracts hazard curves and possibly hazard maps and/or uniform hazard spectra. Use it as /extract/qgis-hazard/rlz-0, etc """ oq = dstore['oqparam'] sitecol = dstore['sitecol'] yield 'sitecol', sitecol yield 'oqparam', oq yield 'realizations', dstore['csm_info'].rlzs yield 'checksum32', dstore['/'].attrs['checksum32'] N = len(sitecol) if oq.poes: pdic = {imt: oq.poes for imt in oq.imtls} for kind, hcurves in getters.PmapGetter(dstore).items(what): logging.info('extracting hazard/%s', kind) yield 'hcurves-' + kind, calc.convert_to_array(hcurves, N, oq.imtls) if oq.poes and oq.uniform_hazard_spectra: yield 'uhs-' + kind, calc.make_uhs(hcurves, oq.imtls, oq.poes, N) if oq.poes and oq.hazard_maps: hmaps = calc.make_hmap(hcurves, oq.imtls, oq.poes) yield 'hmaps-' + kind, calc.convert_to_array(hmaps, N, pdic)
def export_hmaps_xml_json(ekey, dstore): export_type = ekey[1] oq = dstore['oqparam'] sitemesh = get_mesh(dstore['sitecol']) rlzs_assoc = dstore['csm_info'].get_rlzs_assoc() fnames = [] writercls = (hazard_writers.HazardMapGeoJSONWriter if export_type == 'geojson' else hazard_writers.HazardMapXMLWriter) pdic = DictArray({imt: oq.poes for imt in oq.imtls}) nsites = len(sitemesh) for kind in dstore['hcurves']: hcurves = dstore['hcurves/' + kind] hmaps = calc.make_hmap( hcurves, oq.imtls, oq.poes).convert(pdic, nsites) if kind.startswith('rlz-'): rlz = rlzs_assoc.realizations[int(kind[4:])] smlt_path = '_'.join(rlz.sm_lt_path) gsimlt_path = rlz.gsim_rlz.uid else: smlt_path = '' gsimlt_path = '' for imt in oq.imtls: for j, poe in enumerate(oq.poes): suffix = '-%s-%s' % (poe, imt) fname = hazard_curve_name( dstore, ekey, kind + suffix, rlzs_assoc) data = [HazardMap(site[0], site[1], _extract(hmap, imt, j)) for site, hmap in zip(sitemesh, hmaps)] writer = writercls( fname, investigation_time=oq.investigation_time, imt=imt, poe=poe, smlt_path=smlt_path, gsimlt_path=gsimlt_path) writer.serialize(data) fnames.append(fname) return sorted(fnames)
def post_execute(self, result): oq = self.oqparam if not oq.ground_motion_fields and not oq.hazard_curves_from_gmfs: return N = len(self.sitecol.complete) M = len(oq.imtls) L = len(oq.imtls.array) L1 = L // M if result and oq.hazard_curves_from_gmfs: rlzs = self.datastore['full_lt'].get_realizations() # compute and save statistics; this is done in process and can # be very slow if there are thousands of realizations weights = [rlz.weight for rlz in rlzs] # NB: in the future we may want to save to individual hazard # curves if oq.individual_curves is set; for the moment we # save the statistical curves only hstats = oq.hazard_stats() S = len(hstats) pmaps = list(result.values()) R = len(weights) if len(pmaps) != R: # this should never happen, unless I break the # logic tree reduction mechanism during refactoring raise AssertionError('Expected %d pmaps, got %d' % (len(weights), len(pmaps))) if oq.individual_curves: logging.info('Saving individual hazard curves') self.datastore.create_dset('hcurves-rlzs', F32, (N, R, M, L1)) self.datastore.set_shape_attrs('hcurves-rlzs', site_id=N, rlz_id=R, imt=list(oq.imtls), lvl=numpy.arange(L1)) if oq.poes: P = len(oq.poes) M = len(oq.imtls) ds = self.datastore.create_dset('hmaps-rlzs', F32, (N, R, M, P)) self.datastore.set_shape_attrs('hmaps-rlzs', site_id=N, rlz_id=R, imt=list(oq.imtls), poe=oq.poes) for r, pmap in enumerate(pmaps): arr = numpy.zeros((N, M, L1), F32) for sid in pmap: arr[sid] = pmap[sid].array.reshape(M, L1) self.datastore['hcurves-rlzs'][:, r] = arr if oq.poes: hmap = calc.make_hmap(pmap, oq.imtls, oq.poes) for sid in hmap: ds[sid, r] = hmap[sid].array if S: logging.info('Computing statistical hazard curves') self.datastore.create_dset('hcurves-stats', F32, (N, S, M, L1)) self.datastore.set_shape_attrs('hcurves-stats', site_id=N, stat=list(hstats), imt=list(oq.imtls), lvl=numpy.arange(L1)) if oq.poes: P = len(oq.poes) M = len(oq.imtls) ds = self.datastore.create_dset('hmaps-stats', F32, (N, S, M, P)) self.datastore.set_shape_attrs('hmaps-stats', site_id=N, stat=list(hstats), imt=list(oq.imtls), poes=oq.poes) for s, stat in enumerate(hstats): pmap = compute_pmap_stats(pmaps, [hstats[stat]], weights, oq.imtls) arr = numpy.zeros((N, M, L1), F32) for sid in pmap: arr[sid] = pmap[sid].array.reshape(M, L1) self.datastore['hcurves-stats'][:, s] = arr if oq.poes: hmap = calc.make_hmap(pmap, oq.imtls, oq.poes) for sid in hmap: ds[sid, s] = hmap[sid].array if self.datastore.parent: self.datastore.parent.open('r') if oq.compare_with_classical: # compute classical curves export_dir = os.path.join(oq.export_dir, 'cl') if not os.path.exists(export_dir): os.makedirs(export_dir) oq.export_dir = export_dir job_id = logs.init('job') oq.calculation_mode = 'classical' self.cl = ClassicalCalculator(oq, job_id) # TODO: perhaps it is possible to avoid reprocessing the source # model, however usually this is quite fast and do not dominate # the computation self.cl.run() engine.expose_outputs(self.datastore) for imt in oq.imtls: cl_mean_curves = get_mean_curves(self.datastore, imt) eb_mean_curves = get_mean_curves(self.datastore, imt) self.rdiff, index = util.max_rel_diff_index( cl_mean_curves, eb_mean_curves) logging.warning( 'Relative difference with the classical ' 'mean curves: %d%% at site index %d, imt=%s', self.rdiff * 100, index, imt)
def post_execute(self, result): """ Save the SES collection """ oq = self.oqparam if 'ucerf' in oq.calculation_mode: self.rupser.close() self.csm.info.update_eff_ruptures(self.csm.get_num_ruptures()) self.setting_events() N = len(self.sitecol.complete) L = len(oq.imtls.array) if result and oq.hazard_curves_from_gmfs: rlzs = self.csm_info.get_rlzs_assoc().realizations # compute and save statistics; this is done in process and can # be very slow if there are thousands of realizations weights = [rlz.weight for rlz in rlzs] # NB: in the future we may want to save to individual hazard # curves if oq.individual_curves is set; for the moment we # save the statistical curves only hstats = oq.hazard_stats() if len(hstats): logging.info('Computing statistical hazard curves') for statname, stat in hstats: pmap = compute_pmap_stats(result.values(), [stat], weights) arr = numpy.zeros((N, L), F32) for sid in pmap: arr[sid] = pmap[sid].array[:, 0] self.datastore['hcurves/' + statname] = arr if oq.poes: P = len(oq.poes) I = len(oq.imtls) self.datastore.create_dset( 'hmaps/' + statname, F32, (N, P * I)) self.datastore.set_attrs( 'hmaps/' + statname, nbytes=N * P * I * 4) hmap = calc.make_hmap(pmap, oq.imtls, oq.poes) ds = self.datastore['hmaps/' + statname] for sid in hmap: ds[sid] = hmap[sid].array[:, 0] if self.datastore.parent: self.datastore.parent.open('r') if 'gmf_data' in self.datastore: self.save_gmf_bytes() if oq.compare_with_classical: # compute classical curves export_dir = os.path.join(oq.export_dir, 'cl') if not os.path.exists(export_dir): os.makedirs(export_dir) oq.export_dir = export_dir # one could also set oq.number_of_logic_tree_samples = 0 self.cl = ClassicalCalculator(oq) # TODO: perhaps it is possible to avoid reprocessing the source # model, however usually this is quite fast and do not dominate # the computation self.cl.run(close=False) cl_mean_curves = get_mean_curves(self.cl.datastore) eb_mean_curves = get_mean_curves(self.datastore) rdiff, index = util.max_rel_diff_index( cl_mean_curves, eb_mean_curves) logging.warn('Relative difference with the classical ' 'mean curves: %d%% at site index %d', rdiff * 100, index)
def post_execute(self, result): oq = self.oqparam if not oq.ground_motion_fields: return N = len(self.sitecol.complete) L = len(oq.imtls.array) if result and oq.hazard_curves_from_gmfs: rlzs = self.rlzs_assoc.realizations # compute and save statistics; this is done in process and can # be very slow if there are thousands of realizations weights = [rlz.weight for rlz in rlzs] # NB: in the future we may want to save to individual hazard # curves if oq.individual_curves is set; for the moment we # save the statistical curves only hstats = oq.hazard_stats() S = len(hstats) pmaps = list(result.values()) R = len(weights) if len(pmaps) != R: # this should never happen, unless I break the # logic tree reduction mechanism during refactoring raise AssertionError('Expected %d pmaps, got %d' % (len(weights), len(pmaps))) if oq.individual_curves: logging.info('Saving individual hazard curves') self.datastore.create_dset('hcurves-rlzs', F32, (N, R, L)) self.datastore.set_attrs('hcurves-rlzs', nbytes=N * R * L * 4) if oq.poes: P = len(oq.poes) M = len(oq.imtls) ds = self.datastore.create_dset( 'hmaps-rlzs', F32, (N, R, M, P)) self.datastore.set_attrs( 'hmaps-rlzs', nbytes=N * R * P * M * 4) for r, pmap in enumerate(pmaps): arr = numpy.zeros((N, L), F32) for sid in pmap: arr[sid] = pmap[sid].array[:, 0] self.datastore['hcurves-rlzs'][:, r] = arr if oq.poes: hmap = calc.make_hmap(pmap, oq.imtls, oq.poes) for sid in hmap: ds[sid, r] = hmap[sid].array if S: logging.info('Computing statistical hazard curves') self.datastore.create_dset('hcurves-stats', F32, (N, S, L)) self.datastore.set_attrs('hcurves-stats', nbytes=N * S * L * 4) if oq.poes: P = len(oq.poes) M = len(oq.imtls) ds = self.datastore.create_dset( 'hmaps-stats', F32, (N, S, M, P)) self.datastore.set_attrs( 'hmaps-stats', nbytes=N * S * P * M * 4) for s, stat in enumerate(hstats): pmap = compute_pmap_stats( pmaps, [hstats[stat]], weights, oq.imtls) arr = numpy.zeros((N, L), F32) for sid in pmap: arr[sid] = pmap[sid].array[:, 0] self.datastore['hcurves-stats'][:, s] = arr if oq.poes: hmap = calc.make_hmap(pmap, oq.imtls, oq.poes) for sid in hmap: ds[sid, s] = hmap[sid].array if self.datastore.parent: self.datastore.parent.open('r') if oq.compare_with_classical: # compute classical curves export_dir = os.path.join(oq.export_dir, 'cl') if not os.path.exists(export_dir): os.makedirs(export_dir) oq.export_dir = export_dir job_id = logs.init('job') self.cl = ClassicalCalculator(oq, job_id) # TODO: perhaps it is possible to avoid reprocessing the source # model, however usually this is quite fast and do not dominate # the computation self.cl.run(close=False) engine.expose_outputs(self.cl.datastore) cl_mean_curves = get_mean_curves(self.cl.datastore) eb_mean_curves = get_mean_curves(self.datastore) self.rdiff, index = util.max_rel_diff_index( cl_mean_curves, eb_mean_curves) logging.warning('Relative difference with the classical ' 'mean curves: %d%% at site index %d', self.rdiff * 100, index)
def post_execute(self, result): oq = self.oqparam if not oq.ground_motion_fields: return N = len(self.sitecol.complete) L = len(oq.imtls.array) if result and oq.hazard_curves_from_gmfs: rlzs = self.rlzs_assoc.realizations # compute and save statistics; this is done in process and can # be very slow if there are thousands of realizations weights = [rlz.weight for rlz in rlzs] # NB: in the future we may want to save to individual hazard # curves if oq.individual_curves is set; for the moment we # save the statistical curves only hstats = oq.hazard_stats() pmaps = list(result.values()) if len(hstats): logging.info('Computing statistical hazard curves') if len(weights) != len(pmaps): # this should never happen, unless I break the # logic tree reduction mechanism during refactoring raise AssertionError('Expected %d pmaps, got %d' % (len(weights), len(pmaps))) for statname, stat in hstats: pmap = compute_pmap_stats(pmaps, [stat], weights, oq.imtls) arr = numpy.zeros((N, L), F32) for sid in pmap: arr[sid] = pmap[sid].array[:, 0] self.datastore['hcurves/' + statname] = arr if oq.poes: P = len(oq.poes) M = len(oq.imtls) self.datastore.create_dset( 'hmaps/' + statname, F32, (N, M, P)) self.datastore.set_attrs( 'hmaps/' + statname, nbytes=N * P * M * 4) hmap = calc.make_hmap(pmap, oq.imtls, oq.poes) ds = self.datastore['hmaps/' + statname] for sid in hmap: ds[sid] = hmap[sid].array if self.datastore.parent: self.datastore.parent.open('r') if oq.compare_with_classical: # compute classical curves export_dir = os.path.join(oq.export_dir, 'cl') if not os.path.exists(export_dir): os.makedirs(export_dir) oq.export_dir = export_dir job_id = logs.init('job') self.cl = ClassicalCalculator(oq, job_id) # TODO: perhaps it is possible to avoid reprocessing the source # model, however usually this is quite fast and do not dominate # the computation self.cl.run(close=False) cl_mean_curves = get_mean_curves(self.cl.datastore) eb_mean_curves = get_mean_curves(self.datastore) rdiff, index = util.max_rel_diff_index( cl_mean_curves, eb_mean_curves) logging.warning('Relative difference with the classical ' 'mean curves: %d%% at site index %d', rdiff * 100, index)
def postclassical(pgetter, N, hstats, individual_rlzs, max_sites_disagg, amplifier, monitor): """ :param pgetter: an :class:`openquake.commonlib.getters.PmapGetter` :param N: the total number of sites :param hstats: a list of pairs (statname, statfunc) :param individual_rlzs: if True, also build the individual curves :param max_sites_disagg: if there are less sites than this, store rup info :param amplifier: instance of Amplifier or None :param monitor: instance of Monitor :returns: a dictionary kind -> ProbabilityMap The "kind" is a string of the form 'rlz-XXX' or 'mean' of 'quantile-XXX' used to specify the kind of output. """ with monitor('read PoEs', measuremem=True): pgetter.init() if amplifier: with hdf5.File(pgetter.filename, 'r') as f: ampcode = f['sitecol'].ampcode imtls = DictArray({imt: amplifier.amplevels for imt in pgetter.imtls}) else: imtls = pgetter.imtls poes, weights = pgetter.poes, pgetter.weights M = len(imtls) P = len(poes) L = imtls.size R = len(weights) S = len(hstats) pmap_by_kind = {} if R > 1 and individual_rlzs or not hstats: pmap_by_kind['hcurves-rlzs'] = [ProbabilityMap(L) for r in range(R)] if poes: pmap_by_kind['hmaps-rlzs'] = [ ProbabilityMap(M, P) for r in range(R) ] if hstats: pmap_by_kind['hcurves-stats'] = [ProbabilityMap(L) for r in range(S)] if poes: pmap_by_kind['hmaps-stats'] = [ ProbabilityMap(M, P) for r in range(S) ] combine_mon = monitor('combine pmaps', measuremem=False) compute_mon = monitor('compute stats', measuremem=False) for sid in pgetter.sids: with combine_mon: pc = pgetter.get_pcurve(sid) # shape (L, R) if amplifier: pc = amplifier.amplify(ampcode[sid], pc) # NB: the pcurve have soil levels != IMT levels if pc.array.sum() == 0: # no data continue with compute_mon: if hstats: for s, (statname, stat) in enumerate(hstats.items()): sc = getters.build_stat_curve(pc, imtls, stat, weights) pmap_by_kind['hcurves-stats'][s][sid] = sc if poes: hmap = calc.make_hmap(sc, imtls, poes, sid) pmap_by_kind['hmaps-stats'][s].update(hmap) if R > 1 and individual_rlzs or not hstats: for r, pmap in enumerate(pmap_by_kind['hcurves-rlzs']): pmap[sid] = pc.extract(r) if poes: for r in range(R): hmap = calc.make_hmap(pc.extract(r), imtls, poes, sid) pmap_by_kind['hmaps-rlzs'][r].update(hmap) return pmap_by_kind