def plot(calc_id, other_id=None, sites='0'): """ Hazard curves plotter. """ # read the hazard data haz = engine.read(calc_id) other = engine.read(other_id) if other_id else None oq = haz['oqparam'] indices = numpy.array(list(map(int, sites.split(',')))) n_sites = len(haz['sitecol']) if not set(indices) <= set(range(n_sites)): invalid = sorted(set(indices) - set(range(n_sites))) print('The indices %s are invalid: no graph for them' % invalid) valid = sorted(set(range(n_sites)) & set(indices)) print('Found %d site(s); plotting %d of them' % (n_sites, len(valid))) if other is None: mean_curves, pmaps = get_pmaps(haz, indices) single_curve = len(pmaps) == 1 plt = make_figure(valid, n_sites, oq.imtls, mean_curves, [] if single_curve else pmaps, 'mean') else: mean1, _ = get_pmaps(haz, indices) mean2, _ = get_pmaps(other, indices) plt = make_figure(valid, n_sites, oq.imtls, mean1, [mean2], 'reference') plt.show()
def show(what='contents', calc_id=-1, extra=()): """ Show the content of a datastore (by default the last one). """ datadir = datastore.get_datadir() if what == 'all': # show all if not os.path.exists(datadir): return rows = [] for calc_id in datastore.get_calc_ids(datadir): try: ds = engine.read(calc_id) oq = ds['oqparam'] cmode, descr = oq.calculation_mode, oq.description except Exception: # invalid datastore file, or missing calculation_mode # and description attributes, perhaps due to a manual kill f = os.path.join(datadir, 'calc_%s.hdf5' % calc_id) logging.warn('Unreadable datastore %s', f) continue else: rows.append((calc_id, cmode, descr.encode('utf-8'))) for row in sorted(rows, key=lambda row: row[0]): # by calc_id print('#%d %s: %s' % row) return ds = engine.read(calc_id) # this part is experimental if what == 'rlzs' and 'poes' in ds: min_value = 0.01 # used in rmsep getter = getters.PmapGetter(ds) sitecol = ds['sitecol'] pmaps = getter.get_pmaps(sitecol.sids) weights = [rlz.weight for rlz in getter.rlzs] mean = stats.compute_pmap_stats(pmaps, [numpy.mean], weights) dists = [] for rlz, pmap in zip(getter.rlzs, pmaps): dist = rmsep(mean.array, pmap.array, min_value) dists.append((dist, rlz)) print('Realizations in order of distance from the mean curves') for dist, rlz in sorted(dists): print('%s: rmsep=%s' % (rlz, dist)) elif view.keyfunc(what) in view: print(view(what, ds)) elif what.split('/', 1)[0] in extract: print(extract(ds, what, *extra)) elif what in ds: obj = ds[what] if hasattr(obj, 'value'): # an array print(write_csv(io.BytesIO(), obj.value).decode('utf8')) else: print(obj) else: print('%s not found' % what) ds.close()
def checksum(thing): """ Get the checksum of a calculation from the calculation ID (if already done) or from the job.ini/job.zip file (if not done yet). If `thing` is a source model logic tree file, get the checksum of the model by ignoring the job.ini, the gmpe logic tree file and possibly other files. """ try: job_id = int(thing) job_file = None except ValueError: job_id = None job_file = thing if not os.path.exists(job_file): sys.exit('%s does not correspond to an existing file' % job_file) if job_id: dstore = engine.read(job_id) checksum = dstore['/'].attrs['checksum32'] elif job_file.endswith('.xml'): # assume it is a smlt file inputs = {'source_model_logic_tree': job_file} checksum = readinput.get_checksum32(mock.Mock(inputs=inputs)) else: oq = readinput.get_oqparam(job_file) checksum = readinput.get_checksum32(oq) print(checksum)
def export(datastore_key, calc_id=-1, exports='csv', export_dir='.'): """ Export an output from the datastore. """ logging.basicConfig(level=logging.INFO) dstore = engine.read(calc_id) parent_id = dstore['oqparam'].hazard_calculation_id if parent_id: dstore.parent = engine.read(parent_id) dstore.export_dir = export_dir with performance.Monitor('export', measuremem=True) as mon: for fmt in exports.split(','): fnames = export_((datastore_key, fmt), dstore) nbytes = sum(os.path.getsize(f) for f in fnames) print('Exported %s in %s' % (general.humansize(nbytes), fnames)) if mon.duration > 1: print(mon) dstore.close()
def plot_ac(calc_id): """ Aggregate loss curves plotter. """ # read the hazard data dstore = engine.read(calc_id) agg_curve = dstore['agg_curve-rlzs'] plt = make_figure(agg_curve) plt.show()
def plot_losses(calc_id, bins=7): """ losses_by_event plotter """ # read the hazard data dstore = engine.read(calc_id) losses_by_rlzi = dict(extract(dstore, 'losses_by_event')) oq = dstore['oqparam'] plt = make_figure(losses_by_rlzi, oq.loss_dt().names, bins) plt.show()
def plot_memory(calc_id=-1): """ Plot the memory occupation """ dstore = engine.read(calc_id) plots = [] for task_name in dstore['task_info']: mem = dstore['task_info/' + task_name]['mem_gb'] plots.append((task_name, mem)) plt = make_figure(plots) plt.show()
def plot_lc(calc_id, aid=None): """ Plot loss curves given a calculation id and an asset ordinal. """ # read the hazard data dstore = engine.read(calc_id) dset = dstore['agg_curves-rlzs'] if aid is None: # plot the global curves plt = make_figure(dset.attrs['return_periods'], dset.value) else: sys.exit('Not implemented yet') plt.show()
def plot_hmaps(calc_id): """ Mean hazard maps plotter. """ dstore = engine.read(calc_id) oq = dstore['oqparam'] rlzs_assoc = dstore['csm_info'].get_rlzs_assoc() mean = getters.PmapGetter(dstore, rlzs_assoc).get_mean() hmaps = calc.make_hmap(mean, oq.imtls, oq.poes) M, P = len(oq.imtls), len(oq.poes) array = hmaps.array.reshape(len(hmaps.array), M, P) plt = make_figure(dstore['sitecol'], oq.imtls, oq.poes, array) plt.show()
def extract(what, calc_id=-1, server_url='http://127.0.0.1:8800'): """ Extract an output from the datastore and save it into an .hdf5 file. By default use the WebUI, unless server_url is set to 'local', in which case the extraction is done directly bypassing tjhe WebUI and the database. """ logging.basicConfig(level=logging.INFO) if calc_id < 0: calc_id = datastore.get_calc_ids()[calc_id] hdf5path = None if dbserver.get_status() == 'running': job = dbcmd('get_job', calc_id) if job is not None: hdf5path = job.ds_calc_dir + '.hdf5' dstore = engine.read(hdf5path or calc_id) parent_id = dstore['oqparam'].hazard_calculation_id if parent_id: dstore.parent = engine.read(parent_id) urlpath = '/v1/calc/%d/extract/%s' % (calc_id, quote(what)) with performance.Monitor('extract', measuremem=True) as mon, dstore: if server_url == 'local': print('Emulating call to %s' % urlpath) items = extract_(dstore, what) else: print('Calling %s%s' % (server_url, urlpath)) data = urlopen(server_url.rstrip('/') + urlpath).read() items = (item for item in numpy.load(io.BytesIO(data)).items()) if not inspect.isgenerator(items): items = [(items.__class__.__name__, items)] fname = '%s_%d.hdf5' % (what.replace('/', '-').replace('?', '-'), dstore.calc_id) hdf5.save(fname, items) print('Saved', fname) if mon.duration > 1: print(mon)
def show_attrs(key, calc_id=-1): """ Show the attributes of a HDF5 dataset in the datastore. """ ds = engine.read(calc_id) try: attrs = h5py.File.__getitem__(ds.hdf5, key).attrs except KeyError: print('%r is not in %s' % (key, ds)) else: if len(attrs) == 0: print('%s has no attributes' % key) for name, value in attrs.items(): print(name, value) finally: ds.close()
def plot_uhs(calc_id, sites='0'): """ UHS plotter. """ # read the hazard data dstore = engine.read(calc_id) rlzs_assoc = dstore['csm_info'].get_rlzs_assoc() getter = getters.PmapGetter(dstore, rlzs_assoc) getter.init() oq = dstore['oqparam'] indices = list(map(int, sites.split(','))) n_sites = len(dstore['sitecol']) if not set(indices) <= set(range(n_sites)): invalid = sorted(set(indices) - set(range(n_sites))) print('The indices %s are invalid: no graph for them' % invalid) valid = sorted(set(range(n_sites)) & set(indices)) print('Found %d site(s); plotting %d of them' % (n_sites, len(valid))) pmaps = getter.get_pmaps(numpy.array(indices)) plt = make_figure(valid, n_sites, oq.imtls, oq.poes, pmaps) plt.show()
def plot_assets(calc_id=-1): """ Plot the sites and the assets """ # NB: matplotlib is imported inside since it is a costly import import matplotlib.pyplot as p from openquake.hmtk.plotting.patch import PolygonPatch dstore = engine.read(calc_id) try: region = dstore['oqparam'].region except KeyError: region = None sitecol = dstore['sitecol'] try: assetcol = dstore['assetcol'].value except AttributeError: assetcol = dstore['assetcol'].array fig = p.figure() ax = fig.add_subplot(111) if region: pp = PolygonPatch(shapely.wkt.loads(region), alpha=0.1) ax.add_patch(pp) ax.grid(True) if 'site_model' in dstore: sm = dstore['site_model'] sm_lons, sm_lats = sm['lon'], sm['lat'] if len(sm_lons) > 1 and cross_idl(*sm_lons): sm_lons %= 360 p.scatter(sm_lons, sm_lats, marker='.', color='orange') p.scatter(sitecol.complete.lons, sitecol.complete.lats, marker='.', color='gray') p.scatter(assetcol['lon'], assetcol['lat'], marker='.', color='green') p.scatter(sitecol.lons, sitecol.lats, marker='+', color='black') if 'discarded' in dstore: disc = numpy.unique(dstore['discarded'].value[['lon', 'lat']]) p.scatter(disc['lon'], disc['lat'], marker='x', color='red') p.show()
def getdata(what, calc_ids, samplesites): dstores = [engine.read(calc_id) for calc_id in calc_ids] dstore = dstores[0] sitecol = dstore['sitecol'] oq = dstore['oqparam'] imtls = oq.imtls poes = oq.poes if len(sitecol) > samplesites: numpy.random.seed(samplesites) sids = numpy.random.choice(len(sitecol), samplesites, replace=False) else: # keep all sites sids = sitecol.sids arrays = [get(dstore, what, imtls, sids)] dstore.close() for dstore in dstores[1:]: oq = dstore['oqparam'] numpy.testing.assert_equal(dstore['sitecol'].array, sitecol.array) numpy.testing.assert_equal(oq.imtls.array, imtls.array) numpy.testing.assert_equal(oq.poes, poes) arrays.append(get(dstore, what, imtls, sids)) dstore.close() return sids, imtls, poes, numpy.array(arrays) # shape (C, N, L)
def plot_sites(calc_id=-1): """ Plot the sites and the bounding boxes of the sources, enlarged by the maximum distance """ # NB: matplotlib is imported inside since it is a costly import import matplotlib.pyplot as p logging.basicConfig(level=logging.INFO) dstore = engine.read(calc_id) sitecol = dstore['sitecol'] lons, lats = sitecol.lons, sitecol.lats if len(lons) > 1 and cross_idl(*lons): lons %= 360 fig, ax = p.subplots() ax.grid(True) if 'site_model' in dstore: sm = dstore['site_model'] sm_lons, sm_lats = sm['lon'], sm['lat'] if len(sm_lons) > 1 and cross_idl(*sm_lons): sm_lons %= 360 p.scatter(sm_lons, sm_lats, marker='.', color='orange') p.scatter(lons, lats, marker='+') p.show()