def plot(calc_id, other_id=None, sites='0'):
"""
Hazard curves plotter.
"""
# read the hazard data
haz = engine.read(calc_id)
other = engine.read(other_id) if other_id else None
oq = haz['oqparam']
indices = numpy.array(list(map(int, sites.split(','))))
n_sites = len(haz['sitecol'])
if not set(indices) <= set(range(n_sites)):
invalid = sorted(set(indices) - set(range(n_sites)))
print('The indices %s are invalid: no graph for them' % invalid)
valid = sorted(set(range(n_sites)) & set(indices))
print('Found %d site(s); plotting %d of them' % (n_sites, len(valid)))
if other is None:
mean_curves, pmaps = get_pmaps(haz, indices)
single_curve = len(pmaps) == 1
plt = make_figure(valid, n_sites, oq.imtls, mean_curves,
[] if single_curve else pmaps, 'mean')
else:
mean1, _ = get_pmaps(haz, indices)
mean2, _ = get_pmaps(other, indices)
plt = make_figure(valid, n_sites, oq.imtls, mean1, [mean2],
'reference')
plt.show()
def show(what='contents', calc_id=-1, extra=()):
"""
Show the content of a datastore (by default the last one).
"""
datadir = datastore.get_datadir()
if what == 'all': # show all
if not os.path.exists(datadir):
return
rows = []
for calc_id in datastore.get_calc_ids(datadir):
try:
ds = engine.read(calc_id)
oq = ds['oqparam']
cmode, descr = oq.calculation_mode, oq.description
except Exception:
# invalid datastore file, or missing calculation_mode
# and description attributes, perhaps due to a manual kill
f = os.path.join(datadir, 'calc_%s.hdf5' % calc_id)
logging.warn('Unreadable datastore %s', f)
continue
else:
rows.append((calc_id, cmode, descr.encode('utf-8')))
for row in sorted(rows, key=lambda row: row[0]): # by calc_id
print('#%d %s: %s' % row)
return
ds = engine.read(calc_id)
# this part is experimental
if what == 'rlzs' and 'poes' in ds:
min_value = 0.01 # used in rmsep
getter = getters.PmapGetter(ds)
sitecol = ds['sitecol']
pmaps = getter.get_pmaps(sitecol.sids)
weights = [rlz.weight for rlz in getter.rlzs]
mean = stats.compute_pmap_stats(pmaps, [numpy.mean], weights)
dists = []
for rlz, pmap in zip(getter.rlzs, pmaps):
dist = rmsep(mean.array, pmap.array, min_value)
dists.append((dist, rlz))
print('Realizations in order of distance from the mean curves')
for dist, rlz in sorted(dists):
print('%s: rmsep=%s' % (rlz, dist))
elif view.keyfunc(what) in view:
print(view(what, ds))
elif what.split('/', 1)[0] in extract:
print(extract(ds, what, *extra))
elif what in ds:
obj = ds[what]
if hasattr(obj, 'value'): # an array
print(write_csv(io.BytesIO(), obj.value).decode('utf8'))
else:
print(obj)
else:
print('%s not found' % what)
ds.close()
def checksum(thing):
"""
Get the checksum of a calculation from the calculation ID (if already
done) or from the job.ini/job.zip file (if not done yet). If `thing`
is a source model logic tree file, get the checksum of the model by
ignoring the job.ini, the gmpe logic tree file and possibly other files.
"""
try:
job_id = int(thing)
job_file = None
except ValueError:
job_id = None
job_file = thing
if not os.path.exists(job_file):
sys.exit('%s does not correspond to an existing file' % job_file)
if job_id:
dstore = engine.read(job_id)
checksum = dstore['/'].attrs['checksum32']
elif job_file.endswith('.xml'): # assume it is a smlt file
inputs = {'source_model_logic_tree': job_file}
checksum = readinput.get_checksum32(mock.Mock(inputs=inputs))
else:
oq = readinput.get_oqparam(job_file)
checksum = readinput.get_checksum32(oq)
print(checksum)
def export(datastore_key, calc_id=-1, exports='csv', export_dir='.'):
"""
Export an output from the datastore.
"""
logging.basicConfig(level=logging.INFO)
dstore = engine.read(calc_id)
parent_id = dstore['oqparam'].hazard_calculation_id
if parent_id:
dstore.parent = engine.read(parent_id)
dstore.export_dir = export_dir
with performance.Monitor('export', measuremem=True) as mon:
for fmt in exports.split(','):
fnames = export_((datastore_key, fmt), dstore)
nbytes = sum(os.path.getsize(f) for f in fnames)
print('Exported %s in %s' % (general.humansize(nbytes), fnames))
if mon.duration > 1:
print(mon)
dstore.close()
def plot_ac(calc_id):
"""
Aggregate loss curves plotter.
"""
# read the hazard data
dstore = engine.read(calc_id)
agg_curve = dstore['agg_curve-rlzs']
plt = make_figure(agg_curve)
plt.show()
def plot_losses(calc_id, bins=7):
"""
losses_by_event plotter
"""
# read the hazard data
dstore = engine.read(calc_id)
losses_by_rlzi = dict(extract(dstore, 'losses_by_event'))
oq = dstore['oqparam']
plt = make_figure(losses_by_rlzi, oq.loss_dt().names, bins)
plt.show()
def plot_memory(calc_id=-1):
"""
Plot the memory occupation
"""
dstore = engine.read(calc_id)
plots = []
for task_name in dstore['task_info']:
mem = dstore['task_info/' + task_name]['mem_gb']
plots.append((task_name, mem))
plt = make_figure(plots)
plt.show()
def plot_lc(calc_id, aid=None):
"""
Plot loss curves given a calculation id and an asset ordinal.
"""
# read the hazard data
dstore = engine.read(calc_id)
dset = dstore['agg_curves-rlzs']
if aid is None: # plot the global curves
plt = make_figure(dset.attrs['return_periods'], dset.value)
else:
sys.exit('Not implemented yet')
plt.show()
def plot_hmaps(calc_id):
"""
Mean hazard maps plotter.
"""
dstore = engine.read(calc_id)
oq = dstore['oqparam']
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
mean = getters.PmapGetter(dstore, rlzs_assoc).get_mean()
hmaps = calc.make_hmap(mean, oq.imtls, oq.poes)
M, P = len(oq.imtls), len(oq.poes)
array = hmaps.array.reshape(len(hmaps.array), M, P)
plt = make_figure(dstore['sitecol'], oq.imtls, oq.poes, array)
plt.show()
def extract(what, calc_id=-1, server_url='http://127.0.0.1:8800'):
"""
Extract an output from the datastore and save it into an .hdf5 file.
By default use the WebUI, unless server_url is set to 'local', in
which case the extraction is done directly bypassing tjhe WebUI and
the database.
"""
logging.basicConfig(level=logging.INFO)
if calc_id < 0:
calc_id = datastore.get_calc_ids()[calc_id]
hdf5path = None
if dbserver.get_status() == 'running':
job = dbcmd('get_job', calc_id)
if job is not None:
hdf5path = job.ds_calc_dir + '.hdf5'
dstore = engine.read(hdf5path or calc_id)
parent_id = dstore['oqparam'].hazard_calculation_id
if parent_id:
dstore.parent = engine.read(parent_id)
urlpath = '/v1/calc/%d/extract/%s' % (calc_id, quote(what))
with performance.Monitor('extract', measuremem=True) as mon, dstore:
if server_url == 'local':
print('Emulating call to %s' % urlpath)
items = extract_(dstore, what)
else:
print('Calling %s%s' % (server_url, urlpath))
data = urlopen(server_url.rstrip('/') + urlpath).read()
items = (item for item in numpy.load(io.BytesIO(data)).items())
if not inspect.isgenerator(items):
items = [(items.__class__.__name__, items)]
fname = '%s_%d.hdf5' % (what.replace('/', '-').replace('?', '-'),
dstore.calc_id)
hdf5.save(fname, items)
print('Saved', fname)
if mon.duration > 1:
print(mon)
def show_attrs(key, calc_id=-1):
"""
Show the attributes of a HDF5 dataset in the datastore.
"""
ds = engine.read(calc_id)
try:
attrs = h5py.File.__getitem__(ds.hdf5, key).attrs
except KeyError:
print('%r is not in %s' % (key, ds))
else:
if len(attrs) == 0:
print('%s has no attributes' % key)
for name, value in attrs.items():
print(name, value)
finally:
ds.close()
def plot_uhs(calc_id, sites='0'):
"""
UHS plotter.
"""
# read the hazard data
dstore = engine.read(calc_id)
rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
getter = getters.PmapGetter(dstore, rlzs_assoc)
getter.init()
oq = dstore['oqparam']
indices = list(map(int, sites.split(',')))
n_sites = len(dstore['sitecol'])
if not set(indices) <= set(range(n_sites)):
invalid = sorted(set(indices) - set(range(n_sites)))
print('The indices %s are invalid: no graph for them' % invalid)
valid = sorted(set(range(n_sites)) & set(indices))
print('Found %d site(s); plotting %d of them' % (n_sites, len(valid)))
pmaps = getter.get_pmaps(numpy.array(indices))
plt = make_figure(valid, n_sites, oq.imtls, oq.poes, pmaps)
plt.show()
def plot_assets(calc_id=-1):
"""
Plot the sites and the assets
"""
# NB: matplotlib is imported inside since it is a costly import
import matplotlib.pyplot as p
from openquake.hmtk.plotting.patch import PolygonPatch
dstore = engine.read(calc_id)
try:
region = dstore['oqparam'].region
except KeyError:
region = None
sitecol = dstore['sitecol']
try:
assetcol = dstore['assetcol'].value
except AttributeError:
assetcol = dstore['assetcol'].array
fig = p.figure()
ax = fig.add_subplot(111)
if region:
pp = PolygonPatch(shapely.wkt.loads(region), alpha=0.1)
ax.add_patch(pp)
ax.grid(True)
if 'site_model' in dstore:
sm = dstore['site_model']
sm_lons, sm_lats = sm['lon'], sm['lat']
if len(sm_lons) > 1 and cross_idl(*sm_lons):
sm_lons %= 360
p.scatter(sm_lons, sm_lats, marker='.', color='orange')
p.scatter(sitecol.complete.lons,
sitecol.complete.lats,
marker='.',
color='gray')
p.scatter(assetcol['lon'], assetcol['lat'], marker='.', color='green')
p.scatter(sitecol.lons, sitecol.lats, marker='+', color='black')
if 'discarded' in dstore:
disc = numpy.unique(dstore['discarded'].value[['lon', 'lat']])
p.scatter(disc['lon'], disc['lat'], marker='x', color='red')
p.show()
def getdata(what, calc_ids, samplesites):
dstores = [engine.read(calc_id) for calc_id in calc_ids]
dstore = dstores[0]
sitecol = dstore['sitecol']
oq = dstore['oqparam']
imtls = oq.imtls
poes = oq.poes
if len(sitecol) > samplesites:
numpy.random.seed(samplesites)
sids = numpy.random.choice(len(sitecol), samplesites, replace=False)
else: # keep all sites
sids = sitecol.sids
arrays = [get(dstore, what, imtls, sids)]
dstore.close()
for dstore in dstores[1:]:
oq = dstore['oqparam']
numpy.testing.assert_equal(dstore['sitecol'].array, sitecol.array)
numpy.testing.assert_equal(oq.imtls.array, imtls.array)
numpy.testing.assert_equal(oq.poes, poes)
arrays.append(get(dstore, what, imtls, sids))
dstore.close()
return sids, imtls, poes, numpy.array(arrays) # shape (C, N, L)
def plot_sites(calc_id=-1):
"""
Plot the sites and the bounding boxes of the sources, enlarged by
the maximum distance
"""
# NB: matplotlib is imported inside since it is a costly import
import matplotlib.pyplot as p
logging.basicConfig(level=logging.INFO)
dstore = engine.read(calc_id)
sitecol = dstore['sitecol']
lons, lats = sitecol.lons, sitecol.lats
if len(lons) > 1 and cross_idl(*lons):
lons %= 360
fig, ax = p.subplots()
ax.grid(True)
if 'site_model' in dstore:
sm = dstore['site_model']
sm_lons, sm_lats = sm['lon'], sm['lat']
if len(sm_lons) > 1 and cross_idl(*sm_lons):
sm_lons %= 360
p.scatter(sm_lons, sm_lats, marker='.', color='orange')
p.scatter(lons, lats, marker='+')
p.show()
