def extract(what, calc_id=-1, hostport=None):
"""
Extract an output from the datastore and save it into an .hdf5 file.
"""
logging.basicConfig(level=logging.INFO)
if calc_id < 0:
calc_id = datastore.get_calc_ids()[calc_id]
hdf5path = None
if dbserver.get_status() == 'running':
job = dbcmd('get_job', calc_id)
if job is not None:
hdf5path = job.ds_calc_dir + '.hdf5'
dstore = datastore.read(hdf5path or calc_id)
parent_id = dstore['oqparam'].hazard_calculation_id
if parent_id:
dstore.parent = datastore.read(parent_id)
urlpath = '/v1/calc/%d/extract/%s' % (calc_id, quote(what))
with performance.Monitor('extract', measuremem=True) as mon, dstore:
if hostport:
data = urlopen('http://%s%s' % (hostport, urlpath)).read()
items = (item for item in numpy.load(io.BytesIO(data)).items())
else:
print('Emulating call to %s' % urlpath)
items = extract_(dstore, what)
if not inspect.isgenerator(items):
items = [(items.__class__.__name__, items)]
fname = '%s_%d.hdf5' % (what.replace('/', '-').replace(
'?', '-'), dstore.calc_id)
hdf5.save(fname, items)
print('Saved', fname)
if mon.duration > 1:
print(mon)
def extract(what, calc_id=-1):
"""
Extract an output from the datastore and save it into an .hdf5 file.
"""
logging.basicConfig(level=logging.INFO)
if calc_id < 0:
calc_id = datastore.get_calc_ids()[calc_id]
if dbserver.get_status() == 'running':
job = dbcmd('get_job', calc_id)
if job is not None:
calc_id = job.ds_calc_dir + '.hdf5'
dstore = datastore.read(calc_id)
parent_id = dstore['oqparam'].hazard_calculation_id
if parent_id:
dstore.parent = datastore.read(parent_id)
print('Emulating call to /v1/calc/%s/extract/%s' % (calc_id, quote(what)))
with performance.Monitor('extract', measuremem=True) as mon, dstore:
items = extract_(dstore, what)
if not inspect.isgenerator(items):
items = [(items.__class__.__name__, items)]
fname = '%s_%d.hdf5' % (what.replace('/', '-').replace(
'?', '-'), dstore.calc_id)
hdf5.save(fname, items)
print('Saved', fname)
if mon.duration > 1:
print(mon)
def show(what='contents', calc_id=-1, extra=()):
"""
Show the content of a datastore (by default the last one).
"""
datadir = datastore.get_datadir()
if what == 'all': # show all
if not os.path.exists(datadir):
return
rows = []
for calc_id in datastore.get_calc_ids(datadir):
try:
ds = read(calc_id)
oq = ds['oqparam']
cmode, descr = oq.calculation_mode, oq.description
except:
# invalid datastore file, or missing calculation_mode
# and description attributes, perhaps due to a manual kill
f = os.path.join(datadir, 'calc_%s.hdf5' % calc_id)
logging.warn('Unreadable datastore %s', f)
continue
else:
rows.append((calc_id, cmode, descr.encode('utf-8')))
for row in sorted(rows, key=lambda row: row[0]): # by calc_id
print('#%d %s: %s' % row)
return
ds = read(calc_id)
# this part is experimental
if what == 'rlzs' and 'poes' in ds:
min_value = 0.01 # used in rmsep
getter = getters.PmapGetter(ds)
sitecol = ds['sitecol']
pmaps = getter.get_pmaps(sitecol.sids)
weights = [rlz.weight for rlz in getter.rlzs]
mean = stats.compute_pmap_stats(pmaps, [numpy.mean], weights)
dists = []
for rlz, pmap in zip(getter.rlzs, pmaps):
dist = rmsep(mean.array, pmap.array, min_value)
dists.append((dist, rlz))
print('Realizations in order of distance from the mean curves')
for dist, rlz in sorted(dists):
print('%s: rmsep=%s' % (rlz, dist))
elif view.keyfunc(what) in view:
print(view(what, ds))
elif what.split('/', 1)[0] in extract:
print(extract(ds, what, *extra))
elif what in ds:
obj = ds[what]
if hasattr(obj, 'value'): # an array
print(write_csv(io.BytesIO(), obj.value).decode('utf8'))
else:
print(obj)
else:
print('%s not found' % what)
ds.close()
def purge(calc_id, force=False):
"""
Remove the given calculation. If you want to remove all calculations,
use oq reset.
"""
if calc_id < 0:
try:
calc_id = datastore.get_calc_ids(datadir)[calc_id]
except IndexError:
print('Calculation %d not found' % calc_id)
return
purge_one(calc_id, getpass.getuser(), force)
def purge(calc_id):
"""
Remove the given calculation. If you want to remove all calculations,
use oq reset.
"""
if calc_id < 0:
try:
calc_id = datastore.get_calc_ids(datadir)[calc_id]
except IndexError:
print('Calculation %d not found' % calc_id)
return
purge_one(calc_id, getpass.getuser())
def show(what='contents', calc_id=-1, extra=()):
"""
Show the content of a datastore (by default the last one).
"""
datadir = datastore.get_datadir()
if what == 'all': # show all
if not os.path.exists(datadir):
return
rows = []
for calc_id in datastore.get_calc_ids(datadir):
try:
ds = util.read(calc_id)
oq = ds['oqparam']
cmode, descr = oq.calculation_mode, oq.description
except Exception:
# invalid datastore file, or missing calculation_mode
# and description attributes, perhaps due to a manual kill
f = os.path.join(datadir, 'calc_%s.hdf5' % calc_id)
logging.warning('Unreadable datastore %s', f)
continue
else:
rows.append((calc_id, cmode, descr.encode('utf-8')))
for row in sorted(rows, key=lambda row: row[0]): # by calc_id
print('#%d %s: %s' % row)
return
ds = util.read(calc_id)
# this part is experimental
if view.keyfunc(what) in view:
print(view(what, ds))
elif what.split('/', 1)[0] in extract:
obj = extract(ds, what, *extra)
if hasattr(obj, 'dtype') and obj.dtype.names:
print(write_csv(io.BytesIO(), obj).decode('utf8'))
else:
print(obj)
elif what in ds:
obj = ds.getitem(what)
if hasattr(obj, 'items'): # is a group of datasets
print(obj)
else: # is a single dataset
obj.refresh() # for SWMR mode
aw = hdf5.ArrayWrapper.from_(obj)
if hasattr(aw, 'shape_descr'):
print(rst_table(aw.to_table()))
else:
print(write_csv(io.BytesIO(), aw.array).decode('utf8'))
else:
print('%s not found' % what)
ds.close()
def _run(job_inis, concurrent_tasks, calc_id, pdb, loglevel, hc, exports,
params):
global calc_path
assert len(job_inis) in (1, 2), job_inis
# set the logs first of all
calc_id = logs.init(calc_id, getattr(logging, loglevel.upper()))
# disable gzip_input
base.BaseCalculator.gzip_inputs = lambda self: None
with performance.Monitor('total runtime', measuremem=True) as monitor:
if os.environ.get('OQ_DISTRIBUTE') not in ('no', 'processpool'):
os.environ['OQ_DISTRIBUTE'] = 'processpool'
if len(job_inis) == 1: # run hazard or risk
if hc:
hc_id = hc[0]
rlz_ids = hc[1:]
else:
hc_id = None
rlz_ids = ()
oqparam = readinput.get_oqparam(job_inis[0], hc_id=hc_id)
if not oqparam.cachedir: # enable caching
oqparam.cachedir = datastore.get_datadir()
if hc_id and hc_id < 0: # interpret negative calculation ids
calc_ids = datastore.get_calc_ids()
try:
hc_id = calc_ids[hc_id]
except IndexError:
raise SystemExit('There are %d old calculations, cannot '
'retrieve the %s' %
(len(calc_ids), hc_id))
calc = base.calculators(oqparam, calc_id)
calc.run(concurrent_tasks=concurrent_tasks,
pdb=pdb,
exports=exports,
hazard_calculation_id=hc_id,
rlz_ids=rlz_ids,
**params)
else: # run hazard + risk
calc = run2(job_inis[0], job_inis[1], calc_id, concurrent_tasks,
pdb, loglevel, exports, params)
logging.info('Total time spent: %s s', monitor.duration)
logging.info('Memory allocated: %s', general.humansize(monitor.mem))
print('See the output with silx view %s' % calc.datastore.filename)
calc_path, _ = os.path.splitext(calc.datastore.filename) # used below
return calc
def get_calc_id(db, datadir, job_id=None):
"""
Return the latest calc_id by looking both at the datastore
and the database.
:param db: a :class:`openquake.server.dbapi.Db` instance
:param datadir: the directory containing the datastores
:param job_id: a job ID; if None, returns the latest job ID
"""
calcs = datastore.get_calc_ids(datadir)
calc_id = 0 if not calcs else calcs[-1]
if job_id is None:
try:
job_id = db('SELECT seq FROM sqlite_sequence WHERE name="job"',
scalar=True)
except NotFound:
job_id = 0
return max(calc_id, job_id)
def _run(job_ini, concurrent_tasks, pdb, loglevel, hc, exports, params):
global calc_path
logging.basicConfig(level=getattr(logging, loglevel.upper()))
job_inis = job_ini.split(',')
assert len(job_inis) in (1, 2), job_inis
monitor = performance.Monitor('complete runtime', measuremem=True)
if len(job_inis) == 1: # run hazard or risk
if hc:
hc_id = hc[0]
rlz_ids = hc[1:]
else:
hc_id = None
rlz_ids = ()
oqparam = readinput.get_oqparam(job_inis[0], hc_id=hc_id)
if hc_id and hc_id < 0: # interpret negative calculation ids
calc_ids = datastore.get_calc_ids()
try:
hc_id = calc_ids[hc_id]
except IndexError:
raise SystemExit('There are %d old calculations, cannot '
'retrieve the %s' % (len(calc_ids), hc_id))
calc = base.calculators(oqparam, monitor)
with calc._monitor:
calc.run(concurrent_tasks=concurrent_tasks,
pdb=pdb,
exports=exports,
hazard_calculation_id=hc_id,
rlz_ids=rlz_ids,
**params)
else: # run hazard + risk
calc = run2(job_inis[0], job_inis[1], concurrent_tasks, pdb, exports,
params, monitor)
logging.info('Total time spent: %s s', monitor.duration)
logging.info('Memory allocated: %s', general.humansize(monitor.mem))
monitor.flush()
print('See the output with hdfview %s' % calc.datastore.hdf5path)
calc_path = calc.datastore.calc_dir # used for the .pstat filename
return calc
def _run(job_inis, concurrent_tasks, pdb, loglevel, hc, exports, params):
global calc_path
assert len(job_inis) in (1, 2), job_inis
# set the logs first of all
calc_id = logs.init(level=getattr(logging, loglevel.upper()))
with performance.Monitor('total runtime', measuremem=True) as monitor:
if len(job_inis) == 1: # run hazard or risk
if hc:
hc_id = hc[0]
rlz_ids = hc[1:]
else:
hc_id = None
rlz_ids = ()
oqparam = readinput.get_oqparam(job_inis[0], hc_id=hc_id)
vars(oqparam).update(params)
if hc_id and hc_id < 0: # interpret negative calculation ids
calc_ids = datastore.get_calc_ids()
try:
hc_id = calc_ids[hc_id]
except IndexError:
raise SystemExit('There are %d old calculations, cannot '
'retrieve the %s' %
(len(calc_ids), hc_id))
calc = base.calculators(oqparam, calc_id)
calc.run(concurrent_tasks=concurrent_tasks,
pdb=pdb,
exports=exports,
hazard_calculation_id=hc_id,
rlz_ids=rlz_ids)
else: # run hazard + risk
calc = run2(job_inis[0], job_inis[1], calc_id, concurrent_tasks,
pdb, loglevel, exports, params)
logging.info('Total time spent: %s s', monitor.duration)
logging.info('Memory allocated: %s', general.humansize(monitor.mem))
print('See the output with hdfview %s' % calc.datastore.hdf5path)
calc_path, _ = os.path.splitext(calc.datastore.hdf5path) # used below
return calc
def _run(job_inis, concurrent_tasks, pdb, loglevel, hc, exports, params):
global calc_path
assert len(job_inis) in (1, 2), job_inis
# set the logs first of all
calc_id = logs.init(level=getattr(logging, loglevel.upper()))
with performance.Monitor('total runtime', measuremem=True) as monitor:
if len(job_inis) == 1: # run hazard or risk
if hc:
hc_id = hc[0]
rlz_ids = hc[1:]
else:
hc_id = None
rlz_ids = ()
oqparam = readinput.get_oqparam(job_inis[0], hc_id=hc_id)
vars(oqparam).update(params)
if hc_id and hc_id < 0: # interpret negative calculation ids
calc_ids = datastore.get_calc_ids()
try:
hc_id = calc_ids[hc_id]
except IndexError:
raise SystemExit(
'There are %d old calculations, cannot '
'retrieve the %s' % (len(calc_ids), hc_id))
calc = base.calculators(oqparam, calc_id)
calc.run(concurrent_tasks=concurrent_tasks, pdb=pdb,
exports=exports, hazard_calculation_id=hc_id,
rlz_ids=rlz_ids)
else: # run hazard + risk
calc = run2(
job_inis[0], job_inis[1], calc_id, concurrent_tasks, pdb,
loglevel, exports, params)
logging.info('Total time spent: %s s', monitor.duration)
logging.info('Memory allocated: %s', general.humansize(monitor.mem))
print('See the output with silx view %s' % calc.datastore.filename)
calc_path, _ = os.path.splitext(calc.datastore.filename) # used below
return calc
def extract(what, calc_id=-1, server_url='http://127.0.0.1:8800'):
"""
Extract an output from the datastore and save it into an .hdf5 file.
By default use the WebUI, unless server_url is set to 'local', in
which case the extraction is done directly bypassing tjhe WebUI and
the database.
"""
logging.basicConfig(level=logging.INFO)
if calc_id < 0:
calc_id = datastore.get_calc_ids()[calc_id]
filename = None
if dbserver.get_status() == 'running':
job = dbcmd('get_job', calc_id)
if job is not None:
filename = job.ds_calc_dir + '.hdf5'
dstore = util.read(filename or calc_id)
parent_id = dstore['oqparam'].hazard_calculation_id
if parent_id:
dstore.parent = util.read(parent_id)
urlpath = '/v1/calc/%d/extract/%s' % (calc_id, quote(what))
with performance.Monitor('extract', measuremem=True) as mon, dstore:
if server_url == 'local':
print('Emulating call to %s' % urlpath)
items = extract_(dstore, what)
else:
print('Calling %s%s' % (server_url, urlpath))
data = urlopen(server_url.rstrip('/') + urlpath).read()
items = (item for item in numpy.load(io.BytesIO(data)).items())
if not inspect.isgenerator(items):
items = [(items.__class__.__name__, items)]
fname = '%s_%d.hdf5' % (what.replace('/', '-').replace(
'?', '-'), dstore.calc_id)
hdf5.save(fname, items)
print('Saved', fname)
if mon.duration > 1:
print(mon)