def extract(what, calc_id=-1, hostport=None): """ Extract an output from the datastore and save it into an .hdf5 file. """ logging.basicConfig(level=logging.INFO) if calc_id < 0: calc_id = datastore.get_calc_ids()[calc_id] hdf5path = None if dbserver.get_status() == 'running': job = dbcmd('get_job', calc_id) if job is not None: hdf5path = job.ds_calc_dir + '.hdf5' dstore = datastore.read(hdf5path or calc_id) parent_id = dstore['oqparam'].hazard_calculation_id if parent_id: dstore.parent = datastore.read(parent_id) urlpath = '/v1/calc/%d/extract/%s' % (calc_id, quote(what)) with performance.Monitor('extract', measuremem=True) as mon, dstore: if hostport: data = urlopen('http://%s%s' % (hostport, urlpath)).read() items = (item for item in numpy.load(io.BytesIO(data)).items()) else: print('Emulating call to %s' % urlpath) items = extract_(dstore, what) if not inspect.isgenerator(items): items = [(items.__class__.__name__, items)] fname = '%s_%d.hdf5' % (what.replace('/', '-').replace( '?', '-'), dstore.calc_id) hdf5.save(fname, items) print('Saved', fname) if mon.duration > 1: print(mon)
def extract(what, calc_id=-1): """ Extract an output from the datastore and save it into an .hdf5 file. """ logging.basicConfig(level=logging.INFO) if calc_id < 0: calc_id = datastore.get_calc_ids()[calc_id] if dbserver.get_status() == 'running': job = dbcmd('get_job', calc_id) if job is not None: calc_id = job.ds_calc_dir + '.hdf5' dstore = datastore.read(calc_id) parent_id = dstore['oqparam'].hazard_calculation_id if parent_id: dstore.parent = datastore.read(parent_id) print('Emulating call to /v1/calc/%s/extract/%s' % (calc_id, quote(what))) with performance.Monitor('extract', measuremem=True) as mon, dstore: items = extract_(dstore, what) if not inspect.isgenerator(items): items = [(items.__class__.__name__, items)] fname = '%s_%d.hdf5' % (what.replace('/', '-').replace( '?', '-'), dstore.calc_id) hdf5.save(fname, items) print('Saved', fname) if mon.duration > 1: print(mon)
def show(what='contents', calc_id=-1, extra=()): """ Show the content of a datastore (by default the last one). """ datadir = datastore.get_datadir() if what == 'all': # show all if not os.path.exists(datadir): return rows = [] for calc_id in datastore.get_calc_ids(datadir): try: ds = read(calc_id) oq = ds['oqparam'] cmode, descr = oq.calculation_mode, oq.description except: # invalid datastore file, or missing calculation_mode # and description attributes, perhaps due to a manual kill f = os.path.join(datadir, 'calc_%s.hdf5' % calc_id) logging.warn('Unreadable datastore %s', f) continue else: rows.append((calc_id, cmode, descr.encode('utf-8'))) for row in sorted(rows, key=lambda row: row[0]): # by calc_id print('#%d %s: %s' % row) return ds = read(calc_id) # this part is experimental if what == 'rlzs' and 'poes' in ds: min_value = 0.01 # used in rmsep getter = getters.PmapGetter(ds) sitecol = ds['sitecol'] pmaps = getter.get_pmaps(sitecol.sids) weights = [rlz.weight for rlz in getter.rlzs] mean = stats.compute_pmap_stats(pmaps, [numpy.mean], weights) dists = [] for rlz, pmap in zip(getter.rlzs, pmaps): dist = rmsep(mean.array, pmap.array, min_value) dists.append((dist, rlz)) print('Realizations in order of distance from the mean curves') for dist, rlz in sorted(dists): print('%s: rmsep=%s' % (rlz, dist)) elif view.keyfunc(what) in view: print(view(what, ds)) elif what.split('/', 1)[0] in extract: print(extract(ds, what, *extra)) elif what in ds: obj = ds[what] if hasattr(obj, 'value'): # an array print(write_csv(io.BytesIO(), obj.value).decode('utf8')) else: print(obj) else: print('%s not found' % what) ds.close()
def purge(calc_id, force=False): """ Remove the given calculation. If you want to remove all calculations, use oq reset. """ if calc_id < 0: try: calc_id = datastore.get_calc_ids(datadir)[calc_id] except IndexError: print('Calculation %d not found' % calc_id) return purge_one(calc_id, getpass.getuser(), force)
def purge(calc_id): """ Remove the given calculation. If you want to remove all calculations, use oq reset. """ if calc_id < 0: try: calc_id = datastore.get_calc_ids(datadir)[calc_id] except IndexError: print('Calculation %d not found' % calc_id) return purge_one(calc_id, getpass.getuser())
def show(what='contents', calc_id=-1, extra=()): """ Show the content of a datastore (by default the last one). """ datadir = datastore.get_datadir() if what == 'all': # show all if not os.path.exists(datadir): return rows = [] for calc_id in datastore.get_calc_ids(datadir): try: ds = util.read(calc_id) oq = ds['oqparam'] cmode, descr = oq.calculation_mode, oq.description except Exception: # invalid datastore file, or missing calculation_mode # and description attributes, perhaps due to a manual kill f = os.path.join(datadir, 'calc_%s.hdf5' % calc_id) logging.warning('Unreadable datastore %s', f) continue else: rows.append((calc_id, cmode, descr.encode('utf-8'))) for row in sorted(rows, key=lambda row: row[0]): # by calc_id print('#%d %s: %s' % row) return ds = util.read(calc_id) # this part is experimental if view.keyfunc(what) in view: print(view(what, ds)) elif what.split('/', 1)[0] in extract: obj = extract(ds, what, *extra) if hasattr(obj, 'dtype') and obj.dtype.names: print(write_csv(io.BytesIO(), obj).decode('utf8')) else: print(obj) elif what in ds: obj = ds.getitem(what) if hasattr(obj, 'items'): # is a group of datasets print(obj) else: # is a single dataset obj.refresh() # for SWMR mode aw = hdf5.ArrayWrapper.from_(obj) if hasattr(aw, 'shape_descr'): print(rst_table(aw.to_table())) else: print(write_csv(io.BytesIO(), aw.array).decode('utf8')) else: print('%s not found' % what) ds.close()
def _run(job_inis, concurrent_tasks, calc_id, pdb, loglevel, hc, exports, params): global calc_path assert len(job_inis) in (1, 2), job_inis # set the logs first of all calc_id = logs.init(calc_id, getattr(logging, loglevel.upper())) # disable gzip_input base.BaseCalculator.gzip_inputs = lambda self: None with performance.Monitor('total runtime', measuremem=True) as monitor: if os.environ.get('OQ_DISTRIBUTE') not in ('no', 'processpool'): os.environ['OQ_DISTRIBUTE'] = 'processpool' if len(job_inis) == 1: # run hazard or risk if hc: hc_id = hc[0] rlz_ids = hc[1:] else: hc_id = None rlz_ids = () oqparam = readinput.get_oqparam(job_inis[0], hc_id=hc_id) if not oqparam.cachedir: # enable caching oqparam.cachedir = datastore.get_datadir() if hc_id and hc_id < 0: # interpret negative calculation ids calc_ids = datastore.get_calc_ids() try: hc_id = calc_ids[hc_id] except IndexError: raise SystemExit('There are %d old calculations, cannot ' 'retrieve the %s' % (len(calc_ids), hc_id)) calc = base.calculators(oqparam, calc_id) calc.run(concurrent_tasks=concurrent_tasks, pdb=pdb, exports=exports, hazard_calculation_id=hc_id, rlz_ids=rlz_ids, **params) else: # run hazard + risk calc = run2(job_inis[0], job_inis[1], calc_id, concurrent_tasks, pdb, loglevel, exports, params) logging.info('Total time spent: %s s', monitor.duration) logging.info('Memory allocated: %s', general.humansize(monitor.mem)) print('See the output with silx view %s' % calc.datastore.filename) calc_path, _ = os.path.splitext(calc.datastore.filename) # used below return calc
def get_calc_id(db, datadir, job_id=None): """ Return the latest calc_id by looking both at the datastore and the database. :param db: a :class:`openquake.server.dbapi.Db` instance :param datadir: the directory containing the datastores :param job_id: a job ID; if None, returns the latest job ID """ calcs = datastore.get_calc_ids(datadir) calc_id = 0 if not calcs else calcs[-1] if job_id is None: try: job_id = db('SELECT seq FROM sqlite_sequence WHERE name="job"', scalar=True) except NotFound: job_id = 0 return max(calc_id, job_id)
def _run(job_ini, concurrent_tasks, pdb, loglevel, hc, exports, params): global calc_path logging.basicConfig(level=getattr(logging, loglevel.upper())) job_inis = job_ini.split(',') assert len(job_inis) in (1, 2), job_inis monitor = performance.Monitor('complete runtime', measuremem=True) if len(job_inis) == 1: # run hazard or risk if hc: hc_id = hc[0] rlz_ids = hc[1:] else: hc_id = None rlz_ids = () oqparam = readinput.get_oqparam(job_inis[0], hc_id=hc_id) if hc_id and hc_id < 0: # interpret negative calculation ids calc_ids = datastore.get_calc_ids() try: hc_id = calc_ids[hc_id] except IndexError: raise SystemExit('There are %d old calculations, cannot ' 'retrieve the %s' % (len(calc_ids), hc_id)) calc = base.calculators(oqparam, monitor) with calc._monitor: calc.run(concurrent_tasks=concurrent_tasks, pdb=pdb, exports=exports, hazard_calculation_id=hc_id, rlz_ids=rlz_ids, **params) else: # run hazard + risk calc = run2(job_inis[0], job_inis[1], concurrent_tasks, pdb, exports, params, monitor) logging.info('Total time spent: %s s', monitor.duration) logging.info('Memory allocated: %s', general.humansize(monitor.mem)) monitor.flush() print('See the output with hdfview %s' % calc.datastore.hdf5path) calc_path = calc.datastore.calc_dir # used for the .pstat filename return calc
def _run(job_inis, concurrent_tasks, pdb, loglevel, hc, exports, params): global calc_path assert len(job_inis) in (1, 2), job_inis # set the logs first of all calc_id = logs.init(level=getattr(logging, loglevel.upper())) with performance.Monitor('total runtime', measuremem=True) as monitor: if len(job_inis) == 1: # run hazard or risk if hc: hc_id = hc[0] rlz_ids = hc[1:] else: hc_id = None rlz_ids = () oqparam = readinput.get_oqparam(job_inis[0], hc_id=hc_id) vars(oqparam).update(params) if hc_id and hc_id < 0: # interpret negative calculation ids calc_ids = datastore.get_calc_ids() try: hc_id = calc_ids[hc_id] except IndexError: raise SystemExit('There are %d old calculations, cannot ' 'retrieve the %s' % (len(calc_ids), hc_id)) calc = base.calculators(oqparam, calc_id) calc.run(concurrent_tasks=concurrent_tasks, pdb=pdb, exports=exports, hazard_calculation_id=hc_id, rlz_ids=rlz_ids) else: # run hazard + risk calc = run2(job_inis[0], job_inis[1], calc_id, concurrent_tasks, pdb, loglevel, exports, params) logging.info('Total time spent: %s s', monitor.duration) logging.info('Memory allocated: %s', general.humansize(monitor.mem)) print('See the output with hdfview %s' % calc.datastore.hdf5path) calc_path, _ = os.path.splitext(calc.datastore.hdf5path) # used below return calc
def _run(job_inis, concurrent_tasks, pdb, loglevel, hc, exports, params): global calc_path assert len(job_inis) in (1, 2), job_inis # set the logs first of all calc_id = logs.init(level=getattr(logging, loglevel.upper())) with performance.Monitor('total runtime', measuremem=True) as monitor: if len(job_inis) == 1: # run hazard or risk if hc: hc_id = hc[0] rlz_ids = hc[1:] else: hc_id = None rlz_ids = () oqparam = readinput.get_oqparam(job_inis[0], hc_id=hc_id) vars(oqparam).update(params) if hc_id and hc_id < 0: # interpret negative calculation ids calc_ids = datastore.get_calc_ids() try: hc_id = calc_ids[hc_id] except IndexError: raise SystemExit( 'There are %d old calculations, cannot ' 'retrieve the %s' % (len(calc_ids), hc_id)) calc = base.calculators(oqparam, calc_id) calc.run(concurrent_tasks=concurrent_tasks, pdb=pdb, exports=exports, hazard_calculation_id=hc_id, rlz_ids=rlz_ids) else: # run hazard + risk calc = run2( job_inis[0], job_inis[1], calc_id, concurrent_tasks, pdb, loglevel, exports, params) logging.info('Total time spent: %s s', monitor.duration) logging.info('Memory allocated: %s', general.humansize(monitor.mem)) print('See the output with silx view %s' % calc.datastore.filename) calc_path, _ = os.path.splitext(calc.datastore.filename) # used below return calc
def extract(what, calc_id=-1, server_url='http://127.0.0.1:8800'): """ Extract an output from the datastore and save it into an .hdf5 file. By default use the WebUI, unless server_url is set to 'local', in which case the extraction is done directly bypassing tjhe WebUI and the database. """ logging.basicConfig(level=logging.INFO) if calc_id < 0: calc_id = datastore.get_calc_ids()[calc_id] filename = None if dbserver.get_status() == 'running': job = dbcmd('get_job', calc_id) if job is not None: filename = job.ds_calc_dir + '.hdf5' dstore = util.read(filename or calc_id) parent_id = dstore['oqparam'].hazard_calculation_id if parent_id: dstore.parent = util.read(parent_id) urlpath = '/v1/calc/%d/extract/%s' % (calc_id, quote(what)) with performance.Monitor('extract', measuremem=True) as mon, dstore: if server_url == 'local': print('Emulating call to %s' % urlpath) items = extract_(dstore, what) else: print('Calling %s%s' % (server_url, urlpath)) data = urlopen(server_url.rstrip('/') + urlpath).read() items = (item for item in numpy.load(io.BytesIO(data)).items()) if not inspect.isgenerator(items): items = [(items.__class__.__name__, items)] fname = '%s_%d.hdf5' % (what.replace('/', '-').replace( '?', '-'), dstore.calc_id) hdf5.save(fname, items) print('Saved', fname) if mon.duration > 1: print(mon)