Exemple #1
0
def extract(what, calc_id=-1, hostport=None):
    """
    Extract an output from the datastore and save it into an .hdf5 file.
    """
    logging.basicConfig(level=logging.INFO)
    if calc_id < 0:
        calc_id = datastore.get_calc_ids()[calc_id]
    hdf5path = None
    if dbserver.get_status() == 'running':
        job = dbcmd('get_job', calc_id)
        if job is not None:
            hdf5path = job.ds_calc_dir + '.hdf5'
    dstore = datastore.read(hdf5path or calc_id)
    parent_id = dstore['oqparam'].hazard_calculation_id
    if parent_id:
        dstore.parent = datastore.read(parent_id)
    urlpath = '/v1/calc/%d/extract/%s' % (calc_id, quote(what))
    with performance.Monitor('extract', measuremem=True) as mon, dstore:
        if hostport:
            data = urlopen('http://%s%s' % (hostport, urlpath)).read()
            items = (item for item in numpy.load(io.BytesIO(data)).items())
        else:
            print('Emulating call to %s' % urlpath)
            items = extract_(dstore, what)
        if not inspect.isgenerator(items):
            items = [(items.__class__.__name__, items)]
        fname = '%s_%d.hdf5' % (what.replace('/', '-').replace(
            '?', '-'), dstore.calc_id)
        hdf5.save(fname, items)
        print('Saved', fname)
    if mon.duration > 1:
        print(mon)
Exemple #2
0
def extract(what, calc_id=-1):
    """
    Extract an output from the datastore and save it into an .hdf5 file.
    """
    logging.basicConfig(level=logging.INFO)
    if calc_id < 0:
        calc_id = datastore.get_calc_ids()[calc_id]
    if dbserver.get_status() == 'running':
        job = dbcmd('get_job', calc_id)
        if job is not None:
            calc_id = job.ds_calc_dir + '.hdf5'
    dstore = datastore.read(calc_id)
    parent_id = dstore['oqparam'].hazard_calculation_id
    if parent_id:
        dstore.parent = datastore.read(parent_id)
    print('Emulating call to /v1/calc/%s/extract/%s' % (calc_id, quote(what)))
    with performance.Monitor('extract', measuremem=True) as mon, dstore:
        items = extract_(dstore, what)
        if not inspect.isgenerator(items):
            items = [(items.__class__.__name__, items)]
        fname = '%s_%d.hdf5' % (what.replace('/', '-').replace(
            '?', '-'), dstore.calc_id)
        hdf5.save(fname, items)
        print('Saved', fname)
    if mon.duration > 1:
        print(mon)
Exemple #3
0
def show(what='contents', calc_id=-1, extra=()):
    """
    Show the content of a datastore (by default the last one).
    """
    datadir = datastore.get_datadir()
    if what == 'all':  # show all
        if not os.path.exists(datadir):
            return
        rows = []
        for calc_id in datastore.get_calc_ids(datadir):
            try:
                ds = read(calc_id)
                oq = ds['oqparam']
                cmode, descr = oq.calculation_mode, oq.description
            except:
                # invalid datastore file, or missing calculation_mode
                # and description attributes, perhaps due to a manual kill
                f = os.path.join(datadir, 'calc_%s.hdf5' % calc_id)
                logging.warn('Unreadable datastore %s', f)
                continue
            else:
                rows.append((calc_id, cmode, descr.encode('utf-8')))
        for row in sorted(rows, key=lambda row: row[0]):  # by calc_id
            print('#%d %s: %s' % row)
        return

    ds = read(calc_id)

    # this part is experimental
    if what == 'rlzs' and 'poes' in ds:
        min_value = 0.01  # used in rmsep
        getter = getters.PmapGetter(ds)
        sitecol = ds['sitecol']
        pmaps = getter.get_pmaps(sitecol.sids)
        weights = [rlz.weight for rlz in getter.rlzs]
        mean = stats.compute_pmap_stats(pmaps, [numpy.mean], weights)
        dists = []
        for rlz, pmap in zip(getter.rlzs, pmaps):
            dist = rmsep(mean.array, pmap.array, min_value)
            dists.append((dist, rlz))
        print('Realizations in order of distance from the mean curves')
        for dist, rlz in sorted(dists):
            print('%s: rmsep=%s' % (rlz, dist))
    elif view.keyfunc(what) in view:
        print(view(what, ds))
    elif what.split('/', 1)[0] in extract:
        print(extract(ds, what, *extra))
    elif what in ds:
        obj = ds[what]
        if hasattr(obj, 'value'):  # an array
            print(write_csv(io.BytesIO(), obj.value).decode('utf8'))
        else:
            print(obj)
    else:
        print('%s not found' % what)

    ds.close()
Exemple #4
0
def purge(calc_id, force=False):
    """
    Remove the given calculation. If you want to remove all calculations,
    use oq reset.
    """
    if calc_id < 0:
        try:
            calc_id = datastore.get_calc_ids(datadir)[calc_id]
        except IndexError:
            print('Calculation %d not found' % calc_id)
            return
    purge_one(calc_id, getpass.getuser(), force)
Exemple #5
0
def purge(calc_id):
    """
    Remove the given calculation. If you want to remove all calculations,
    use oq reset.
    """
    if calc_id < 0:
        try:
            calc_id = datastore.get_calc_ids(datadir)[calc_id]
        except IndexError:
            print('Calculation %d not found' % calc_id)
            return
    purge_one(calc_id, getpass.getuser())
Exemple #6
0
def show(what='contents', calc_id=-1, extra=()):
    """
    Show the content of a datastore (by default the last one).
    """
    datadir = datastore.get_datadir()
    if what == 'all':  # show all
        if not os.path.exists(datadir):
            return
        rows = []
        for calc_id in datastore.get_calc_ids(datadir):
            try:
                ds = util.read(calc_id)
                oq = ds['oqparam']
                cmode, descr = oq.calculation_mode, oq.description
            except Exception:
                # invalid datastore file, or missing calculation_mode
                # and description attributes, perhaps due to a manual kill
                f = os.path.join(datadir, 'calc_%s.hdf5' % calc_id)
                logging.warning('Unreadable datastore %s', f)
                continue
            else:
                rows.append((calc_id, cmode, descr.encode('utf-8')))
        for row in sorted(rows, key=lambda row: row[0]):  # by calc_id
            print('#%d %s: %s' % row)
        return

    ds = util.read(calc_id)

    # this part is experimental
    if view.keyfunc(what) in view:
        print(view(what, ds))
    elif what.split('/', 1)[0] in extract:
        obj = extract(ds, what, *extra)
        if hasattr(obj, 'dtype') and obj.dtype.names:
            print(write_csv(io.BytesIO(), obj).decode('utf8'))
        else:
            print(obj)
    elif what in ds:
        obj = ds.getitem(what)
        if hasattr(obj, 'items'):  # is a group of datasets
            print(obj)
        else:  # is a single dataset
            obj.refresh()  # for SWMR mode
            aw = hdf5.ArrayWrapper.from_(obj)
            if hasattr(aw, 'shape_descr'):
                print(rst_table(aw.to_table()))
            else:
                print(write_csv(io.BytesIO(), aw.array).decode('utf8'))
    else:
        print('%s not found' % what)

    ds.close()
Exemple #7
0
def _run(job_inis, concurrent_tasks, calc_id, pdb, loglevel, hc, exports,
         params):
    global calc_path
    assert len(job_inis) in (1, 2), job_inis
    # set the logs first of all
    calc_id = logs.init(calc_id, getattr(logging, loglevel.upper()))
    # disable gzip_input
    base.BaseCalculator.gzip_inputs = lambda self: None
    with performance.Monitor('total runtime', measuremem=True) as monitor:
        if os.environ.get('OQ_DISTRIBUTE') not in ('no', 'processpool'):
            os.environ['OQ_DISTRIBUTE'] = 'processpool'
        if len(job_inis) == 1:  # run hazard or risk
            if hc:
                hc_id = hc[0]
                rlz_ids = hc[1:]
            else:
                hc_id = None
                rlz_ids = ()
            oqparam = readinput.get_oqparam(job_inis[0], hc_id=hc_id)
            if not oqparam.cachedir:  # enable caching
                oqparam.cachedir = datastore.get_datadir()
            if hc_id and hc_id < 0:  # interpret negative calculation ids
                calc_ids = datastore.get_calc_ids()
                try:
                    hc_id = calc_ids[hc_id]
                except IndexError:
                    raise SystemExit('There are %d old calculations, cannot '
                                     'retrieve the %s' %
                                     (len(calc_ids), hc_id))
            calc = base.calculators(oqparam, calc_id)
            calc.run(concurrent_tasks=concurrent_tasks,
                     pdb=pdb,
                     exports=exports,
                     hazard_calculation_id=hc_id,
                     rlz_ids=rlz_ids,
                     **params)
        else:  # run hazard + risk
            calc = run2(job_inis[0], job_inis[1], calc_id, concurrent_tasks,
                        pdb, loglevel, exports, params)

    logging.info('Total time spent: %s s', monitor.duration)
    logging.info('Memory allocated: %s', general.humansize(monitor.mem))
    print('See the output with silx view %s' % calc.datastore.filename)
    calc_path, _ = os.path.splitext(calc.datastore.filename)  # used below
    return calc
Exemple #8
0
def get_calc_id(db, datadir, job_id=None):
    """
    Return the latest calc_id by looking both at the datastore
    and the database.

    :param db: a :class:`openquake.server.dbapi.Db` instance
    :param datadir: the directory containing the datastores
    :param job_id: a job ID; if None, returns the latest job ID
    """
    calcs = datastore.get_calc_ids(datadir)
    calc_id = 0 if not calcs else calcs[-1]
    if job_id is None:
        try:
            job_id = db('SELECT seq FROM sqlite_sequence WHERE name="job"',
                        scalar=True)
        except NotFound:
            job_id = 0
    return max(calc_id, job_id)
Exemple #9
0
def _run(job_ini, concurrent_tasks, pdb, loglevel, hc, exports, params):
    global calc_path
    logging.basicConfig(level=getattr(logging, loglevel.upper()))
    job_inis = job_ini.split(',')
    assert len(job_inis) in (1, 2), job_inis
    monitor = performance.Monitor('complete runtime', measuremem=True)
    if len(job_inis) == 1:  # run hazard or risk
        if hc:
            hc_id = hc[0]
            rlz_ids = hc[1:]
        else:
            hc_id = None
            rlz_ids = ()
        oqparam = readinput.get_oqparam(job_inis[0], hc_id=hc_id)
        if hc_id and hc_id < 0:  # interpret negative calculation ids
            calc_ids = datastore.get_calc_ids()
            try:
                hc_id = calc_ids[hc_id]
            except IndexError:
                raise SystemExit('There are %d old calculations, cannot '
                                 'retrieve the %s' % (len(calc_ids), hc_id))
        calc = base.calculators(oqparam, monitor)
        with calc._monitor:
            calc.run(concurrent_tasks=concurrent_tasks,
                     pdb=pdb,
                     exports=exports,
                     hazard_calculation_id=hc_id,
                     rlz_ids=rlz_ids,
                     **params)
    else:  # run hazard + risk
        calc = run2(job_inis[0], job_inis[1], concurrent_tasks, pdb, exports,
                    params, monitor)

    logging.info('Total time spent: %s s', monitor.duration)
    logging.info('Memory allocated: %s', general.humansize(monitor.mem))
    monitor.flush()
    print('See the output with hdfview %s' % calc.datastore.hdf5path)
    calc_path = calc.datastore.calc_dir  # used for the .pstat filename
    return calc
Exemple #10
0
def _run(job_inis, concurrent_tasks, pdb, loglevel, hc, exports, params):
    global calc_path
    assert len(job_inis) in (1, 2), job_inis
    # set the logs first of all
    calc_id = logs.init(level=getattr(logging, loglevel.upper()))
    with performance.Monitor('total runtime', measuremem=True) as monitor:
        if len(job_inis) == 1:  # run hazard or risk
            if hc:
                hc_id = hc[0]
                rlz_ids = hc[1:]
            else:
                hc_id = None
                rlz_ids = ()
            oqparam = readinput.get_oqparam(job_inis[0], hc_id=hc_id)
            vars(oqparam).update(params)
            if hc_id and hc_id < 0:  # interpret negative calculation ids
                calc_ids = datastore.get_calc_ids()
                try:
                    hc_id = calc_ids[hc_id]
                except IndexError:
                    raise SystemExit('There are %d old calculations, cannot '
                                     'retrieve the %s' %
                                     (len(calc_ids), hc_id))
            calc = base.calculators(oqparam, calc_id)
            calc.run(concurrent_tasks=concurrent_tasks,
                     pdb=pdb,
                     exports=exports,
                     hazard_calculation_id=hc_id,
                     rlz_ids=rlz_ids)
        else:  # run hazard + risk
            calc = run2(job_inis[0], job_inis[1], calc_id, concurrent_tasks,
                        pdb, loglevel, exports, params)

    logging.info('Total time spent: %s s', monitor.duration)
    logging.info('Memory allocated: %s', general.humansize(monitor.mem))
    print('See the output with hdfview %s' % calc.datastore.hdf5path)
    calc_path, _ = os.path.splitext(calc.datastore.hdf5path)  # used below
    return calc
Exemple #11
0
def _run(job_inis, concurrent_tasks, pdb, loglevel, hc, exports, params):
    global calc_path
    assert len(job_inis) in (1, 2), job_inis
    # set the logs first of all
    calc_id = logs.init(level=getattr(logging, loglevel.upper()))
    with performance.Monitor('total runtime', measuremem=True) as monitor:
        if len(job_inis) == 1:  # run hazard or risk
            if hc:
                hc_id = hc[0]
                rlz_ids = hc[1:]
            else:
                hc_id = None
                rlz_ids = ()
            oqparam = readinput.get_oqparam(job_inis[0], hc_id=hc_id)
            vars(oqparam).update(params)
            if hc_id and hc_id < 0:  # interpret negative calculation ids
                calc_ids = datastore.get_calc_ids()
                try:
                    hc_id = calc_ids[hc_id]
                except IndexError:
                    raise SystemExit(
                        'There are %d old calculations, cannot '
                        'retrieve the %s' % (len(calc_ids), hc_id))
            calc = base.calculators(oqparam, calc_id)
            calc.run(concurrent_tasks=concurrent_tasks, pdb=pdb,
                     exports=exports, hazard_calculation_id=hc_id,
                     rlz_ids=rlz_ids)
        else:  # run hazard + risk
            calc = run2(
                job_inis[0], job_inis[1], calc_id, concurrent_tasks, pdb,
                loglevel, exports, params)

    logging.info('Total time spent: %s s', monitor.duration)
    logging.info('Memory allocated: %s', general.humansize(monitor.mem))
    print('See the output with silx view %s' % calc.datastore.filename)
    calc_path, _ = os.path.splitext(calc.datastore.filename)  # used below
    return calc
Exemple #12
0
def extract(what, calc_id=-1, server_url='http://127.0.0.1:8800'):
    """
    Extract an output from the datastore and save it into an .hdf5 file.
    By default use the WebUI, unless server_url is set to 'local', in
    which case the extraction is done directly bypassing tjhe WebUI and
    the database.
    """
    logging.basicConfig(level=logging.INFO)
    if calc_id < 0:
        calc_id = datastore.get_calc_ids()[calc_id]
    filename = None
    if dbserver.get_status() == 'running':
        job = dbcmd('get_job', calc_id)
        if job is not None:
            filename = job.ds_calc_dir + '.hdf5'
    dstore = util.read(filename or calc_id)
    parent_id = dstore['oqparam'].hazard_calculation_id
    if parent_id:
        dstore.parent = util.read(parent_id)
    urlpath = '/v1/calc/%d/extract/%s' % (calc_id, quote(what))
    with performance.Monitor('extract', measuremem=True) as mon, dstore:
        if server_url == 'local':
            print('Emulating call to %s' % urlpath)
            items = extract_(dstore, what)
        else:
            print('Calling %s%s' % (server_url, urlpath))
            data = urlopen(server_url.rstrip('/') + urlpath).read()
            items = (item for item in numpy.load(io.BytesIO(data)).items())
        if not inspect.isgenerator(items):
            items = [(items.__class__.__name__, items)]
        fname = '%s_%d.hdf5' % (what.replace('/', '-').replace(
            '?', '-'), dstore.calc_id)
        hdf5.save(fname, items)
        print('Saved', fname)
    if mon.duration > 1:
        print(mon)