Ejemplo n.º 1
0
def expose_outputs(dstore):
    """
    Build a correspondence between the outputs in the datastore and the
    ones in the database.

    :param dstore: datastore
    """
    oq = dstore['oqparam']
    exportable = set(ekey[0] for ekey in export.export)
    calcmode = oq.calculation_mode
    dskeys = set(dstore) & exportable  # exportable datastore keys
    if oq.uniform_hazard_spectra:
        dskeys.add('uhs')  # export them
    if oq.hazard_maps:
        dskeys.add('hmaps')  # export them
    if 'rcurves-rlzs' in dstore or 'loss_curves-rlzs' in dstore:
        dskeys.add('loss_maps-rlzs')
    if 'rcurves-stats' in dstore or 'loss_curves-stats' in dstore:
        dskeys.add('loss_maps-stats')
    try:
        rlzs = dstore['realizations']
    except KeyError:
        rlzs = []
    if 'all_loss_ratios' in dskeys:
        dskeys.remove('all_loss_ratios')  # export only specific IDs
    if 'realizations' in dskeys and len(rlzs) <= 1:
        dskeys.remove('realizations')  # do not export a single realization
    if 'ruptures' in dskeys and 'scenario' in calcmode:
        exportable.remove('ruptures')  # do not export, as requested by Vitor
    logs.dbcmd('create_outputs', dstore.calc_id, sorted(dskeys & exportable))
Ejemplo n.º 2
0
def run_job(cfg_file, log_level, log_file, exports='',
            hazard_calculation_id=None):
    """
    Run a job using the specified config file and other options.

    :param str cfg_file:
        Path to calculation config (INI-style) files.
    :param str log_level:
        'debug', 'info', 'warn', 'error', or 'critical'
    :param str log_file:
        Path to log file.
    :param exports:
        A comma-separated string of export types requested by the user.
        Currently only 'xml' is supported.
    :param hazard_calculation_id:
        ID of the previous calculation or None
    """
    job_id, oqparam = dbcmd(
        'job_from_file', cfg_file, getpass.getuser(), hazard_calculation_id)
    calc = engine.run_calc(job_id, oqparam, log_level, log_file, exports,
                           hazard_calculation_id=hazard_calculation_id)
    duration = calc.monitor.duration  # set this before monitor.flush()
    calc.monitor.flush()
    dbcmd('print_results', job_id, duration)
    return job_id
Ejemplo n.º 3
0
def expose_outputs(dstore):
    """
    Build a correspondence between the outputs in the datastore and the
    ones in the database.

    :param dstore: datastore
    """
    oq = dstore['oqparam']
    exportable = set(ekey[0] for ekey in export.export)
    # small hack: remove the sescollection outputs from scenario
    # calculators, as requested by Vitor
    calcmode = oq.calculation_mode
    dskeys = set(dstore) & exportable  # exportable datastore keys
    if oq.uniform_hazard_spectra:
        dskeys.add('uhs')  # export them
    if oq.hazard_maps:
        dskeys.add('hmaps')  # export them
    try:
        rlzs = dstore['realizations']
    except KeyError:
        rlzs = []
    if 'realizations' in dskeys and len(rlzs) <= 1:
        dskeys.remove('realizations')  # do not export a single realization
    if 'sescollection' in dskeys and 'scenario' in calcmode:
        exportable.remove('sescollection')  # do not export
    logs.dbcmd('create_outputs', dstore.calc_id, sorted(dskeys))
Ejemplo n.º 4
0
def dbserver(cmd):
    """
    start/stop/restart the database server, or return its status
    """
    if valid.boolean(config.get('dbserver', 'multi_user')):
        sys.exit('oq dbserver only works in single user mode')

    status = get_status()
    if cmd == 'status':
        print(status)
    elif cmd == 'stop':
        if status == 'running':
            logs.dbcmd('stop')
            print('stopped')
        else:
            print('already stopped')
    elif cmd == 'start':
        if status == 'not-running':
            runserver()
            print('started')
        else:
            print('already running')
    elif cmd == 'restart':
        if status == 'running':
            logs.dbcmd('stop')
            print('stopped')
        runserver()
        print('started')
Ejemplo n.º 5
0
def calc_remove(request, calc_id):
    """
    Remove the calculation id by setting the field oq_job.relevant to False.
    """
    try:
        logs.dbcmd('set_relevant', calc_id, False)
    except models.NotFound:
        return HttpResponseNotFound()
    return HttpResponse(content=json.dumps([]),
                        content_type=JSON, status=200)
Ejemplo n.º 6
0
def del_calculation(job_id, confirmed=False):
    """
    Delete a calculation and all associated outputs.
    """
    if confirmed or confirm(
            'Are you sure you want to delete this calculation and all '
            'associated outputs?\nThis action cannot be undone. (y/n): '):
        try:
            logs.dbcmd('del_calc', job_id, getpass.getuser())
        except RuntimeError as err:
            print(err)
Ejemplo n.º 7
0
def make_report(isodate='today'):
    """
    Build a HTML report with the computations performed at the given isodate.
    Return the name of the report, which is saved in the current directory.
    """
    if isodate == 'today':
        isodate = date.today()
    else:
        isodate = date(*time.strptime(isodate, '%Y-%m-%d')[:3])
    isodate1 = isodate + timedelta(1)  # +1 day

    tag_ids = []
    tag_status = []
    tag_contents = []

    # the fetcher returns an header which is stripped with [1:]
    jobs = dbcmd(
        'fetch', ALL_JOBS, isodate.isoformat(), isodate1.isoformat())[1:]
    page = '<h2>%d job(s) finished before midnight of %s</h2>' % (
        len(jobs), isodate)
    for job_id, user, status, ds_calc in jobs:
        tag_ids.append(job_id)
        tag_status.append(status)
        stats = dbcmd('fetch', JOB_STATS, job_id)
        if not stats[1:]:
            continue
        (job_id, user, start_time, stop_time, status) = stats[1]
        try:
            ds = read(job_id, datadir=os.path.dirname(ds_calc))
            txt = view_fullreport('fullreport', ds).decode('utf-8')
            report = html_parts(txt)
        except Exception as exc:
            report = dict(
                html_title='Could not generate report: %s' % cgi.escape(
                    unicode(exc), quote=True),
                fragment='')

        page = report['html_title']

        add_duration(stats)
        page += html(stats)

        page += report['fragment']

        tag_contents.append(page)

    page = make_tabs(tag_ids, tag_status, tag_contents) + (
        'Report last updated: %s' % datetime.now())
    fname = 'jobs-%s.html' % isodate
    with open(fname, 'w') as f:
        f.write(PAGE_TEMPLATE % page.encode('utf-8'))
    return fname
Ejemplo n.º 8
0
def run_job(cfg_file, log_level, log_file, exports='',
            hazard_calculation_id=None):
    """
    Run a job using the specified config file and other options.

    :param str cfg_file:
        Path to calculation config (INI-style) files.
    :param str log_level:
        'debug', 'info', 'warn', 'error', or 'critical'
    :param str log_file:
        Path to log file.
    :param exports:
        A comma-separated string of export types requested by the user.
    :param hazard_calculation_id:
        ID of the previous calculation or None
    """
    # if the master dies, automatically kill the workers
    concurrent_futures_process_monkeypatch()
    job_ini = os.path.abspath(cfg_file)
    job_id, oqparam = eng.job_from_file(
        job_ini, getpass.getuser(), hazard_calculation_id)
    calc = eng.run_calc(job_id, oqparam, log_level, log_file, exports,
                        hazard_calculation_id=hazard_calculation_id)
    calc.monitor.flush()
    for line in logs.dbcmd('list_outputs', job_id, False):
        print(line)
    return job_id
Ejemplo n.º 9
0
def calc(request, id=None):
    """
    Get a list of calculations and report their id, status, job_type,
    is_running, description, and a url where more detailed information
    can be accessed. This is called several times by the Javascript.

    Responses are in JSON.
    """
    base_url = _get_base_url(request)

    user = utils.get_user_data(request)

    calc_data = logs.dbcmd('get_calcs', request.GET,
                           user['name'], user['acl_on'], id)

    response_data = []
    for hc_id, owner, status, job_type, is_running, desc in calc_data:
        url = urlparse.urljoin(base_url, 'v1/calc/%d' % hc_id)
        response_data.append(
            dict(id=hc_id, owner=owner, status=status, job_type=job_type,
                 is_running=is_running, description=desc, url=url))

    # if id is specified the related dictionary is returned instead the list
    if id is not None:
        [response_data] = response_data

    return HttpResponse(content=json.dumps(response_data),
                        content_type=JSON)
Ejemplo n.º 10
0
def get_result(request, result_id):
    """
    Download a specific result, by ``result_id``.

    The common abstracted functionality for getting hazard or risk results.

    :param request:
        `django.http.HttpRequest` object. Can contain a `export_type` GET
        param (the default is 'xml' if no param is specified).
    :param result_id:
        The id of the requested artifact.
    :returns:
        If the requested ``result_id`` is not available in the format
        designated by the `export_type`.

        Otherwise, return a `django.http.HttpResponse` containing the content
        of the requested artifact.

    Parameters for the GET request can include an `export_type`, such as 'xml',
    'geojson', 'csv', etc.
    """
    # If the result for the requested ID doesn't exist, OR
    # the job which it is related too is not complete,
    # throw back a 404.
    try:
        job_id, job_status, datadir, ds_key = logs.dbcmd(
            'get_result', result_id)
        if not job_status == 'complete':
            return HttpResponseNotFound()
    except models.NotFound:
        return HttpResponseNotFound()

    etype = request.GET.get('export_type')
    export_type = etype or DEFAULT_EXPORT_TYPE

    tmpdir = tempfile.mkdtemp()
    try:
        exported = core.export_from_datastore(
            (ds_key, export_type), job_id, datadir, tmpdir)
    except DataStoreExportError as exc:
        # TODO: there should be a better error page
        return HttpResponse(content='%s: %s' % (exc.__class__.__name__, exc),
                            content_type='text/plain', status=500)
    if exported is None:
        # Throw back a 404 if the exact export parameters are not supported
        return HttpResponseNotFound(
            'export_type=%s is not supported for %s' % (export_type, ds_key))

    content_type = EXPORT_CONTENT_TYPE_MAP.get(
        export_type, DEFAULT_CONTENT_TYPE)
    try:
        fname = 'output-%s-%s' % (result_id, os.path.basename(exported))
        # 'b' is needed when running the WebUI on Windows
        data = open(exported, 'rb').read()
        response = HttpResponse(data, content_type=content_type)
        response['Content-Length'] = len(data)
        response['Content-Disposition'] = 'attachment; filename=%s' % fname
        return response
    finally:
        shutil.rmtree(tmpdir)
Ejemplo n.º 11
0
def calc_results(request, calc_id):
    """
    Get a summarized list of calculation results for a given ``calc_id``.
    Result is a JSON array of objects containing the following attributes:

        * id
        * name
        * type (hazard_curve, hazard_map, etc.)
        * url (the exact url where the full result can be accessed)
    """
    user = utils.get_user_data(request)

    # If the specified calculation doesn't exist OR is not yet complete,
    # throw back a 404.
    try:
        info = logs.dbcmd('calc_info', calc_id)
        if user['acl_on'] and info['user_name'] != user['name']:
            return HttpResponseNotFound()
    except models.NotFound:
        return HttpResponseNotFound()
    base_url = _get_base_url(request)

    # NB: export_output has as keys the list (output_type, extension)
    # so this returns an ordered map output_type -> extensions such as
    # OrderedDict([('agg_loss_curve', ['xml', 'csv']), ...])
    output_types = groupby(export, lambda oe: oe[0],
                           lambda oes: [e for o, e in oes])
    results = logs.dbcmd('get_outputs', calc_id)
    if not results:
        return HttpResponseNotFound()

    response_data = []
    for result in results:
        try:  # output from the datastore
            rtype = result.ds_key
            # Catalina asked to remove the .txt outputs (used for the GMFs)
            outtypes = [ot for ot in output_types[rtype] if ot != 'txt']
        except KeyError:
            continue  # non-exportable outputs should not be shown
        url = urlparse.urljoin(base_url, 'v1/calc/result/%d' % result.id)
        datum = dict(
            id=result.id, name=result.display_name, type=rtype,
            outtypes=outtypes, url=url)
        response_data.append(datum)

    return HttpResponse(content=json.dumps(response_data))
Ejemplo n.º 12
0
def get_log_size(request, calc_id):
    """
    Get the current number of lines in the log
    """
    try:
        response_data = logs.dbcmd('get_log_size', calc_id)
    except models.NotFound:
        return HttpResponseNotFound()
    return HttpResponse(content=json.dumps(response_data), content_type=JSON)
Ejemplo n.º 13
0
def run_calc(job_id, oqparam, log_level, log_file, exports,
             hazard_calculation_id=None):
    """
    Run a calculation.

    :param job_id:
        ID of the current job
    :param oqparam:
        :class:`openquake.commonlib.oqvalidation.OqParam` instance
    :param str log_level:
        The desired logging level. Valid choices are 'debug', 'info',
        'progress', 'warn', 'error', and 'critical'.
    :param str log_file:
        Complete path (including file name) to file where logs will be written.
        If `None`, logging will just be printed to standard output.
    :param exports:
        A comma-separated string of export types.
    """
    monitor = Monitor('total runtime', measuremem=True)
    with logs.handle(job_id, log_level, log_file):  # run the job
        if USE_CELERY and os.environ.get('OQ_DISTRIBUTE') == 'celery':
            set_concurrent_tasks_default()
        calc = base.calculators(oqparam, monitor, calc_id=job_id)
        tb = 'None\n'
        try:
            logs.dbcmd('set_status', job_id, 'executing')
            _do_run_calc(calc, exports, hazard_calculation_id)
            expose_outputs(calc.datastore)
            records = views.performance_view(calc.datastore)
            logs.dbcmd('save_performance', job_id, records)
            calc.datastore.close()
            logs.LOG.info('Calculation %d finished correctly in %d seconds',
                          job_id, calc.monitor.duration)
            logs.dbcmd('finish', job_id, 'complete')
        except:
            tb = traceback.format_exc()
            try:
                logs.LOG.critical(tb)
                logs.dbcmd('finish', job_id, 'failed')
            except:  # an OperationalError may always happen
                sys.stderr.write(tb)
            raise
        finally:
            # if there was an error in the calculation, this part may fail;
            # in such a situation, we simply log the cleanup error without
            # taking further action, so that the real error can propagate
            try:
                if USE_CELERY:
                    celery_cleanup(TERMINATE, parallel.TaskManager.task_ids)
            except:
                # log the finalization error only if there is no real error
                if tb == 'None\n':
                    logs.LOG.error('finalizing', exc_info=True)
    return calc
Ejemplo n.º 14
0
def get_traceback(request, calc_id):
    """
    Get the traceback as a list of lines for a given ``calc_id``.
    """
    # If the specified calculation doesn't exist throw back a 404.
    try:
        response_data = logs.dbcmd('get_traceback', calc_id)
    except models.NotFound:
        return HttpResponseNotFound()
    return HttpResponse(content=json.dumps(response_data), content_type=JSON)
Ejemplo n.º 15
0
def export_outputs(job_id, target_dir, export_types):
    # make it possible commands like `oq engine --eos -1 /tmp`
    datadir, dskeys = logs.dbcmd('get_results', job_id)
    if not dskeys:
        yield('Found nothing to export for job %s' % job_id)
    for dskey in dskeys:
        yield('Exporting %s...' % dskey)
        for line in export_output(
                dskey, job_id, datadir, target_dir, export_types):
            yield line
Ejemplo n.º 16
0
def get_log_slice(request, calc_id, start, stop):
    """
    Get a slice of the calculation log as a JSON list of rows
    """
    start = start or 0
    stop = stop or None
    try:
        response_data = logs.dbcmd('get_log_slice', calc_id, start, stop)
    except models.NotFound:
        return HttpResponseNotFound()
    return HttpResponse(content=json.dumps(response_data), content_type=JSON)
Ejemplo n.º 17
0
def calc_info(request, calc_id):
    """
    Get a JSON blob containing all of parameters for the given calculation
    (specified by ``calc_id``). Also includes the current job status (
    executing, complete, etc.).
    """
    try:
        info = logs.dbcmd('calc_info', calc_id)
    except models.NotFound:
        return HttpResponseNotFound()
    return HttpResponse(content=json.dumps(info), content_type=JSON)
Ejemplo n.º 18
0
def calc_info(request, calc_id):
    """
    Get a JSON blob containing all of parameters for the given calculation
    (specified by ``calc_id``). Also includes the current job status (
    executing, complete, etc.).
    """
    try:
        info = logs.dbcmd('calc_info', calc_id)
    except models.NotFound:
        return HttpResponseNotFound()
    return HttpResponse(content=json.dumps(info), content_type=JSON)
Ejemplo n.º 19
0
def purge_one(calc_id, user):
    """
    Remove one calculation ID from the database and remove its datastore
    """
    hdf5path = os.path.join(datastore.DATADIR, 'calc_%s.hdf5' % calc_id)
    err = dbcmd('del_calc', calc_id, user)
    if err:
        print(err)
    if os.path.exists(hdf5path):
        os.remove(hdf5path)
        print('Removed %s' % hdf5path)
Ejemplo n.º 20
0
def get_log_slice(request, calc_id, start, stop):
    """
    Get a slice of the calculation log as a JSON list of rows
    """
    start = start or 0
    stop = stop or None
    try:
        response_data = logs.dbcmd('get_log_slice', calc_id, start, stop)
    except models.NotFound:
        return HttpResponseNotFound()
    return HttpResponse(content=json.dumps(response_data), content_type=JSON)
Ejemplo n.º 21
0
def expose_outputs(dstore):
    """
    Build a correspondence between the outputs in the datastore and the
    ones in the database.

    :param dstore: datastore
    """
    oq = dstore['oqparam']
    exportable = set(ekey[0] for ekey in export.export)
    # small hack: remove the sescollection outputs from scenario
    # calculators, as requested by Vitor
    calcmode = oq.calculation_mode
    dskeys = set(dstore) & exportable  # exportable datastore keys
    if oq.uniform_hazard_spectra:
        dskeys.add('uhs')  # export them
    if 'hmaps' in dskeys and not oq.hazard_maps:
        dskeys.remove('hmaps')  # do not export
    if 'realizations' in dskeys and len(dstore['realizations']) <= 1:
        dskeys.remove('realizations')  # do not export a single realization
    if 'sescollection' in dskeys and 'scenario' in calcmode:
        exportable.remove('sescollection')  # do not export
    logs.dbcmd('create_outputs', dstore.calc_id, sorted(dskeys))
Ejemplo n.º 22
0
def run_calc(job_id,
             oqparam,
             log_level,
             log_file,
             exports,
             hazard_calculation_id=None):
    """
    Run a calculation.

    :param job_id:
        ID of the current job
    :param oqparam:
        :class:`openquake.commonlib.oqvalidation.OqParam` instance
    :param str log_level:
        The desired logging level. Valid choices are 'debug', 'info',
        'progress', 'warn', 'error', and 'critical'.
    :param str log_file:
        Complete path (including file name) to file where logs will be written.
        If `None`, logging will just be printed to standard output.
    :param exports:
        A comma-separated string of export types.
    """
    monitor = Monitor('total runtime', measuremem=True)
    with logs.handle(job_id, log_level, log_file):  # run the job
        if USE_CELERY and os.environ.get('OQ_DISTRIBUTE') == 'celery':
            set_concurrent_tasks_default()
        calc = base.calculators(oqparam, monitor, calc_id=job_id)
        tb = 'None\n'
        try:
            _do_run_calc(calc, exports, hazard_calculation_id)
            logs.dbcmd('finish', job_id, 'complete')
            expose_outputs(calc.datastore)
            records = views.performance_view(calc.datastore)
            logs.dbcmd('save_performance', job_id, records)
            calc.datastore.close()
            logs.LOG.info('Calculation %d finished correctly in %d seconds',
                          job_id, calc.monitor.duration)
        except:
            tb = traceback.format_exc()
            try:
                logs.LOG.critical(tb)
                logs.dbcmd('finish', job_id, 'failed')
            except:  # an OperationalError may always happen
                sys.stderr.write(tb)
            raise
        finally:
            # if there was an error in the calculation, this part may fail;
            # in such a situation, we simply log the cleanup error without
            # taking further action, so that the real error can propagate
            try:
                if USE_CELERY:
                    celery_cleanup(TERMINATE, parallel.TaskManager.task_ids)
            except:
                # log the finalization error only if there is no real error
                if tb == 'None\n':
                    logs.LOG.error('finalizing', exc_info=True)
    return calc
Ejemplo n.º 23
0
def job_from_file(cfg_file, username, hazard_calculation_id=None):
    """
    Create a full job profile from a job config file.

    :param str cfg_file:
        Path to a job.ini file.
    :param str username:
        The user who will own this job profile and all results
    :param str datadir:
        Data directory of the user
    :param hazard_calculation_id:
        ID of a previous calculation or None
    :returns:
        a pair (job_id, oqparam)
    """
    oq = readinput.get_oqparam(cfg_file)
    job_id = logs.dbcmd('create_job', oq.calculation_mode, oq.description,
                        username, datastore.DATADIR, hazard_calculation_id)
    return job_id, oq
Ejemplo n.º 24
0
def job_from_file(cfg_file, username, hazard_calculation_id=None):
    """
    Create a full job profile from a job config file.

    :param str cfg_file:
        Path to a job.ini file.
    :param str username:
        The user who will own this job profile and all results
    :param str datadir:
        Data directory of the user
    :param hazard_calculation_id:
        ID of a previous calculation or None
    :returns:
        a pair (job_id, oqparam)
    """
    oq = readinput.get_oqparam(cfg_file)
    job_id = logs.dbcmd('create_job', oq.calculation_mode, oq.description,
                        username, datastore.DATADIR, hazard_calculation_id)
    return job_id, oq
Ejemplo n.º 25
0
def db(cmd, args=''):
    """
    Run a database command
    """
    args = shlex.split(args)
    if cmd not in commands:
        okcmds = '\n'.join(
            '%s %s' % (name, repr(' '.join(args)) if args else '')
            for name, args in sorted(commands.items()))
        print('Invalid command "%s": choose one from\n%s' % (cmd, okcmds))
    elif len(args) != len(commands[cmd]):
        print('Wrong number of arguments, expected %s, got %s' % (
            commands[cmd], args))
    else:
        dbserver.ensure_on()
        res = logs.dbcmd(cmd, *convert(args))
        if hasattr(res, '_fields'):
            print(rst_table(res))
        else:
            print(res)
Ejemplo n.º 26
0
def get_datastore(request, job_id):
    """
    Download a full datastore file.

    :param request:
        `django.http.HttpRequest` object.
    :param job_id:
        The id of the requested datastore
    :returns:
        A `django.http.HttpResponse` containing the content
        of the requested artifact, if present, else throws a 404
    """
    try:
        job = logs.dbcmd('get_job', int(job_id), getpass.getuser())
    except dbapi.NotFound:
        return HttpResponseNotFound()

    fname = job.ds_calc_dir + '.hdf5'
    response = FileResponse(
        FileWrapper(open(fname, 'rb')), content_type=HDF5)
    response['Content-Disposition'] = 'attachment; filename=%s' % fname
    return response
Ejemplo n.º 27
0
def get_result(request, result_id):
    """
    Download a specific result, by ``result_id``.

    The common abstracted functionality for getting hazard or risk results.

    :param request:
        `django.http.HttpRequest` object. Can contain a `export_type` GET
        param (the default is 'xml' if no param is specified).
    :param result_id:
        The id of the requested artifact.
    :returns:
        If the requested ``result_id`` is not available in the format
        designated by the `export_type`.

        Otherwise, return a `django.http.HttpResponse` containing the content
        of the requested artifact.

    Parameters for the GET request can include an `export_type`, such as 'xml',
    'geojson', 'csv', etc.
    """
    # If the result for the requested ID doesn't exist, OR
    # the job which it is related too is not complete,
    # throw back a 404.
    try:
        job_id, job_status, datadir, ds_key = logs.dbcmd(
            'get_result', result_id)
        if not job_status == 'complete':
            return HttpResponseNotFound()
    except models.NotFound:
        return HttpResponseNotFound()

    etype = request.GET.get('export_type')
    export_type = etype or DEFAULT_EXPORT_TYPE

    tmpdir = tempfile.mkdtemp()
    try:
        exported = core.export_from_datastore((ds_key, export_type), job_id,
                                              datadir, tmpdir)
    except DataStoreExportError as exc:
        # TODO: there should be a better error page
        return HttpResponse(content='%s: %s' % (exc.__class__.__name__, exc),
                            content_type='text/plain',
                            status=500)
    if exported is None:
        # Throw back a 404 if the exact export parameters are not supported
        return HttpResponseNotFound('export_type=%s is not supported for %s' %
                                    (export_type, ds_key))

    content_type = EXPORT_CONTENT_TYPE_MAP.get(export_type,
                                               DEFAULT_CONTENT_TYPE)
    try:
        fname = 'output-%s-%s' % (result_id, os.path.basename(exported))
        # 'b' is needed when running the WebUI on Windows
        data = open(exported, 'rb').read()
        response = HttpResponse(data, content_type=content_type)
        response['Content-Length'] = len(data)
        response['Content-Disposition'] = 'attachment; filename=%s' % fname
        return response
    finally:
        shutil.rmtree(tmpdir)
Ejemplo n.º 28
0
    # make sure we use the server on the temporary db
    config.DBS_ADDRESS = ('localhost', port)


def parse_args(argv):
    # manages the argument "tmpdb=XXX" used in the functional tests
    args = []
    dbname = None
    for arg in argv:
        if arg.startswith('tmpdb='):
            dbname = arg[6:]
        else:
            args.append(arg)
    return args, dbname

# the code here is run in development mode; for instance
# $ python manage.py runserver 0.0.0.0:8800
if __name__ == "__main__":
    os.environ.setdefault(
        "DJANGO_SETTINGS_MODULE", "openquake.server.settings")
    argv, tmpfile_port = parse_args(sys.argv)
    if tmpfile_port:  # this is used in the functional tests
        use_tmp_db(tmpfile_port)
    else:
        # check the database version
        logs.dbcmd('check_outdated')
        # reset is_running
        logs.dbcmd('reset_is_running')
    with executor:
        execute_from_command_line(argv)
Ejemplo n.º 29
0
def engine(log_file, no_distribute, yes, config_file, make_html_report,
           upgrade_db, version_db, what_if_I_upgrade,
           run_hazard, run_risk, run,
           list_hazard_calculations, list_risk_calculations,
           delete_calculation, delete_uncompleted_calculations,
           hazard_calculation_id, list_outputs, show_log,
           export_output, export_outputs, exports='',
           log_level='info'):
    """
    Run a calculation using the traditional command line API
    """
    config.abort_if_no_config_available()

    if run or run_hazard or run_risk:
        # the logging will be configured in engine.py
        pass
    else:
        # configure a basic logging
        logging.basicConfig(level=logging.INFO)

    if config_file:
        os.environ[config.OQ_CONFIG_FILE_VAR] = os.path.abspath(
            os.path.expanduser(config_file))
        config.refresh()

    if no_distribute:
        os.environ['OQ_DISTRIBUTE'] = 'no'

    # check if the datadir exists
    if not os.path.exists(datastore.DATADIR):
        os.makedirs(datastore.DATADIR)

    dbserver.ensure_on()
    
    if upgrade_db:
        logs.set_level('info')
        msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
        if msg.startswith('Your database is already updated'):
            pass
        elif yes or confirm('Proceed? (y/n) '):
            logs.dbcmd('upgrade_db')
        sys.exit(0)

    if version_db:
        print(logs.dbcmd('version_db'))
        sys.exit(0)

    if what_if_I_upgrade:
        print(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts'))
        sys.exit(0)

    # check if the db is outdated
    outdated = logs.dbcmd('check_outdated')
    if outdated:
        sys.exit(outdated)

    # hazard or hazard+risk
    if hazard_calculation_id:
        hc_id = get_job_id(hazard_calculation_id)
    else:
        hc_id = None
    if run:
        job_inis = [os.path.expanduser(ini) for ini in run.split(',')]
        if len(job_inis) not in (1, 2):
            sys.exit('%s should be a .ini filename or a pair of filenames '
                     'separated by a comma' % run)
        for job_ini in job_inis:
            open(job_ini).read()  # raise an IOError if the file does not exist
        log_file = os.path.expanduser(log_file) \
            if log_file is not None else None

        if len(job_inis) == 2:
            # run hazard
            job_id = run_job(job_inis[0], log_level,
                             log_file, exports)
            # run risk
            run_job(job_inis[1], log_level, log_file,
                    exports, hazard_calculation_id=job_id)
        else:
            run_job(
                os.path.expanduser(run), log_level, log_file,
                exports, hazard_calculation_id=hc_id)
    # hazard
    elif list_hazard_calculations:
        for line in logs.dbcmd(
                'list_calculations', 'hazard', getpass.getuser()):
            print(line)
    elif run_hazard is not None:
        print('WARN: --rh/--run-hazard are deprecated, use --run instead',
              file=sys.stderr)
        log_file = os.path.expanduser(log_file) \
            if log_file is not None else None
        run_job(os.path.expanduser(run_hazard), log_level,
                log_file, exports)
    elif delete_calculation is not None:
        del_calculation(delete_calculation, yes)
    # risk
    elif list_risk_calculations:
        for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
            print(line)
    elif run_risk is not None:
        print('WARN: --rr/--run-risk are deprecated, use --run instead',
              file=sys.stderr)
        if hazard_calculation_id is None:
            sys.exit(MISSING_HAZARD_MSG)
        log_file = os.path.expanduser(log_file) \
            if log_file is not None else None
        run_job(
            os.path.expanduser(run_risk),
            log_level, log_file, exports,
            hazard_calculation_id=hc_id)

    # export
    elif make_html_report:
        print('Written %s' % make_report(make_html_report))
        sys.exit(0)

    elif list_outputs is not None:
        hc_id = get_job_id(list_outputs)
        for line in logs.dbcmd('list_outputs', hc_id):
            print(line)
    elif show_log is not None:
        hc_id = get_job_id(show_log)
        for line in logs.dbcmd('get_log', hc_id):
            print(line)

    elif export_output is not None:
        output_id, target_dir = export_output
        dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
        for line in core.export_output(
                dskey, calc_id, datadir, os.path.expanduser(target_dir),
                exports or 'xml,csv'):
            print(line)

    elif export_outputs is not None:
        job_id, target_dir = export_outputs
        hc_id = get_job_id(job_id)
        for line in core.export_outputs(
                hc_id, os.path.expanduser(target_dir), exports or 'xml,csv'):
            print(line)

    elif delete_uncompleted_calculations:
        logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())

    else:
        engine.parentparser.prog = 'oq engine'
        engine.parentparser.print_usage()
Ejemplo n.º 30
0
    config.DBS_ADDRESS = ('localhost', port)


def parse_args(argv):
    # manages the argument "tmpdb=XXX" used in the functional tests
    args = []
    dbname = None
    for arg in argv:
        if arg.startswith('tmpdb='):
            dbname = arg[6:]
        else:
            args.append(arg)
    return args, dbname


# the code here is run in development mode; for instance
# $ python manage.py runserver 0.0.0.0:8800
if __name__ == "__main__":
    os.environ.setdefault("DJANGO_SETTINGS_MODULE",
                          "openquake.server.settings")
    argv, tmpfile_port = parse_args(sys.argv)
    if tmpfile_port:  # this is used in the functional tests
        use_tmp_db(tmpfile_port)
    else:
        # check the database version
        logs.dbcmd('check_outdated')
        # reset is_running
        logs.dbcmd('reset_is_running')
    with executor:
        execute_from_command_line(argv)
Ejemplo n.º 31
0
def main():
    arg_parser = set_up_arg_parser()

    args = arg_parser.parse_args()

    exports = args.exports or 'xml,csv'

    if args.version:
        print openquake.engine.__version__
        sys.exit(0)

    if args.run or args.run_hazard or args.run_risk:
        # the logging will be configured in engine.py
        pass
    else:
        # configure a basic logging
        logging.basicConfig(level=logging.INFO)

    if args.config_file:
        os.environ[config.OQ_CONFIG_FILE_VAR] = \
            abspath(expanduser(args.config_file))
        config.refresh()

    if args.no_distribute:
        os.environ['OQ_DISTRIBUTE'] = 'no'

    # check if the DbServer is up
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
        err = sock.connect_ex(config.DBS_ADDRESS)
    finally:
        sock.close()
    if err:
        multi_user = valid.boolean(config.get('dbserver', 'multi_user'))
        if multi_user:
            sys.exit('Please start the DbServer: '
                     'see the documentation for details')
        # otherwise start the DbServer automatically
        dblog = os.path.expanduser('~/oq-dbserver.log')
        subprocess.Popen([sys.executable, '-m', 'openquake.server.dbserver',
                          '-l', 'INFO'], stderr=open(dblog, 'w'))
    if args.upgrade_db:
        logs.set_level('info')
        msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
        if msg.startswith('Your database is already updated'):
            pass
        elif args.yes or confirm('Proceed? (y/n) '):
            logs.dbcmd('upgrade_db')
        sys.exit(0)

    if args.version_db:
        print logs.dbcmd('version_db')
        sys.exit(0)

    if args.what_if_I_upgrade:
        print logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts')
        sys.exit(0)

    # check if the db is outdated
    outdated = logs.dbcmd('check_outdated')
    if outdated:
        sys.exit(outdated)

    # hazard or hazard+risk
    if args.hazard_calculation_id:
        hc_id = get_job_id(args.hazard_calculation_id)
    else:
        hc_id = None
    if args.run:
        job_inis = map(expanduser, args.run.split(','))
        if len(job_inis) not in (1, 2):
            sys.exit('%s should be a .ini filename or a pair of filenames '
                     'separated by a comma' % args.run)
        for job_ini in job_inis:
            open(job_ini).read()  # raise an IOError if the file does not exist
        log_file = expanduser(args.log_file) \
            if args.log_file is not None else None

        if len(job_inis) == 2:
            # run hazard
            job_id = run_job(job_inis[0], args.log_level,
                             log_file, args.exports)
            # run risk
            run_job(job_inis[1], args.log_level, log_file,
                    args.exports, hazard_calculation_id=job_id)
        else:
            run_job(
                expanduser(args.run), args.log_level, log_file,
                args.exports, hazard_calculation_id=hc_id)
    # hazard
    elif args.list_hazard_calculations:
        for line in logs.dbcmd(
                'list_calculations', 'hazard', getpass.getuser()):
            print line
    elif args.run_hazard is not None:
        log_file = expanduser(args.log_file) \
            if args.log_file is not None else None
        run_job(expanduser(args.run_hazard), args.log_level,
                log_file, args.exports)
    elif args.delete_calculation is not None:
        delete_calculation(args.delete_calculation, args.yes)
    # risk
    elif args.list_risk_calculations:
        for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
            print line
    elif args.run_risk is not None:
        if args.hazard_calculation_id is None:
            sys.exit(MISSING_HAZARD_MSG)
        log_file = expanduser(args.log_file) \
            if args.log_file is not None else None
        run_job(
            expanduser(args.run_risk),
            args.log_level, log_file, args.exports,
            hazard_calculation_id=hc_id)

    # export
    elif args.make_html_report:
        print 'Written', make_report(args.make_html_report)
        sys.exit(0)

    elif args.list_outputs is not None:
        hc_id = get_job_id(args.list_outputs)
        for line in logs.dbcmd('list_outputs', hc_id):
            print line
    elif args.show_view is not None:
        job_id, view_name = args.show_view
        print views.view(view_name, datastore.read(int(job_id)))
    elif args.show_log is not None:
        hc_id = get_job_id(args.show_log[0])
        for line in logs.dbcmd('get_log', hc_id):
            print line

    elif args.export_output is not None:
        output_id, target_dir = args.export_output
        dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
        for line in core.export_output(
                dskey, calc_id, datadir, expanduser(target_dir), exports):
            print line

    elif args.export_outputs is not None:
        job_id, target_dir = args.export_outputs
        hc_id = get_job_id(job_id)
        for line in core.export_outputs(
                hc_id, expanduser(target_dir), exports):
            print line

    elif args.delete_uncompleted_calculations:
        logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())
    else:
        arg_parser.print_usage()
Ejemplo n.º 32
0
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2016 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import os
from django.core.wsgi import get_wsgi_application

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "openquake.server.settings")

from django.db import connection
from openquake.engine import logs

logs.dbcmd('reset_is_running')

# This application object is used by the development server
# as well as any WSGI server configured to use this file.
application = get_wsgi_application()
Ejemplo n.º 33
0
def get_job_id(job_id, username=None):
    username = username or getpass.getuser()
    job_id = logs.dbcmd('get_job_id', job_id, username)
    if not job_id:
        sys.exit('Job %s of %s not found' % (job_id, username))
    return job_id
Ejemplo n.º 34
0
 def read(calc_id):
     job = logs.dbcmd('get_job', calc_id, getpass.getuser())
     datadir = os.path.dirname(job.ds_calc_dir)
     return datastore.read(job.id, datadir=datadir)
Ejemplo n.º 35
0
def main():
    arg_parser = set_up_arg_parser()

    args = arg_parser.parse_args()

    exports = args.exports or 'xml,csv'

    if args.version:
        print __version__
        sys.exit(0)

    if args.run or args.run_hazard or args.run_risk:
        # the logging will be configured in engine.py
        pass
    else:
        # configure a basic logging
        logging.basicConfig(level=logging.INFO)

    if args.config_file:
        os.environ[config.OQ_CONFIG_FILE_VAR] = \
            abspath(expanduser(args.config_file))
        config.refresh()

    if args.no_distribute:
        os.environ['OQ_DISTRIBUTE'] = 'no'

    if args.upgrade_db:
        logs.set_level('info')
        msg = upgrade_manager.what_if_I_upgrade(
            conn, extract_scripts='read_scripts')
        print msg
        if msg.startswith('Your database is already updated'):
            pass
        elif args.yes or utils.confirm('Proceed? (y/n) '):
            upgrade_manager.upgrade_db(conn)
        sys.exit(0)

    if args.version_db:
        print upgrade_manager.version_db(conn)
        sys.exit(0)

    if args.what_if_I_upgrade:
        print upgrade_manager.what_if_I_upgrade(conn)
        sys.exit(0)

    # check if the db is outdated
    outdated = dbcmd('check_outdated')
    if outdated:
        sys.exit(outdated)

    # hazard or hazard+risk
    hc_id = args.hazard_calculation_id
    if hc_id:
        hc_id = int(hc_id)
        if hc_id < 0:
            # make it possible to run `oq-engine --run job_risk.ini --hc -1`
            hc_id = dbcmd('get_hc_id', hc_id)
    if args.run:
        job_inis = map(expanduser, args.run.split(','))
        if len(job_inis) not in (1, 2):
            sys.exit('%s should be a .ini filename or a pair of filenames '
                     'separated by a comma' % args.run)
        for job_ini in job_inis:
            open(job_ini).read()  # raise an IOError if the file does not exist
        log_file = expanduser(args.log_file) \
            if args.log_file is not None else None

        if len(job_inis) == 2:
            # run hazard
            job_id = run_job(job_inis[0], args.log_level,
                             log_file, args.exports)
            # run risk
            run_job(job_inis[1], args.log_level, log_file,
                    args.exports, hazard_calculation_id=job_id)
        else:
            run_job(
                expanduser(args.run), args.log_level, log_file,
                args.exports, hazard_calculation_id=hc_id)
    # hazard
    elif args.list_hazard_calculations:
        dbcmd('list_calculations', 'hazard')
    elif args.run_hazard is not None:
        log_file = expanduser(args.log_file) \
            if args.log_file is not None else None
        run_job(expanduser(args.run_hazard), args.log_level,
                log_file, args.exports)
    elif args.delete_calculation is not None:
        dbcmd('delete_calculation', args.delete_calculation, args.yes)
    # risk
    elif args.list_risk_calculations:
        dbcmd('list_calculations', 'risk')
    elif args.run_risk is not None:
        if args.hazard_calculation_id is None:
            sys.exit(MISSING_HAZARD_MSG)
        log_file = expanduser(args.log_file) \
            if args.log_file is not None else None
        run_job(
            expanduser(args.run_risk),
            args.log_level, log_file, args.exports,
            hazard_calculation_id=hc_id)

    # export
    elif args.make_html_report:
        print 'Written', make_report(conn, args.make_html_report)
        sys.exit(0)

    elif args.list_outputs is not None:
        hc_id = dbcmd('get_hc_id', args.list_outputs)
        dbcmd('list_outputs', hc_id)
    elif args.show_view is not None:
        job_id, view_name = args.show_view
        print views.view(view_name, datastore.read(int(job_id)))
    elif args.show_log is not None:
        hc_id = dbcmd('get_hc_id', args.show_log[0])
        print dbcmd('get_log', hc_id)

    elif args.export_output is not None:
        output_id, target_dir = args.export_output
        dbcmd('export_output', int(output_id), expanduser(target_dir),
              exports)

    elif args.export_outputs is not None:
        job_id, target_dir = args.export_outputs
        hc_id = dbcmd('get_hc_id', job_id)
        dbcmd('export_outputs', hc_id, expanduser(target_dir), exports)

    elif args.delete_uncompleted_calculations:
        dbcmd('delete_uncompleted_calculations')
    else:
        arg_parser.print_usage()
Ejemplo n.º 36
0
def main():
    arg_parser = set_up_arg_parser()

    args = arg_parser.parse_args()

    exports = args.exports or 'xml,csv'

    if args.version:
        print __version__
        sys.exit(0)

    if args.run or args.run_hazard or args.run_risk:
        # the logging will be configured in engine.py
        pass
    else:
        # configure a basic logging
        logging.basicConfig(level=logging.INFO)

    if args.config_file:
        os.environ[utils.config.OQ_CONFIG_FILE_VAR] = \
            abspath(expanduser(args.config_file))
        utils.config.refresh()

    if args.no_distribute:
        os.environ[openquake.engine.NO_DISTRIBUTE_VAR] = '1'

    if args.upgrade_db:
        logs.set_level('info')
        msg = upgrade_manager.what_if_I_upgrade(
            conn, extract_scripts='read_scripts')
        print msg
        if msg.startswith('Your database is already updated'):
            pass
        elif args.yes or utils.confirm('Proceed? (y/n) '):
            upgrade_manager.upgrade_db(conn)
        sys.exit(0)

    if args.version_db:
        print upgrade_manager.version_db(conn)
        sys.exit(0)

    if args.what_if_I_upgrade:
        print upgrade_manager.what_if_I_upgrade(conn)
        sys.exit(0)

    # check if the db is outdated
    outdated = dbcmd('check_outdated')
    if outdated:
        sys.exit(outdated)

    # hazard or hazard+risk
    hc_id = args.hazard_calculation_id
    if hc_id and int(hc_id) < 0:
        # make it possible commands like `oq-engine --run job_risk.ini --hc -1`
        hc_id = dbcmd('get_hc_id', int(hc_id))
    if args.run:
        job_inis = map(expanduser, args.run.split(','))
        if len(job_inis) not in (1, 2):
            sys.exit('%s should be a .ini filename or a pair of filenames '
                     'separated by a comma' % args.run)
        for job_ini in job_inis:
            open(job_ini).read()  # raise an IOError if the file does not exist
        log_file = expanduser(args.log_file) \
            if args.log_file is not None else None

        if len(job_inis) == 2:
            # run hazard
            job_id = run_job(job_inis[0], args.log_level,
                             log_file, args.exports)
            # run risk
            run_job(job_inis[1], args.log_level, log_file,
                    args.exports, hazard_calculation_id=job_id)
        else:
            run_job(
                expanduser(args.run), args.log_level, log_file,
                args.exports, hazard_calculation_id=hc_id)
    # hazard
    elif args.list_hazard_calculations:
        dbcmd('list_calculations', 'hazard')
    elif args.run_hazard is not None:
        log_file = expanduser(args.log_file) \
            if args.log_file is not None else None
        run_job(expanduser(args.run_hazard), args.log_level,
                log_file, args.exports)
    elif args.delete_calculation is not None:
        dbcmd('delete_calculation', args.delete_calculation, args.yes)
    # risk
    elif args.list_risk_calculations:
        dbcmd('list_calculations', 'risk')
    elif args.run_risk is not None:
        if args.hazard_calculation_id is None:
            sys.exit(MISSING_HAZARD_MSG)
        log_file = expanduser(args.log_file) \
            if args.log_file is not None else None
        run_job(
            expanduser(args.run_risk),
            args.log_level, log_file, args.exports,
            hazard_calculation_id=hc_id)

    # export
    elif args.make_html_report:
        print 'Written', make_report(conn, args.make_html_report)
        sys.exit(0)

    elif args.list_outputs is not None:
        hc_id = dbcmd('get_hc_id', args.list_outputs)
        dbcmd('list_outputs', hc_id)
    elif args.show_view is not None:
        job_id, view_name = args.show_view
        print views.view(view_name, datastore.read(int(job_id)))
    elif args.show_log is not None:
        hc_id = dbcmd('get_hc_id', args.show_log[0])
        print dbcmd('get_log', hc_id)

    elif args.export_output is not None:
        output_id, target_dir = args.export_output
        dbcmd('export_output', int(output_id), expanduser(target_dir),
              exports)

    elif args.export_outputs is not None:
        job_id, target_dir = args.export_outputs
        hc_id = dbcmd('get_hc_id', job_id)
        dbcmd('export_outputs', hc_id, expanduser(target_dir), exports)

    elif args.delete_uncompleted_calculations:
        dbcmd('delete_uncompleted_calculations')
    else:
        arg_parser.print_usage()
Ejemplo n.º 37
0
from openquake.server.dbapi import Db
from openquake.engine import logs

db = Db(sqlite3.connect, DATABASE['NAME'], isolation_level=None,
        detect_types=sqlite3.PARSE_DECLTYPES, timeout=20)
# NB: I am increasing the timeout from 5 to 20 seconds to see if the random
# OperationalError: "database is locked" disappear in the WebUI tests


# bypass the DbServer and run the action directly
def dbcmd(action, *args):
    """
    A dispatcher to the database server.

    :param action: database action to perform
    :param args: arguments
    """
    return getattr(actions, action)(db, *args)


# the code here is run in development mode; for instance
# $ python manage.py runserver 0.0.0.0:8800
if __name__ == "__main__":
    os.environ.setdefault(
        "DJANGO_SETTINGS_MODULE", "openquake.server.settings")
    if '--nothreading' in sys.argv:
        logs.dbcmd = dbcmd  # turn this on when debugging
    logs.dbcmd('upgrade_db')  # make sure the DB exists
    with executor:
        execute_from_command_line(sys.argv)
Ejemplo n.º 38
0
#  -*- coding: utf-8 -*-
#  vim: tabstop=4 shiftwidth=4 softtabstop=4

#  Copyright (C) 2016 GEM Foundation

#  OpenQuake is free software: you can redistribute it and/or modify it
#  under the terms of the GNU Affero General Public License as published
#  by the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.

#  OpenQuake is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.

#  You should have received a copy of the GNU Affero General Public License
#  along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.
from openquake.engine.logs import dbcmd

if __name__ == '__main__':
    dbcmd('stop')