Esempio n. 1
0
def calc(request, id=None):
    """
    Get a list of calculations and report their id, status, job_type,
    is_running, description, and a url where more detailed information
    can be accessed.

    Responses are in JSON.
    """
    base_url = _get_base_url(request)

    user = utils.get_user_data(request)

    calc_data = _get_calcs(request.GET, user['name'], user['is_super'], id=id)

    response_data = []
    for hc_id, owner, status, job_type, is_running, desc in calc_data:
        url = urlparse.urljoin(base_url, 'v1/calc/%d' % hc_id)
        response_data.append(
            dict(id=hc_id, owner=owner, status=status, job_type=job_type,
                 is_running=is_running, description=desc, url=url))

    # if id is specified the related dictionary is returned instead the list
    if id is not None:
        [response_data] = response_data

    return HttpResponse(content=json.dumps(response_data),
                        content_type=JSON)
Esempio n. 2
0
def run_calc(request):
    """
    Run a calculation.

    :param request:
        a `django.http.HttpRequest` object.
    """
    callback_url = request.POST.get('callback_url')
    foreign_calc_id = request.POST.get('foreign_calculation_id')

    hazard_output_id = request.POST.get('hazard_output_id')
    hazard_job_id = request.POST.get('hazard_job_id')

    if hazard_output_id or hazard_job_id:
        candidates = ("job_risk.ini", "job.ini")
    else:
        candidates = ("job_hazard.ini", "job.ini")
    einfo, exctype, monitor = safely_call(
        _prepare_job, (request, hazard_output_id, hazard_job_id, candidates))
    if exctype:
        tasks.update_calculation(callback_url, status="failed", einfo=einfo)
        return HttpResponse(json.dumps(einfo.splitlines()),
                            content_type=JSON, status=500)
    if not einfo:
        msg = 'Could not find any file of the form %s' % str(candidates)
        logging.error(msg)
        return HttpResponse(content=json.dumps([msg]), content_type=JSON,
                            status=500)

    temp_dir = os.path.dirname(einfo[0])

    user = utils.get_user_data(request)

    try:
        job, _fut = submit_job(einfo[0], temp_dir, request.POST['database'],
                               user['name'], callback_url, foreign_calc_id,
                               hazard_output_id, hazard_job_id)
    except Exception as exc:  # no job created, for instance missing .xml file
        # get the exception message
        exc_msg = exc.args[0]
        if isinstance(exc_msg, bytes):
            exc_msg = exc_msg.decode('utf-8')   # make it a unicode object
        else:
            assert isinstance(exc_msg, unicode), exc_msg
        logging.error(exc_msg)
        response_data = exc_msg.splitlines()
        status = 500
    else:
        calc = oqe_models.OqJob.objects.get(pk=job.id)
        response_data = vars(calc.get_oqparam())
        response_data['job_id'] = job.id
        response_data['status'] = calc.status
        status = 200
    return HttpResponse(content=json.dumps(response_data), content_type=JSON,
                        status=status)
Esempio n. 3
0
def calc_results(request, calc_id):
    """
    Get a summarized list of calculation results for a given ``calc_id``.
    Result is a JSON array of objects containing the following attributes:

        * id
        * name
        * type (hazard_curve, hazard_map, etc.)
        * url (the exact url where the full result can be accessed)
    """
    user = utils.get_user_data(request)

    # If the specified calculation doesn't exist OR is not yet complete,
    # throw back a 404.
    try:
        oqjob = oqe_models.OqJob.objects.get(id=calc_id)
        if not user['is_super'] and oqjob.user_name != user['name']:
            return HttpResponseNotFound()
        if not oqjob.status == 'complete':
            return HttpResponseNotFound()
    except ObjectDoesNotExist:
        return HttpResponseNotFound()
    base_url = _get_base_url(request)

    # NB: export_output has as keys the list (output_type, extension)
    # so this returns an ordered map output_type -> extensions such as
    # OrderedDict([('agg_loss_curve', ['xml', 'csv']), ...])
    output_types = groupby(export_output, lambda oe: oe[0],
                           lambda oes: [e for o, e in oes])
    results = oq_engine.get_outputs(calc_id)
    if not results:
        return HttpResponseNotFound()

    response_data = []
    for result in results:
        try:  # output from the old calculators
            rtype = result.output_type
            outtypes = output_types[rtype]
        except KeyError:
            try:  # output from the datastore
                rtype = result.ds_key
                outtypes = output_types[rtype]
            except KeyError:
                continue  # non-exportable outputs should not be shown
        url = urlparse.urljoin(base_url, 'v1/calc/result/%d' % result.id)
        datum = dict(
            id=result.id, name=result.display_name, type=rtype,
            outtypes=outtypes, url=url)
        response_data.append(datum)

    return HttpResponse(content=json.dumps(response_data))
Esempio n. 4
0
def run_calc(request):
    """
    Run a calculation.

    :param request:
        a `django.http.HttpRequest` object.
        If the request has the attribute `hazard_job_id`, the results of the
        specified hazard calculations will be re-used as input by the risk
        calculation.
        The request also needs to contain the files needed to perform the
        calculation. They can be uploaded as separate files, or zipped
        together.
    """
    hazard_job_id = request.POST.get('hazard_job_id')

    if hazard_job_id:
        candidates = ("job_risk.ini", "job.ini")
    else:
        candidates = ("job_hazard.ini", "job_haz.ini", "job.ini")
    einfo, exctype, monitor = safely_call(
        _prepare_job, (request, hazard_job_id, candidates))
    if exctype:
        return HttpResponse(json.dumps(einfo.splitlines()),
                            content_type=JSON, status=500)
    if not einfo:
        msg = 'Could not find any file of the form %s' % str(candidates)
        logging.error(msg)
        return HttpResponse(content=json.dumps([msg]), content_type=JSON,
                            status=500)

    user = utils.get_user_data(request)
    try:
        job_id, fut = submit_job(einfo[0], user['name'], hazard_job_id)
        # restart the process pool at the end of each job
        fut .add_done_callback(lambda f: TaskManager.restart())
    except Exception as exc:  # no job created, for instance missing .xml file
        # get the exception message
        exc_msg = str(exc)
        logging.error(exc_msg)
        response_data = exc_msg.splitlines()
        status = 500
    else:
        response_data = dict(job_id=job_id, status='created')
        status = 200
    return HttpResponse(content=json.dumps(response_data), content_type=JSON,
                        status=status)
Esempio n. 5
0
def calc_results(request, calc_id):
    """
    Get a summarized list of calculation results for a given ``calc_id``.
    Result is a JSON array of objects containing the following attributes:

        * id
        * name
        * type (hazard_curve, hazard_map, etc.)
        * url (the exact url where the full result can be accessed)
    """
    user = utils.get_user_data(request)

    # If the specified calculation doesn't exist OR is not yet complete,
    # throw back a 404.
    try:
        info = logs.dbcmd('calc_info', calc_id)
        if user['acl_on'] and info['user_name'] != user['name']:
            return HttpResponseNotFound()
    except models.NotFound:
        return HttpResponseNotFound()
    base_url = _get_base_url(request)

    # NB: export_output has as keys the list (output_type, extension)
    # so this returns an ordered map output_type -> extensions such as
    # OrderedDict([('agg_loss_curve', ['xml', 'csv']), ...])
    output_types = groupby(export, lambda oe: oe[0],
                           lambda oes: [e for o, e in oes])
    results = logs.dbcmd('get_outputs', calc_id)
    if not results:
        return HttpResponseNotFound()

    response_data = []
    for result in results:
        try:  # output from the datastore
            rtype = result.ds_key
            # Catalina asked to remove the .txt outputs (used for the GMFs)
            outtypes = [ot for ot in output_types[rtype] if ot != 'txt']
        except KeyError:
            continue  # non-exportable outputs should not be shown
        url = urlparse.urljoin(base_url, 'v1/calc/result/%d' % result.id)
        datum = dict(
            id=result.id, name=result.display_name, type=rtype,
            outtypes=outtypes, url=url)
        response_data.append(datum)

    return HttpResponse(content=json.dumps(response_data))
Esempio n. 6
0
def calc_remove(request, calc_id):
    """
    Remove the calculation id
    """
    user = utils.get_user_data(request)['name']
    try:
        message = logs.dbcmd('del_calc', calc_id, user)
    except dbapi.NotFound:
        return HttpResponseNotFound()
    if isinstance(message, list):  # list of removed files
        return HttpResponse(content=json.dumps(message),
                            content_type=JSON,
                            status=200)
    else:  # FIXME: the error is not passed properly to the javascript
        logging.error(message)
        return HttpResponse(content=message,
                            content_type='text/plain',
                            status=500)
Esempio n. 7
0
def run_calc(request):
    """
    Run a calculation.

    :param request:
        a `django.http.HttpRequest` object.
        If the request has the attribute `hazard_job_id`, the results of the
        specified hazard calculations will be re-used as input by the risk
        calculation.
        The request also needs to contain the files needed to perform the
        calculation. They can be uploaded as separate files, or zipped
        together.
    """
    hazard_job_id = request.POST.get('hazard_job_id')

    if hazard_job_id:
        hazard_job_id = int(hazard_job_id)
        candidates = ("job_risk.ini", "job.ini")
    else:
        candidates = ("job_hazard.ini", "job_haz.ini", "job.ini")
    einfo, exctype, monitor = safely_call(_prepare_job, (request, candidates))
    if exctype:
        return HttpResponse(json.dumps(einfo.splitlines()),
                            content_type=JSON, status=500)
    if not einfo:
        msg = 'Could not find any file of the form %s' % str(candidates)
        logging.error(msg)
        return HttpResponse(content=json.dumps([msg]), content_type=JSON,
                            status=500)

    user = utils.get_user_data(request)
    try:
        job_id, _pid = submit_job(einfo[0], user['name'], hazard_job_id)
    except Exception as exc:  # no job created, for instance missing .xml file
        # get the exception message
        exc_msg = str(exc)
        logging.error(exc_msg)
        response_data = exc_msg.splitlines()
        status = 500
    else:
        response_data = dict(job_id=job_id, status='created')
        status = 200
    return HttpResponse(content=json.dumps(response_data), content_type=JSON,
                        status=status)
Esempio n. 8
0
def calc_list(request, id=None):
    # view associated to the endpoints /v1/calc/list and /v1/calc/:id/status
    """
    Get a list of calculations and report their id, status, calculation_mode,
    is_running, description, and a url where more detailed information
    can be accessed. This is called several times by the Javascript.

    Responses are in JSON.
    """
    base_url = _get_base_url(request)

    user = utils.get_user_data(request)
    allowed_users = user['group_members'] or [user['name']]
    calc_data = logs.dbcmd('get_calcs', request.GET, allowed_users,
                           user['acl_on'], id)

    response_data = []
    for hc_id, owner, status, calculation_mode, is_running, desc, pid \
            in calc_data:
        url = urlparse.urljoin(base_url, 'v1/calc/%d' % hc_id)
        abortable = False
        if is_running:
            try:
                if (psutil.Process(pid).username() == psutil.Process(
                        os.getpid()).username()):
                    abortable = True
            except psutil.NoSuchProcess:
                pass
        response_data.append(
            dict(id=hc_id,
                 owner=owner,
                 calculation_mode=calculation_mode,
                 status=status,
                 is_running=bool(is_running),
                 description=desc,
                 url=url,
                 abortable=abortable))

    # if id is specified the related dictionary is returned instead the list
    if id is not None:
        [response_data] = response_data

    return HttpResponse(content=json.dumps(response_data), content_type=JSON)
Esempio n. 9
0
def run_calc(request):
    """
    Run a calculation.

    :param request:
        a `django.http.HttpRequest` object.
    """
    hazard_job_id = request.POST.get('hazard_job_id')

    if hazard_job_id:
        candidates = ("job_risk.ini", "job.ini")
    else:
        candidates = ("job_hazard.ini", "job_haz.ini", "job.ini")
    einfo, exctype, monitor = safely_call(
        _prepare_job, (request, hazard_job_id, candidates))
    if exctype:
        return HttpResponse(json.dumps(einfo.splitlines()),
                            content_type=JSON, status=500)
    if not einfo:
        msg = 'Could not find any file of the form %s' % str(candidates)
        logging.error(msg)
        return HttpResponse(content=json.dumps([msg]), content_type=JSON,
                            status=500)

    user = utils.get_user_data(request)
    try:
        job_id, _fut = submit_job(einfo[0], user['name'], hazard_job_id)
    except Exception as exc:  # no job created, for instance missing .xml file
        # get the exception message
        exc_msg = exc.args[0]
        if isinstance(exc_msg, bytes):
            exc_msg = exc_msg.decode('utf-8')   # make it a unicode object
        else:
            assert isinstance(exc_msg, unicode), exc_msg
        logging.error(exc_msg)
        response_data = exc_msg.splitlines()
        status = 500
    else:
        response_data = dict(job_id=job_id, status='executing')
        status = 200
    return HttpResponse(content=json.dumps(response_data), content_type=JSON,
                        status=status)
Esempio n. 10
0
def calc_remove(request, calc_id):
    """
    Remove the calculation id
    """
    user = utils.get_user_data(request)['name']
    try:
        message = logs.dbcmd('del_calc', calc_id, user)
    except dbapi.NotFound:
        return HttpResponseNotFound()

    if 'success' in message:
        return HttpResponse(content=json.dumps(message),
                            content_type=JSON, status=200)
    elif 'error' in message:
        logging.error(message['error'])
        return HttpResponse(content=json.dumps(message),
                            content_type=JSON, status=403)
    else:
        # This is an untrapped server error
        logging.error(message)
        return HttpResponse(content=message,
                            content_type='text/plain', status=500)
Esempio n. 11
0
def calc_abort(request, calc_id):
    """
    Abort the given calculation, it is it running
    """
    job = logs.dbcmd('get_job', calc_id)
    if job is None:
        message = {'error': 'Unknown job %s' % calc_id}
        return HttpResponse(content=json.dumps(message), content_type=JSON)

    if job.status not in ('executing', 'running'):
        message = {'error': 'Job %s is not running' % job.id}
        return HttpResponse(content=json.dumps(message), content_type=JSON)

    user = utils.get_user_data(request)
    info = logs.dbcmd('calc_info', calc_id)
    allowed_users = user['group_members'] or [user['name']]
    if user['acl_on'] and info['user_name'] not in allowed_users:
        message = {
            'error': ('User %s has no permission to abort job %s' %
                      (info['user_name'], job.id))
        }
        return HttpResponse(content=json.dumps(message),
                            content_type=JSON,
                            status=403)

    if job.pid:  # is a spawned job
        try:
            os.kill(job.pid, signal.SIGTERM)
        except Exception as exc:
            logging.error(exc)
        else:
            logging.warn('Aborting job %d, pid=%d', job.id, job.pid)
            logs.dbcmd('set_status', job.id, 'aborted')
        message = {'success': 'Killing job %d' % job.id}
        return HttpResponse(content=json.dumps(message), content_type=JSON)

    message = {'error': 'PID for job %s not found' % job.id}
    return HttpResponse(content=json.dumps(message), content_type=JSON)
Esempio n. 12
0
def get_datastore(request, job_id):
    """
    Download a full datastore file.

    :param request:
        `django.http.HttpRequest` object.
    :param job_id:
        The id of the requested datastore
    :returns:
        A `django.http.HttpResponse` containing the content
        of the requested artifact, if present, else throws a 404
    """
    user = utils.get_user_data(request)
    username = user['name'] if user['acl_on'] else None
    job = logs.dbcmd('get_job', int(job_id), username)
    if job is None:
        return HttpResponseNotFound()

    fname = job.ds_calc_dir + '.hdf5'
    response = FileResponse(FileWrapper(open(fname, 'rb')), content_type=HDF5)
    response['Content-Disposition'] = ('attachment; filename=%s' %
                                       os.path.basename(fname))
    return response