Пример #1
0
 def worker(self, sock):
     """
     :param sock: a zeromq.Socket of kind PULL receiving (cmd, args)
     """
     setproctitle('oq-zworker')
     with sock:
         for cmd, args, mon in sock:
             parallel.safely_call(cmd, args, mon)
Пример #2
0
 def worker(self, sock):
     """
     :param sock: a zeromq.Socket of kind PULL receiving (cmd, args)
     """
     setproctitle('oq-zworker')
     with sock:
         for cmd, args, taskno, mon in sock:
             parallel.safely_call(cmd, args, taskno, mon)
Пример #3
0
def worker(sock, executing):
    """
    :param sock: a zeromq.Socket of kind PULL
    :param executing: a path inside /tmp/calc_XXX
    """
    setproctitle('oq-zworker')
    with sock:
        for cmd, args, taskno, mon in sock:
            fname = os.path.join(executing, '%s-%s' % (mon.calc_id, taskno))
            open(fname, 'w').close()
            parallel.safely_call(cmd, args, taskno, mon)
            os.remove(fname)
Пример #4
0
def worker(sock, executing):
    """
    :param sock: a zeromq.Socket of kind PULL
    :param executing: a path inside /tmp/calc_XXX
    """
    setproctitle('oq-zworker')
    with sock:
        for cmd, args, taskno, mon in sock:
            fname = os.path.join(executing, '%s-%s' % (mon.calc_id, taskno))
            # NB: very hackish way of keeping track of the running tasks,
            # used in get_executing, could litter the file system
            open(fname, 'w').close()
            parallel.safely_call(cmd, args, taskno, mon)
            os.remove(fname)
Пример #5
0
 def dworker(self, sock):
     # a database worker responding to commands
     with sock:
         for cmd_ in sock:
             cmd, args = cmd_[0], cmd_[1:]
             if cmd == 'getpid':
                 sock.send(self.pid)
                 continue
             try:
                 func = getattr(actions, cmd)
             except AttributeError:  # missing action
                 sock.send(safely_call(self.db, (cmd,) + args))
             else:
                 sock.send(safely_call(func, (self.db,) + args))
Пример #6
0
 def dworker(self, sock):
     # a database worker responding to commands
     with sock:
         for cmd_ in sock:
             cmd, args = cmd_[0], cmd_[1:]
             if cmd == 'getpid':
                 sock.send(self.pid)
                 continue
             try:
                 func = getattr(actions, cmd)
             except AttributeError:  # SQL string
                 sock.send(safely_call(self.db, (cmd, ) + args))
             else:  # action
                 sock.send(safely_call(func, (self.db, ) + args))
Пример #7
0
 def test_no_flush(self):
     mon = parallel.Monitor('test')
     res = parallel.safely_call(get_len, ('ab', mon))
     self.assertIn('Monitor(\'test\').flush() must not be called'
                   ' by get_len!', res[0])
     self.assertEqual(res[1], RuntimeError)
     self.assertEqual(res[2].operation, mon.operation)
Пример #8
0
 def test_no_flush(self):
     mon = parallel.Monitor('test')
     res = parallel.safely_call(get_len, ('ab', mon))
     self.assertIn(
         'Monitor(\'test\').flush() must not be called in a worker', res[0])
     self.assertEqual(res[1], RuntimeError)
     self.assertEqual(res[2].operation, mon.operation)
Пример #9
0
 def dworker(self, sock):
     # a database worker responding to commands
     with sock:
         for cmd_ in sock:
             cmd, args = cmd_[0], cmd_[1:]
             if cmd == 'getpid':
                 sock.send(self.pid)
                 continue
             elif cmd.startswith('zmq_') and self.zmaster:
                 msg = getattr(self.zmaster, cmd[4:])()
                 logging.info(msg)
                 sock.send(msg)
                 continue
             try:
                 func = getattr(actions, cmd)
             except AttributeError:  # SQL string
                 sock.send(safely_call(self.db, (cmd, ) + args))
             else:  # action
                 sock.send(safely_call(func, (self.db, ) + args))
Пример #10
0
 def dworker(self, sock):
     # a database worker responding to commands
     with sock:
         for cmd_ in sock:
             cmd, args = cmd_[0], cmd_[1:]
             if cmd == 'getpid':
                 sock.send(self.pid)
                 continue
             elif cmd.startswith('workers_'):
                 # engine.run_jobs calls logs.dbcmd(cmd)
                 msg = getattr(p, cmd)()
                 logging.info(msg)
                 sock.send(msg)
                 continue
             try:
                 func = getattr(actions, cmd)
             except AttributeError:  # SQL string
                 sock.send(p.safely_call(self.db, (cmd,) + args))
             else:  # action
                 sock.send(p.safely_call(func, (self.db,) + args))
Пример #11
0
 def dworker(self, sock):
     # a database worker responding to commands
     for cmd_ in sock:
         cmd, args = cmd_[0], cmd_[1:]
         if cmd == 'getpid':
             sock.send((self.pid, None, None))
             continue
         try:
             func = getattr(actions, cmd)
         except AttributeError:
             sock.send(('Invalid command ' + cmd, ValueError, None))
         else:
             sock.send(safely_call(func, (self.db, ) + args))
Пример #12
0
def calc_run(request):
    """
    Run a calculation.

    :param request:
        a `django.http.HttpRequest` object.
        If the request has the attribute `hazard_job_id`, the results of the
        specified hazard calculations will be re-used as input by the risk
        calculation.
        The request also needs to contain the files needed to perform the
        calculation. They can be uploaded as separate files, or zipped
        together.
    """
    hazard_job_id = request.POST.get('hazard_job_id')
    job_ini = request.POST.get('job_ini')

    if hazard_job_id:
        hazard_job_id = int(hazard_job_id)
        candidates = [job_ini] if job_ini else ("job_risk.ini", "job.ini")
    else:
        candidates = [job_ini] if job_ini else ("job_hazard.ini",
                                                "job_haz.ini", "job.ini")
    result = safely_call(_prepare_job, (request, candidates))
    if result.tb_str:
        return HttpResponse(json.dumps(result.tb_str.splitlines()),
                            content_type=JSON,
                            status=500)
    inifiles = result.get()
    if not inifiles:
        msg = 'Could not find any file of the form %s' % str(candidates)
        logging.error(msg)
        return HttpResponse(content=json.dumps([msg]),
                            content_type=JSON,
                            status=500)

    user = utils.get_user(request)
    try:
        job_id, pid = submit_job(inifiles[0], user, hazard_job_id)
    except Exception as exc:  # no job created, for instance missing .xml file
        # get the exception message
        exc_msg = str(exc)
        logging.error(exc_msg)
        response_data = exc_msg.splitlines()
        status = 500
    else:
        response_data = dict(job_id=job_id, status='created', pid=pid)
        status = 200
    return HttpResponse(content=json.dumps(response_data),
                        content_type=JSON,
                        status=status)
Пример #13
0
def run_calc(request):
    """
    Run a calculation.

    :param request:
        a `django.http.HttpRequest` object.
        If the request has the attribute `hazard_job_id`, the results of the
        specified hazard calculations will be re-used as input by the risk
        calculation.
        The request also needs to contain the files needed to perform the
        calculation. They can be uploaded as separate files, or zipped
        together.
    """
    hazard_job_id = request.POST.get('hazard_job_id')

    if hazard_job_id:
        hazard_job_id = int(hazard_job_id)
        candidates = ("job_risk.ini", "job.ini")
    else:
        candidates = ("job_hazard.ini", "job_haz.ini", "job.ini")
    einfo, exctype, monitor = safely_call(_prepare_job, (request, candidates))
    if exctype:
        return HttpResponse(json.dumps(einfo.splitlines()),
                            content_type=JSON,
                            status=500)
    if not einfo:
        msg = 'Could not find any file of the form %s' % str(candidates)
        logging.error(msg)
        return HttpResponse(content=json.dumps([msg]),
                            content_type=JSON,
                            status=500)

    user = utils.get_user_data(request)
    try:
        job_id, fut = submit_job(einfo[0], user['name'], hazard_job_id)
        # restart the process pool at the end of each job
        fut.add_done_callback(lambda f: Starmap.restart())
    except Exception as exc:  # no job created, for instance missing .xml file
        # get the exception message
        exc_msg = str(exc)
        logging.error(exc_msg)
        response_data = exc_msg.splitlines()
        status = 500
    else:
        response_data = dict(job_id=job_id, status='created')
        status = 200
    return HttpResponse(content=json.dumps(response_data),
                        content_type=JSON,
                        status=status)
Пример #14
0
def run_calc(request):
    """
    Run a calculation.

    :param request:
        a `django.http.HttpRequest` object.
        If the request has the attribute `hazard_job_id`, the results of the
        specified hazard calculations will be re-used as input by the risk
        calculation.
        The request also needs to contain the files needed to perform the
        calculation. They can be uploaded as separate files, or zipped
        together.
    """
    hazard_job_id = request.POST.get('hazard_job_id')

    if hazard_job_id:
        candidates = ("job_risk.ini", "job.ini")
    else:
        candidates = ("job_hazard.ini", "job_haz.ini", "job.ini")
    einfo, exctype, monitor = safely_call(
        _prepare_job, (request, hazard_job_id, candidates))
    if exctype:
        return HttpResponse(json.dumps(einfo.splitlines()),
                            content_type=JSON, status=500)
    if not einfo:
        msg = 'Could not find any file of the form %s' % str(candidates)
        logging.error(msg)
        return HttpResponse(content=json.dumps([msg]), content_type=JSON,
                            status=500)

    user = utils.get_user_data(request)
    try:
        job_id, fut = submit_job(einfo[0], user['name'], hazard_job_id)
        # restart the process pool at the end of each job
        fut .add_done_callback(lambda f: TaskManager.restart())
    except Exception as exc:  # no job created, for instance missing .xml file
        # get the exception message
        exc_msg = str(exc)
        logging.error(exc_msg)
        response_data = exc_msg.splitlines()
        status = 500
    else:
        response_data = dict(job_id=job_id, status='created')
        status = 200
    return HttpResponse(content=json.dumps(response_data), content_type=JSON,
                        status=status)
Пример #15
0
def calc_run(request):
    """
    Run a calculation.

    :param request:
        a `django.http.HttpRequest` object.
        If the request has the attribute `hazard_job_id`, the results of the
        specified hazard calculations will be re-used as input by the risk
        calculation.
        The request also needs to contain the files needed to perform the
        calculation. They can be uploaded as separate files, or zipped
        together.
    """
    hazard_job_id = request.POST.get('hazard_job_id')

    if hazard_job_id:
        hazard_job_id = int(hazard_job_id)
        candidates = ("job_risk.ini", "job.ini")
    else:
        candidates = ("job_hazard.ini", "job_haz.ini", "job.ini")
    result = safely_call(_prepare_job, (request, candidates))
    if result.tb_str:
        return HttpResponse(json.dumps(result.tb_str.splitlines()),
                            content_type=JSON, status=500)
    inifiles = result.get()
    if not inifiles:
        msg = 'Could not find any file of the form %s' % str(candidates)
        logging.error(msg)
        return HttpResponse(content=json.dumps([msg]), content_type=JSON,
                            status=500)

    user = utils.get_user(request)
    try:
        job_id, pid = submit_job(inifiles[0], user, hazard_job_id)
    except Exception as exc:  # no job created, for instance missing .xml file
        # get the exception message
        exc_msg = str(exc)
        logging.error(exc_msg)
        response_data = exc_msg.splitlines()
        status = 500
    else:
        response_data = dict(job_id=job_id, status='created', pid=pid)
        status = 200
    return HttpResponse(content=json.dumps(response_data), content_type=JSON,
                        status=status)