def worker(self, sock): """ :param sock: a zeromq.Socket of kind PULL receiving (cmd, args) """ setproctitle('oq-zworker') with sock: for cmd, args, mon in sock: parallel.safely_call(cmd, args, mon)
def worker(self, sock): """ :param sock: a zeromq.Socket of kind PULL receiving (cmd, args) """ setproctitle('oq-zworker') with sock: for cmd, args, taskno, mon in sock: parallel.safely_call(cmd, args, taskno, mon)
def worker(sock, executing): """ :param sock: a zeromq.Socket of kind PULL :param executing: a path inside /tmp/calc_XXX """ setproctitle('oq-zworker') with sock: for cmd, args, taskno, mon in sock: fname = os.path.join(executing, '%s-%s' % (mon.calc_id, taskno)) open(fname, 'w').close() parallel.safely_call(cmd, args, taskno, mon) os.remove(fname)
def worker(sock, executing): """ :param sock: a zeromq.Socket of kind PULL :param executing: a path inside /tmp/calc_XXX """ setproctitle('oq-zworker') with sock: for cmd, args, taskno, mon in sock: fname = os.path.join(executing, '%s-%s' % (mon.calc_id, taskno)) # NB: very hackish way of keeping track of the running tasks, # used in get_executing, could litter the file system open(fname, 'w').close() parallel.safely_call(cmd, args, taskno, mon) os.remove(fname)
def dworker(self, sock): # a database worker responding to commands with sock: for cmd_ in sock: cmd, args = cmd_[0], cmd_[1:] if cmd == 'getpid': sock.send(self.pid) continue try: func = getattr(actions, cmd) except AttributeError: # missing action sock.send(safely_call(self.db, (cmd,) + args)) else: sock.send(safely_call(func, (self.db,) + args))
def dworker(self, sock): # a database worker responding to commands with sock: for cmd_ in sock: cmd, args = cmd_[0], cmd_[1:] if cmd == 'getpid': sock.send(self.pid) continue try: func = getattr(actions, cmd) except AttributeError: # SQL string sock.send(safely_call(self.db, (cmd, ) + args)) else: # action sock.send(safely_call(func, (self.db, ) + args))
def test_no_flush(self): mon = parallel.Monitor('test') res = parallel.safely_call(get_len, ('ab', mon)) self.assertIn('Monitor(\'test\').flush() must not be called' ' by get_len!', res[0]) self.assertEqual(res[1], RuntimeError) self.assertEqual(res[2].operation, mon.operation)
def test_no_flush(self): mon = parallel.Monitor('test') res = parallel.safely_call(get_len, ('ab', mon)) self.assertIn( 'Monitor(\'test\').flush() must not be called in a worker', res[0]) self.assertEqual(res[1], RuntimeError) self.assertEqual(res[2].operation, mon.operation)
def dworker(self, sock): # a database worker responding to commands with sock: for cmd_ in sock: cmd, args = cmd_[0], cmd_[1:] if cmd == 'getpid': sock.send(self.pid) continue elif cmd.startswith('zmq_') and self.zmaster: msg = getattr(self.zmaster, cmd[4:])() logging.info(msg) sock.send(msg) continue try: func = getattr(actions, cmd) except AttributeError: # SQL string sock.send(safely_call(self.db, (cmd, ) + args)) else: # action sock.send(safely_call(func, (self.db, ) + args))
def dworker(self, sock): # a database worker responding to commands with sock: for cmd_ in sock: cmd, args = cmd_[0], cmd_[1:] if cmd == 'getpid': sock.send(self.pid) continue elif cmd.startswith('workers_'): # engine.run_jobs calls logs.dbcmd(cmd) msg = getattr(p, cmd)() logging.info(msg) sock.send(msg) continue try: func = getattr(actions, cmd) except AttributeError: # SQL string sock.send(p.safely_call(self.db, (cmd,) + args)) else: # action sock.send(p.safely_call(func, (self.db,) + args))
def dworker(self, sock): # a database worker responding to commands for cmd_ in sock: cmd, args = cmd_[0], cmd_[1:] if cmd == 'getpid': sock.send((self.pid, None, None)) continue try: func = getattr(actions, cmd) except AttributeError: sock.send(('Invalid command ' + cmd, ValueError, None)) else: sock.send(safely_call(func, (self.db, ) + args))
def calc_run(request): """ Run a calculation. :param request: a `django.http.HttpRequest` object. If the request has the attribute `hazard_job_id`, the results of the specified hazard calculations will be re-used as input by the risk calculation. The request also needs to contain the files needed to perform the calculation. They can be uploaded as separate files, or zipped together. """ hazard_job_id = request.POST.get('hazard_job_id') job_ini = request.POST.get('job_ini') if hazard_job_id: hazard_job_id = int(hazard_job_id) candidates = [job_ini] if job_ini else ("job_risk.ini", "job.ini") else: candidates = [job_ini] if job_ini else ("job_hazard.ini", "job_haz.ini", "job.ini") result = safely_call(_prepare_job, (request, candidates)) if result.tb_str: return HttpResponse(json.dumps(result.tb_str.splitlines()), content_type=JSON, status=500) inifiles = result.get() if not inifiles: msg = 'Could not find any file of the form %s' % str(candidates) logging.error(msg) return HttpResponse(content=json.dumps([msg]), content_type=JSON, status=500) user = utils.get_user(request) try: job_id, pid = submit_job(inifiles[0], user, hazard_job_id) except Exception as exc: # no job created, for instance missing .xml file # get the exception message exc_msg = str(exc) logging.error(exc_msg) response_data = exc_msg.splitlines() status = 500 else: response_data = dict(job_id=job_id, status='created', pid=pid) status = 200 return HttpResponse(content=json.dumps(response_data), content_type=JSON, status=status)
def run_calc(request): """ Run a calculation. :param request: a `django.http.HttpRequest` object. If the request has the attribute `hazard_job_id`, the results of the specified hazard calculations will be re-used as input by the risk calculation. The request also needs to contain the files needed to perform the calculation. They can be uploaded as separate files, or zipped together. """ hazard_job_id = request.POST.get('hazard_job_id') if hazard_job_id: hazard_job_id = int(hazard_job_id) candidates = ("job_risk.ini", "job.ini") else: candidates = ("job_hazard.ini", "job_haz.ini", "job.ini") einfo, exctype, monitor = safely_call(_prepare_job, (request, candidates)) if exctype: return HttpResponse(json.dumps(einfo.splitlines()), content_type=JSON, status=500) if not einfo: msg = 'Could not find any file of the form %s' % str(candidates) logging.error(msg) return HttpResponse(content=json.dumps([msg]), content_type=JSON, status=500) user = utils.get_user_data(request) try: job_id, fut = submit_job(einfo[0], user['name'], hazard_job_id) # restart the process pool at the end of each job fut.add_done_callback(lambda f: Starmap.restart()) except Exception as exc: # no job created, for instance missing .xml file # get the exception message exc_msg = str(exc) logging.error(exc_msg) response_data = exc_msg.splitlines() status = 500 else: response_data = dict(job_id=job_id, status='created') status = 200 return HttpResponse(content=json.dumps(response_data), content_type=JSON, status=status)
def run_calc(request): """ Run a calculation. :param request: a `django.http.HttpRequest` object. If the request has the attribute `hazard_job_id`, the results of the specified hazard calculations will be re-used as input by the risk calculation. The request also needs to contain the files needed to perform the calculation. They can be uploaded as separate files, or zipped together. """ hazard_job_id = request.POST.get('hazard_job_id') if hazard_job_id: candidates = ("job_risk.ini", "job.ini") else: candidates = ("job_hazard.ini", "job_haz.ini", "job.ini") einfo, exctype, monitor = safely_call( _prepare_job, (request, hazard_job_id, candidates)) if exctype: return HttpResponse(json.dumps(einfo.splitlines()), content_type=JSON, status=500) if not einfo: msg = 'Could not find any file of the form %s' % str(candidates) logging.error(msg) return HttpResponse(content=json.dumps([msg]), content_type=JSON, status=500) user = utils.get_user_data(request) try: job_id, fut = submit_job(einfo[0], user['name'], hazard_job_id) # restart the process pool at the end of each job fut .add_done_callback(lambda f: TaskManager.restart()) except Exception as exc: # no job created, for instance missing .xml file # get the exception message exc_msg = str(exc) logging.error(exc_msg) response_data = exc_msg.splitlines() status = 500 else: response_data = dict(job_id=job_id, status='created') status = 200 return HttpResponse(content=json.dumps(response_data), content_type=JSON, status=status)
def calc_run(request): """ Run a calculation. :param request: a `django.http.HttpRequest` object. If the request has the attribute `hazard_job_id`, the results of the specified hazard calculations will be re-used as input by the risk calculation. The request also needs to contain the files needed to perform the calculation. They can be uploaded as separate files, or zipped together. """ hazard_job_id = request.POST.get('hazard_job_id') if hazard_job_id: hazard_job_id = int(hazard_job_id) candidates = ("job_risk.ini", "job.ini") else: candidates = ("job_hazard.ini", "job_haz.ini", "job.ini") result = safely_call(_prepare_job, (request, candidates)) if result.tb_str: return HttpResponse(json.dumps(result.tb_str.splitlines()), content_type=JSON, status=500) inifiles = result.get() if not inifiles: msg = 'Could not find any file of the form %s' % str(candidates) logging.error(msg) return HttpResponse(content=json.dumps([msg]), content_type=JSON, status=500) user = utils.get_user(request) try: job_id, pid = submit_job(inifiles[0], user, hazard_job_id) except Exception as exc: # no job created, for instance missing .xml file # get the exception message exc_msg = str(exc) logging.error(exc_msg) response_data = exc_msg.splitlines() status = 500 else: response_data = dict(job_id=job_id, status='created', pid=pid) status = 200 return HttpResponse(content=json.dumps(response_data), content_type=JSON, status=status)