def worker(self, sock):
"""
:param sock: a zeromq.Socket of kind PULL receiving (cmd, args)
"""
setproctitle('oq-zworker')
with sock:
for cmd, args, mon in sock:
parallel.safely_call(cmd, args, mon)
def worker(self, sock):
"""
:param sock: a zeromq.Socket of kind PULL receiving (cmd, args)
"""
setproctitle('oq-zworker')
with sock:
for cmd, args, taskno, mon in sock:
parallel.safely_call(cmd, args, taskno, mon)
def worker(sock, executing):
"""
:param sock: a zeromq.Socket of kind PULL
:param executing: a path inside /tmp/calc_XXX
"""
setproctitle('oq-zworker')
with sock:
for cmd, args, taskno, mon in sock:
fname = os.path.join(executing, '%s-%s' % (mon.calc_id, taskno))
open(fname, 'w').close()
parallel.safely_call(cmd, args, taskno, mon)
os.remove(fname)
def worker(sock, executing):
"""
:param sock: a zeromq.Socket of kind PULL
:param executing: a path inside /tmp/calc_XXX
"""
setproctitle('oq-zworker')
with sock:
for cmd, args, taskno, mon in sock:
fname = os.path.join(executing, '%s-%s' % (mon.calc_id, taskno))
# NB: very hackish way of keeping track of the running tasks,
# used in get_executing, could litter the file system
open(fname, 'w').close()
parallel.safely_call(cmd, args, taskno, mon)
os.remove(fname)
def dworker(self, sock):
# a database worker responding to commands
with sock:
for cmd_ in sock:
cmd, args = cmd_[0], cmd_[1:]
if cmd == 'getpid':
sock.send(self.pid)
continue
try:
func = getattr(actions, cmd)
except AttributeError: # missing action
sock.send(safely_call(self.db, (cmd,) + args))
else:
sock.send(safely_call(func, (self.db,) + args))
def dworker(self, sock):
# a database worker responding to commands
with sock:
for cmd_ in sock:
cmd, args = cmd_[0], cmd_[1:]
if cmd == 'getpid':
sock.send(self.pid)
continue
try:
func = getattr(actions, cmd)
except AttributeError: # SQL string
sock.send(safely_call(self.db, (cmd, ) + args))
else: # action
sock.send(safely_call(func, (self.db, ) + args))
def test_no_flush(self):
mon = parallel.Monitor('test')
res = parallel.safely_call(get_len, ('ab', mon))
self.assertIn('Monitor(\'test\').flush() must not be called'
' by get_len!', res[0])
self.assertEqual(res[1], RuntimeError)
self.assertEqual(res[2].operation, mon.operation)
def test_no_flush(self):
mon = parallel.Monitor('test')
res = parallel.safely_call(get_len, ('ab', mon))
self.assertIn(
'Monitor(\'test\').flush() must not be called in a worker', res[0])
self.assertEqual(res[1], RuntimeError)
self.assertEqual(res[2].operation, mon.operation)
def dworker(self, sock):
# a database worker responding to commands
with sock:
for cmd_ in sock:
cmd, args = cmd_[0], cmd_[1:]
if cmd == 'getpid':
sock.send(self.pid)
continue
elif cmd.startswith('zmq_') and self.zmaster:
msg = getattr(self.zmaster, cmd[4:])()
logging.info(msg)
sock.send(msg)
continue
try:
func = getattr(actions, cmd)
except AttributeError: # SQL string
sock.send(safely_call(self.db, (cmd, ) + args))
else: # action
sock.send(safely_call(func, (self.db, ) + args))
def dworker(self, sock):
# a database worker responding to commands
with sock:
for cmd_ in sock:
cmd, args = cmd_[0], cmd_[1:]
if cmd == 'getpid':
sock.send(self.pid)
continue
elif cmd.startswith('workers_'):
# engine.run_jobs calls logs.dbcmd(cmd)
msg = getattr(p, cmd)()
logging.info(msg)
sock.send(msg)
continue
try:
func = getattr(actions, cmd)
except AttributeError: # SQL string
sock.send(p.safely_call(self.db, (cmd,) + args))
else: # action
sock.send(p.safely_call(func, (self.db,) + args))
def dworker(self, sock):
# a database worker responding to commands
for cmd_ in sock:
cmd, args = cmd_[0], cmd_[1:]
if cmd == 'getpid':
sock.send((self.pid, None, None))
continue
try:
func = getattr(actions, cmd)
except AttributeError:
sock.send(('Invalid command ' + cmd, ValueError, None))
else:
sock.send(safely_call(func, (self.db, ) + args))
def calc_run(request):
"""
Run a calculation.
:param request:
a `django.http.HttpRequest` object.
If the request has the attribute `hazard_job_id`, the results of the
specified hazard calculations will be re-used as input by the risk
calculation.
The request also needs to contain the files needed to perform the
calculation. They can be uploaded as separate files, or zipped
together.
"""
hazard_job_id = request.POST.get('hazard_job_id')
job_ini = request.POST.get('job_ini')
if hazard_job_id:
hazard_job_id = int(hazard_job_id)
candidates = [job_ini] if job_ini else ("job_risk.ini", "job.ini")
else:
candidates = [job_ini] if job_ini else ("job_hazard.ini",
"job_haz.ini", "job.ini")
result = safely_call(_prepare_job, (request, candidates))
if result.tb_str:
return HttpResponse(json.dumps(result.tb_str.splitlines()),
content_type=JSON,
status=500)
inifiles = result.get()
if not inifiles:
msg = 'Could not find any file of the form %s' % str(candidates)
logging.error(msg)
return HttpResponse(content=json.dumps([msg]),
content_type=JSON,
status=500)
user = utils.get_user(request)
try:
job_id, pid = submit_job(inifiles[0], user, hazard_job_id)
except Exception as exc: # no job created, for instance missing .xml file
# get the exception message
exc_msg = str(exc)
logging.error(exc_msg)
response_data = exc_msg.splitlines()
status = 500
else:
response_data = dict(job_id=job_id, status='created', pid=pid)
status = 200
return HttpResponse(content=json.dumps(response_data),
content_type=JSON,
status=status)
def run_calc(request):
"""
Run a calculation.
:param request:
a `django.http.HttpRequest` object.
If the request has the attribute `hazard_job_id`, the results of the
specified hazard calculations will be re-used as input by the risk
calculation.
The request also needs to contain the files needed to perform the
calculation. They can be uploaded as separate files, or zipped
together.
"""
hazard_job_id = request.POST.get('hazard_job_id')
if hazard_job_id:
hazard_job_id = int(hazard_job_id)
candidates = ("job_risk.ini", "job.ini")
else:
candidates = ("job_hazard.ini", "job_haz.ini", "job.ini")
einfo, exctype, monitor = safely_call(_prepare_job, (request, candidates))
if exctype:
return HttpResponse(json.dumps(einfo.splitlines()),
content_type=JSON,
status=500)
if not einfo:
msg = 'Could not find any file of the form %s' % str(candidates)
logging.error(msg)
return HttpResponse(content=json.dumps([msg]),
content_type=JSON,
status=500)
user = utils.get_user_data(request)
try:
job_id, fut = submit_job(einfo[0], user['name'], hazard_job_id)
# restart the process pool at the end of each job
fut.add_done_callback(lambda f: Starmap.restart())
except Exception as exc: # no job created, for instance missing .xml file
# get the exception message
exc_msg = str(exc)
logging.error(exc_msg)
response_data = exc_msg.splitlines()
status = 500
else:
response_data = dict(job_id=job_id, status='created')
status = 200
return HttpResponse(content=json.dumps(response_data),
content_type=JSON,
status=status)
def run_calc(request):
"""
Run a calculation.
:param request:
a `django.http.HttpRequest` object.
If the request has the attribute `hazard_job_id`, the results of the
specified hazard calculations will be re-used as input by the risk
calculation.
The request also needs to contain the files needed to perform the
calculation. They can be uploaded as separate files, or zipped
together.
"""
hazard_job_id = request.POST.get('hazard_job_id')
if hazard_job_id:
candidates = ("job_risk.ini", "job.ini")
else:
candidates = ("job_hazard.ini", "job_haz.ini", "job.ini")
einfo, exctype, monitor = safely_call(
_prepare_job, (request, hazard_job_id, candidates))
if exctype:
return HttpResponse(json.dumps(einfo.splitlines()),
content_type=JSON, status=500)
if not einfo:
msg = 'Could not find any file of the form %s' % str(candidates)
logging.error(msg)
return HttpResponse(content=json.dumps([msg]), content_type=JSON,
status=500)
user = utils.get_user_data(request)
try:
job_id, fut = submit_job(einfo[0], user['name'], hazard_job_id)
# restart the process pool at the end of each job
fut .add_done_callback(lambda f: TaskManager.restart())
except Exception as exc: # no job created, for instance missing .xml file
# get the exception message
exc_msg = str(exc)
logging.error(exc_msg)
response_data = exc_msg.splitlines()
status = 500
else:
response_data = dict(job_id=job_id, status='created')
status = 200
return HttpResponse(content=json.dumps(response_data), content_type=JSON,
status=status)
def calc_run(request):
"""
Run a calculation.
:param request:
a `django.http.HttpRequest` object.
If the request has the attribute `hazard_job_id`, the results of the
specified hazard calculations will be re-used as input by the risk
calculation.
The request also needs to contain the files needed to perform the
calculation. They can be uploaded as separate files, or zipped
together.
"""
hazard_job_id = request.POST.get('hazard_job_id')
if hazard_job_id:
hazard_job_id = int(hazard_job_id)
candidates = ("job_risk.ini", "job.ini")
else:
candidates = ("job_hazard.ini", "job_haz.ini", "job.ini")
result = safely_call(_prepare_job, (request, candidates))
if result.tb_str:
return HttpResponse(json.dumps(result.tb_str.splitlines()),
content_type=JSON, status=500)
inifiles = result.get()
if not inifiles:
msg = 'Could not find any file of the form %s' % str(candidates)
logging.error(msg)
return HttpResponse(content=json.dumps([msg]), content_type=JSON,
status=500)
user = utils.get_user(request)
try:
job_id, pid = submit_job(inifiles[0], user, hazard_job_id)
except Exception as exc: # no job created, for instance missing .xml file
# get the exception message
exc_msg = str(exc)
logging.error(exc_msg)
response_data = exc_msg.splitlines()
status = 500
else:
response_data = dict(job_id=job_id, status='created', pid=pid)
status = 200
return HttpResponse(content=json.dumps(response_data), content_type=JSON,
status=status)