def smart_run(job_ini, oqparam, log_level, log_file, exports, reuse_hazard):
"""
Run calculations by storing their hazard checksum and reusing previous
calculations if requested.
"""
haz_checksum = readinput.get_checksum32(oqparam, hazard=True)
# retrieve an old calculation with the right checksum, if any
job = logs.dbcmd('get_job_from_checksum', haz_checksum)
reuse = reuse_hazard and job and os.path.exists(job.ds_calc_dir + '.hdf5')
# recompute the hazard and store the checksum
ebr = (oqparam.calculation_mode == 'event_based_risk' and
'gmfs' not in oqparam.inputs)
if ebr:
kw = dict(calculation_mode='event_based')
if (oqparam.sites or 'sites' in oqparam.inputs or
'site_model' in oqparam.inputs):
# remove exposure from the hazard
kw['exposure_file'] = ''
else:
kw = {}
if not reuse:
hc_id = run_job(job_ini, log_level, log_file, exports, **kw)
if job is None:
logs.dbcmd('add_checksum', hc_id, haz_checksum)
elif not reuse_hazard or not os.path.exists(job.ds_calc_dir + '.hdf5'):
logs.dbcmd('update_job_checksum', hc_id, haz_checksum)
if ebr:
run_job(job_ini, log_level, log_file,
exports, hazard_calculation_id=hc_id)
else:
hc_id = job.id
logging.info('Reusing job #%d', job.id)
run_job(job_ini, log_level, log_file,
exports, hazard_calculation_id=hc_id)
def calc_abort(request, calc_id):
"""
Abort the given calculation, it is it running
"""
job = logs.dbcmd('get_job', calc_id)
if job is None:
message = {'error': 'Unknown job %s' % calc_id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
if job.status not in ('submitted', 'executing'):
message = {'error': 'Job %s is not running' % job.id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
if not utils.user_has_permission(request, job.user_name):
message = {'error': ('User %s has no permission to abort job %s' %
(job.user_name, job.id))}
return HttpResponse(content=json.dumps(message), content_type=JSON,
status=403)
if job.pid: # is a spawned job
try:
os.kill(job.pid, signal.SIGINT)
except Exception as exc:
logging.error(exc)
else:
logging.warning('Aborting job %d, pid=%d', job.id, job.pid)
logs.dbcmd('set_status', job.id, 'aborted')
message = {'success': 'Killing job %d' % job.id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
message = {'error': 'PID for job %s not found' % job.id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
def dbserver(cmd, dbhostport=None,
dbpath=os.path.expanduser(config.dbserver.file),
foreground=False):
"""
start/stop/restart the database server, or return its status
"""
if config.dbserver.multi_user and getpass.getuser() != 'openquake':
sys.exit('oq dbserver only works in single user mode')
status = dbs.get_status()
if cmd == 'status':
print('dbserver ' + status)
elif cmd == 'stop':
if status == 'running':
pid = logs.dbcmd('getpid')
os.kill(pid, signal.SIGINT) # this is trapped by the DbServer
else:
print('dbserver already stopped')
elif cmd == 'start':
if status == 'not-running':
dbs.run_server(dbpath, dbhostport)
else:
print('dbserver already running')
elif cmd == 'restart':
if status == 'running':
pid = logs.dbcmd('getpid')
os.kill(pid, signal.SIGINT)
dbs.run_server(dbpath, dbhostport)
def abort(job_id):
"""
Abort the given job
"""
job = logs.dbcmd('get_job', job_id) # job_id can be negative
if job is None:
print('There is no job %d' % job_id)
return
elif job.status not in ('executing', 'running'):
print('Job %d is %s' % (job.id, job.status))
return
name = 'oq-job-%d' % job.id
for p in psutil.process_iter():
if p.name() == name:
try:
os.kill(p.pid, signal.SIGINT)
logs.dbcmd('set_status', job.id, 'aborted')
print('Job %d aborted' % job.id)
except Exception as exc:
print(exc)
break
else: # no break
# set job as failed if it is set as 'executing' or 'running' in the db
# but the corresponding process is not running anymore
logs.dbcmd('set_status', job.id, 'failed')
print('Unable to find a process for job %d,'
' setting it as failed' % job.id)
def expose_outputs(dstore, owner=getpass.getuser(), status='complete'):
"""
Build a correspondence between the outputs in the datastore and the
ones in the database.
:param dstore: datastore
"""
oq = dstore['oqparam']
exportable = set(ekey[0] for ekey in export.export)
calcmode = oq.calculation_mode
dskeys = set(dstore) & exportable # exportable datastore keys
dskeys.add('fullreport')
rlzs = dstore['csm_info'].rlzs
if len(rlzs) > 1:
dskeys.add('realizations')
if len(dstore['csm_info/sg_data']) > 1: # export sourcegroups.csv
dskeys.add('sourcegroups')
hdf5 = dstore.hdf5
if 'hcurves-stats' in hdf5 or 'hcurves-rlzs' in hdf5:
if oq.hazard_stats() or oq.individual_curves or len(rlzs) == 1:
dskeys.add('hcurves')
if oq.uniform_hazard_spectra:
dskeys.add('uhs') # export them
if oq.hazard_maps:
dskeys.add('hmaps') # export them
if 'avg_losses-stats' in dstore or (
'avg_losses-rlzs' in dstore and len(rlzs)):
dskeys.add('avg_losses-stats')
if 'curves-rlzs' in dstore and len(rlzs) == 1:
dskeys.add('loss_curves-rlzs')
if 'curves-stats' in dstore and len(rlzs) > 1:
dskeys.add('loss_curves-stats')
if oq.conditional_loss_poes: # expose loss_maps outputs
if 'loss_curves-stats' in dstore:
dskeys.add('loss_maps-stats')
if 'all_loss_ratios' in dskeys:
dskeys.remove('all_loss_ratios') # export only specific IDs
if 'ruptures' in dskeys and 'scenario' in calcmode:
exportable.remove('ruptures') # do not export, as requested by Vitor
if 'rup_loss_table' in dskeys: # keep it hidden for the moment
dskeys.remove('rup_loss_table')
if 'hmaps' in dskeys and not oq.hazard_maps:
dskeys.remove('hmaps') # do not export the hazard maps
if logs.dbcmd('get_job', dstore.calc_id) is None:
# the calculation has not been imported in the db yet
logs.dbcmd('import_job', dstore.calc_id, oq.calculation_mode,
oq.description + ' [parent]', owner, status,
oq.hazard_calculation_id, dstore.datadir)
keysize = []
for key in sorted(dskeys & exportable):
try:
size_mb = dstore.get_attr(key, 'nbytes') / MB
except (KeyError, AttributeError):
size_mb = None
keysize.append((key, size_mb))
ds_size = os.path.getsize(dstore.filename) / MB
logs.dbcmd('create_outputs', dstore.calc_id, keysize, ds_size)
def set_concurrent_tasks_default(job_id):
"""
Set the default for concurrent_tasks based on the number of available
celery workers.
"""
stats = celery.task.control.inspect(timeout=1).stats()
if not stats:
logs.LOG.critical("No live compute nodes, aborting calculation")
logs.dbcmd('finish', job_id, 'failed')
sys.exit(1)
ncores = sum(stats[k]['pool']['max-concurrency'] for k in stats)
OqParam.concurrent_tasks.default = ncores * 3
logs.LOG.warn('Using %s, %d cores', ', '.join(sorted(stats)), ncores)
def importcalc(calc_id):
"""
Import a remote calculation into the local database. server, username
and password must be specified in an openquake.cfg file.
NB: calc_id can be a local pathname to a datastore not already
present in the database: in that case it is imported in the db.
"""
dbserver.ensure_on()
try:
calc_id = int(calc_id)
except ValueError: # assume calc_id is a pathname
calc_id, datadir = datastore.extract_calc_id_datadir(calc_id)
status = 'complete'
remote = False
else:
remote = True
job = logs.dbcmd('get_job', calc_id)
if job is not None:
sys.exit('There is already a job #%d in the local db' % calc_id)
if remote:
datadir = datastore.get_datadir()
webex = WebExtractor(calc_id)
status = webex.status['status']
hc_id = webex.oqparam.hazard_calculation_id
if hc_id:
sys.exit('The job has a parent (#%d) and cannot be '
'downloaded' % hc_id)
webex.dump('%s/calc_%d.hdf5' % (datadir, calc_id))
webex.close()
with datastore.read(calc_id) as dstore:
engine.expose_outputs(dstore, status=status)
logging.info('Imported calculation %d successfully', calc_id)
def run_job(job_ini, log_level='info', log_file=None, exports='',
username=getpass.getuser(), **kw):
"""
Run a job using the specified config file and other options.
:param str job_ini:
Path to calculation config (INI-style) files.
:param str log_level:
'debug', 'info', 'warn', 'error', or 'critical'
:param str log_file:
Path to log file.
:param exports:
A comma-separated string of export types requested by the user.
:param username:
Name of the user running the job
:param kw:
Extra parameters like hazard_calculation_id and calculation_mode
"""
job_id = logs.init('job', getattr(logging, log_level.upper()))
with logs.handle(job_id, log_level, log_file):
job_ini = os.path.abspath(job_ini)
oqparam = eng.job_from_file(job_ini, job_id, username, **kw)
kw['username'] = username
eng.run_calc(job_id, oqparam, exports, **kw)
for line in logs.dbcmd('list_outputs', job_id, False):
safeprint(line)
return job_id
def reset(yes):
"""
Remove all the datastores and the database of the current user
"""
ok = yes or confirm('Do you really want to destroy all your data? (y/n) ')
if not ok:
return
dbpath = os.path.realpath(os.path.expanduser(config.dbserver.file))
# user must be able to access and write the databse file to remove it
if os.path.isfile(dbpath) and os.access(dbpath, os.W_OK):
if dbserver.get_status() == 'running':
if config.dbserver.multi_user:
sys.exit('The oq dbserver must be stopped '
'before proceeding')
else:
pid = logs.dbcmd('getpid')
os.kill(pid, signal.SIGTERM)
time.sleep(.5) # give time to stop
assert dbserver.get_status() == 'not-running'
print('dbserver stopped')
try:
os.remove(dbpath)
print('Removed %s' % dbpath)
except OSError as exc:
print(exc, file=sys.stderr)
# fast way of removing everything
purge_all(fast=True) # datastore of the current user
def calc_datastore(request, job_id):
"""
Download a full datastore file.
:param request:
`django.http.HttpRequest` object.
:param job_id:
The id of the requested datastore
:returns:
A `django.http.HttpResponse` containing the content
of the requested artifact, if present, else throws a 404
"""
job = logs.dbcmd('get_job', int(job_id))
if job is None:
return HttpResponseNotFound()
if not utils.user_has_permission(request, job.user_name):
return HttpResponseForbidden()
fname = job.ds_calc_dir + '.hdf5'
response = FileResponse(
FileWrapper(open(fname, 'rb')), content_type=HDF5)
response['Content-Disposition'] = (
'attachment; filename=%s' % os.path.basename(fname))
response['Content-Length'] = str(os.path.getsize(fname))
return response
def make_report(isodate='today'):
"""
Build a HTML report with the computations performed at the given isodate.
Return the name of the report, which is saved in the current directory.
"""
if isodate == 'today':
isodate = date.today()
else:
isodate = date(*time.strptime(isodate, '%Y-%m-%d')[:3])
isodate1 = isodate + timedelta(1) # +1 day
tag_ids = []
tag_status = []
tag_contents = []
# the fetcher returns an header which is stripped with [1:]
jobs = dbcmd(
'fetch', ALL_JOBS, isodate.isoformat(), isodate1.isoformat())
page = '<h2>%d job(s) finished before midnight of %s</h2>' % (
len(jobs), isodate)
for job_id, user, status, ds_calc in jobs:
tag_ids.append(job_id)
tag_status.append(status)
[stats] = dbcmd('fetch', JOB_STATS, job_id)
(job_id, user, start_time, stop_time, status, duration) = stats
try:
ds = read(job_id, datadir=os.path.dirname(ds_calc))
txt = view_fullreport('fullreport', ds)
report = html_parts(txt)
except Exception as exc:
report = dict(
html_title='Could not generate report: %s' % cgi.escape(
str(exc), quote=True),
fragment='')
page = report['html_title']
page += html([stats._fields, stats])
page += report['fragment']
tag_contents.append(page)
page = make_tabs(tag_ids, tag_status, tag_contents) + (
'Report last updated: %s' % datetime.now())
fname = 'jobs-%s.html' % isodate
with open(fname, 'w') as f:
f.write(PAGE_TEMPLATE % page)
return fname
def calc_log_size(request, calc_id):
"""
Get the current number of lines in the log
"""
try:
response_data = logs.dbcmd('get_log_size', calc_id)
except dbapi.NotFound:
return HttpResponseNotFound()
return HttpResponse(content=json.dumps(response_data), content_type=JSON)
def calc_results(request, calc_id):
"""
Get a summarized list of calculation results for a given ``calc_id``.
Result is a JSON array of objects containing the following attributes:
* id
* name
* type (hazard_curve, hazard_map, etc.)
* url (the exact url where the full result can be accessed)
"""
# If the specified calculation doesn't exist OR is not yet complete,
# throw back a 404.
try:
info = logs.dbcmd('calc_info', calc_id)
if not utils.user_has_permission(request, info['user_name']):
return HttpResponseForbidden()
except dbapi.NotFound:
return HttpResponseNotFound()
base_url = _get_base_url(request)
# NB: export_output has as keys the list (output_type, extension)
# so this returns an ordered map output_type -> extensions such as
# {'agg_loss_curve': ['xml', 'csv'], ...}
output_types = groupby(export, lambda oe: oe[0],
lambda oes: [e for o, e in oes])
results = logs.dbcmd('get_outputs', calc_id)
if not results:
return HttpResponseNotFound()
response_data = []
for result in results:
try: # output from the datastore
rtype = result.ds_key
# Catalina asked to remove the .txt outputs (used for the GMFs)
outtypes = [ot for ot in output_types[rtype] if ot != 'txt']
except KeyError:
continue # non-exportable outputs should not be shown
url = urlparse.urljoin(base_url, 'v1/calc/result/%d' % result.id)
datum = dict(
id=result.id, name=result.display_name, type=rtype,
outtypes=outtypes, url=url, size_mb=result.size_mb)
response_data.append(datum)
return HttpResponse(content=json.dumps(response_data))
def calc_traceback(request, calc_id):
"""
Get the traceback as a list of lines for a given ``calc_id``.
"""
# If the specified calculation doesn't exist throw back a 404.
try:
response_data = logs.dbcmd('get_traceback', calc_id)
except dbapi.NotFound:
return HttpResponseNotFound()
return HttpResponse(content=json.dumps(response_data), content_type=JSON)
def submit_job(job_ini, username, hazard_job_id=None):
"""
Create a job object from the given job.ini file in the job directory
and run it in a new process. Returns the job ID and PID.
"""
job_id = logs.init('job')
oq = engine.job_from_file(
job_ini, job_id, username, hazard_calculation_id=hazard_job_id)
pik = pickle.dumps(oq, protocol=0) # human readable protocol
code = RUNCALC % dict(job_id=job_id, hazard_job_id=hazard_job_id, pik=pik,
username=username)
tmp_py = gettemp(code, suffix='.py')
# print(code, tmp_py) # useful when debugging
devnull = subprocess.DEVNULL
popen = subprocess.Popen([sys.executable, tmp_py],
stdin=devnull, stdout=devnull, stderr=devnull)
threading.Thread(target=popen.wait).start()
logs.dbcmd('update_job', job_id, {'pid': popen.pid})
return job_id, popen.pid
def export_outputs(job_id, target_dir, export_types):
# make it possible commands like `oq engine --eos -1 /tmp`
datadir, dskeys = logs.dbcmd('get_results', job_id)
if not dskeys:
yield('Found nothing to export for job %s' % job_id)
for dskey in dskeys:
yield('Exporting %s...' % dskey)
for line in export_output(
dskey, job_id, datadir, target_dir, export_types):
yield line
def purge_one(calc_id, user):
"""
Remove one calculation ID from the database and remove its datastore
"""
hdf5path = os.path.join(datastore.DATADIR, 'calc_%s.hdf5' % calc_id)
err = dbcmd('del_calc', calc_id, user)
if err:
print(err)
if os.path.exists(hdf5path): # not removed yet
os.remove(hdf5path)
print('Removed %s' % hdf5path)
def calc_log(request, calc_id, start, stop):
"""
Get a slice of the calculation log as a JSON list of rows
"""
start = start or 0
stop = stop or 0
try:
response_data = logs.dbcmd('get_log_slice', calc_id, start, stop)
except dbapi.NotFound:
return HttpResponseNotFound()
return HttpResponse(content=json.dumps(response_data), content_type=JSON)
def get_log_slice(request, calc_id, start, stop):
"""
Get a slice of the calculation log as a JSON list of rows
"""
start = start or 0
stop = stop or 0
try:
response_data = logs.dbcmd('get_log_slice', calc_id, start, stop)
except dbapi.NotFound:
return HttpResponseNotFound()
return HttpResponse(content=json.dumps(response_data), content_type=JSON)
def check_foreign():
"""
Check if we the DbServer is the right one
"""
if not config.dbserver.multi_user:
remote_server_path = logs.dbcmd('get_path')
if different_paths(server_path, remote_server_path):
return ('You are trying to contact a DbServer from another'
' instance (got %s, expected %s)\n'
'Check the configuration or stop the foreign'
' DbServer instance') % (remote_server_path, server_path)
def purge_one(calc_id, user):
"""
Remove one calculation ID from the database and remove its datastore
"""
filename = os.path.join(datadir, 'calc_%s.hdf5' % calc_id)
err = dbcmd('del_calc', calc_id, user)
if err:
print(err)
elif os.path.exists(filename): # not removed yet
os.remove(filename)
print('Removed %s' % filename)
def get_oqparam(request, job_id):
"""
Return the calculation parameters as a JSON
"""
try:
job = logs.dbcmd('get_job', int(job_id), getpass.getuser())
except dbapi.NotFound:
return HttpResponseNotFound()
with datastore.read(job.ds_calc_dir + '.hdf5') as ds:
oq = ds['oqparam']
return HttpResponse(content=json.dumps(vars(oq)), content_type=JSON)
def calc_info(request, calc_id):
"""
Get a JSON blob containing all of parameters for the given calculation
(specified by ``calc_id``). Also includes the current job status (
executing, complete, etc.).
"""
try:
info = logs.dbcmd('calc_info', calc_id)
except dbapi.NotFound:
return HttpResponseNotFound()
return HttpResponse(content=json.dumps(info), content_type=JSON)
def check_foreign():
"""
Check if we the DbServer is the right one
"""
if not config.dbserver.multi_user:
remote_server_path = logs.dbcmd('get_path')
if different_paths(server_path, remote_server_path):
return('You are trying to contact a DbServer from another'
' instance (got %s, expected %s)\n'
'Check the configuration or stop the foreign'
' DbServer instance') % (remote_server_path, server_path)
def classical_split_filter(srcs, srcfilter, gsims, params, monitor):
"""
Split the given sources, filter the subsources and the compute the
PoEs. Yield back subtasks if the split sources contain more than
maxweight ruptures.
"""
# first check if we are sampling the sources
ss = int(os.environ.get('OQ_SAMPLE_SOURCES', 0))
if ss:
splits, stime = split_sources(srcs)
srcs = random_filtered_sources(splits, srcfilter, ss)
yield classical(srcs, srcfilter, gsims, params, monitor)
return
# NB: splitting all the sources improves the distribution significantly,
# compared to splitting only the big sources
with monitor("splitting/filtering sources"):
splits, _stime = split_sources(srcs)
sources = list(srcfilter.filter(splits))
if not sources:
yield {'pmap': {}}
return
maxw = params['max_weight']
N = len(srcfilter.sitecol.complete)
def weight(src):
n = 10 * numpy.sqrt(len(src.indices) / N)
return src.weight * params['rescale_weight'] * n
blocks = list(block_splitter(sources, maxw, weight))
subtasks = len(blocks) - 1
for block in blocks[:-1]:
yield classical, block, srcfilter, gsims, params
if monitor.calc_id and subtasks:
msg = 'produced %d subtask(s) with mean weight %d' % (
subtasks, numpy.mean([b.weight for b in blocks[:-1]]))
try:
logs.dbcmd('log', monitor.calc_id, datetime.utcnow(), 'DEBUG',
'classical_split_filter#%d' % monitor.task_no, msg)
except Exception:
# a foreign key error in case of `oq run` is expected
print(msg)
yield classical(blocks[-1], srcfilter, gsims, params, monitor)
def del_calculation(job_id, confirmed=False):
"""
Delete a calculation and all associated outputs.
"""
if logs.dbcmd('get_job', job_id) is None:
print('There is no job %d' % job_id)
return
if confirmed or confirm(
'Are you sure you want to (abort and) delete this calculation and '
'all associated outputs?\nThis action cannot be undone. (y/n): '):
try:
abort(job_id)
resp = logs.dbcmd('del_calc', job_id, getpass.getuser())
except RuntimeError as err:
safeprint(err)
else:
if 'success' in resp:
print('Removed %d' % job_id)
else:
print(resp['error'])
def set_concurrent_tasks_default(calc):
"""
Look at the number of available workers and update the parameter
OqParam.concurrent_tasks.default. Abort the calculations if no
workers are available. Do nothing for trivial distributions.
"""
if OQ_DISTRIBUTE in 'no processpool': # do nothing
num_workers = 0 if OQ_DISTRIBUTE == 'no' else parallel.CT // 2
logging.warning('Using %d cores on %s', num_workers, platform.node())
return
num_workers = sum(total
for host, running, total in parallel.workers_wait())
if num_workers == 0:
logging.critical("No live compute nodes, aborting calculation")
logs.dbcmd('finish', calc.datastore.calc_id, 'failed')
sys.exit(1)
parallel.CT = num_workers * 2
OqParam.concurrent_tasks.default = num_workers * 2
logging.warning('Using %d %s workers', num_workers, OQ_DISTRIBUTE)
def get_oqparam(request, job_id):
"""
Return the calculation parameters as a JSON
"""
user = utils.get_user_data(request)
username = user['name'] if user['acl_on'] else None
job = logs.dbcmd('get_job', int(job_id), username)
if job is None:
return HttpResponseNotFound()
with datastore.read(job.ds_calc_dir + '.hdf5') as ds:
oq = ds['oqparam']
return HttpResponse(content=json.dumps(vars(oq)), content_type=JSON)
def del_calculation(job_id, confirmed=False):
"""
Delete a calculation and all associated outputs.
"""
if logs.dbcmd('get_job', job_id) is None:
print('There is no job %d' % job_id)
return
if confirmed or confirm(
'Are you sure you want to (abort and) delete this calculation and '
'all associated outputs?\nThis action cannot be undone. (y/n): '):
try:
abort.func(job_id)
resp = logs.dbcmd('del_calc', job_id, getpass.getuser())
except RuntimeError as err:
safeprint(err)
else:
if 'success' in resp:
print('Removed %d' % job_id)
else:
print(resp['error'])
def poll_queue(job_id, poll_time):
"""
Check the queue of executing/submitted jobs and exit when there is
a free slot.
"""
offset = config.distribution.serialize_jobs - 1
if offset >= 0:
first_time = True
while True:
running = logs.dbcmd(GET_JOBS)
previous = [job for job in running if job.id < job_id - offset]
if previous:
if first_time:
logs.dbcmd('update_job', job_id,
{'status': 'submitted', 'pid': _PID})
first_time = False
# the logging is not yet initialized, so use a print
print('Waiting for jobs %s' % [p.id for p in previous])
time.sleep(poll_time)
else:
break
def main(calc_id: int, aggregate_by):
"""
Re-run the postprocessing after an event based risk calculation
"""
parent = util.read(calc_id)
oqp = parent['oqparam']
aggby = aggregate_by.split(',')
for tagname in aggby:
if tagname not in oqp.aggregate_by:
raise ValueError('%r not in %s' % (tagname, oqp.aggregate_by))
job_id = logs.init('job', level=logging.INFO)
dic = dict(
calculation_mode='reaggregate',
description=oqp.description + '[aggregate_by=%s]' % aggregate_by,
user_name=getpass.getuser(), is_running=1, status='executing',
pid=os.getpid(), hazard_calculation_id=job_id)
logs.dbcmd('update_job', job_id, dic)
if os.environ.get('OQ_DISTRIBUTE') not in ('no', 'processpool'):
os.environ['OQ_DISTRIBUTE'] = 'processpool'
with logs.handle(job_id, logging.INFO):
oqp.hazard_calculation_id = parent.calc_id
parallel.Starmap.init()
prc = PostRiskCalculator(oqp, job_id)
try:
prc.run(aggregate_by=aggby)
engine.expose_outputs(prc.datastore)
logs.dbcmd('finish', job_id, 'complete')
except Exception:
logs.dbcmd('finish', job_id, 'failed')
finally:
parallel.Starmap.shutdown()
def submit_job(request_files, ini, username, hc_id):
"""
Create a job object from the given files and run it in a new process.
:returns: a job ID
"""
# build a LogContext object associated to a database job
[job] = engine.create_jobs([
dict(calculation_mode='preclassical',
description='Calculation waiting to start')
], config.distribution.log_level, None, username, hc_id)
# store the request files and perform some validation
try:
job_ini = store(request_files, ini, job.calc_id)
job.oqparam = oq = readinput.get_oqparam(
job_ini, kw={'hazard_calculation_id': hc_id})
if oq.sensitivity_analysis:
logs.dbcmd('set_status', job.calc_id, 'deleted') # hide it
jobs = engine.create_jobs([job_ini], config.distribution.log_level,
None, username, hc_id, True)
else:
dic = dict(calculation_mode=oq.calculation_mode,
description=oq.description,
hazard_calculation_id=hc_id)
logs.dbcmd('update_job', job.calc_id, dic)
jobs = [job]
except Exception:
tb = traceback.format_exc()
logs.dbcmd('log', job.calc_id, datetime.utcnow(), 'CRITICAL',
'before starting', tb)
logs.dbcmd('finish', job.calc_id, 'failed')
raise
custom_tmp = os.path.dirname(job_ini)
submit_cmd = config.distribution.submit_cmd.split()
big_job = oq.get_input_size() > int(config.distribution.min_input_size)
if submit_cmd == ENGINE: # used for debugging
for job in jobs:
subprocess.Popen(submit_cmd + [save(job, custom_tmp)])
elif submit_cmd == KUBECTL and big_job:
for job in jobs:
with open(os.path.join(CWD, 'job.yaml')) as f:
yaml = string.Template(f.read()).substitute(
CALC_PIK=save(job, custom_tmp),
CALC_NAME='calc%d' % job.calc_id)
subprocess.run(submit_cmd, input=yaml.encode('ascii'))
else:
Process(target=engine.run_jobs, args=(jobs, )).start()
return job.calc_id
def ebrisk(rupgetter, srcfilter, param, monitor):
"""
:param rupgetter: RuptureGetter with multiple ruptures
:param srcfilter: a SourceFilter
:param param: dictionary of parameters coming from oqparam
:param monitor: a Monitor instance
:returns: a dictionary with keys elt, alt, ...
"""
mon_rup = monitor('getting ruptures', measuremem=False)
mon_haz = monitor('getting hazard', measuremem=False)
gmfs = []
gmf_info = []
gg = getters.GmfGetter(rupgetter, srcfilter, param['oqparam'],
param['amplifier'])
nbytes = 0
for c in gg.gen_computers(mon_rup):
with mon_haz:
data, time_by_rup = c.compute_all(gg.min_iml, gg.rlzs_by_gsim)
if len(data):
gmfs.append(data)
nbytes += data.nbytes
gmf_info.append((c.ebrupture.id, mon_haz.task_no, len(c.sids),
data.nbytes, mon_haz.dt))
if nbytes > param['ebrisk_maxsize']:
msg = 'produced subtask'
try:
logs.dbcmd('log', monitor.calc_id, datetime.utcnow(), 'DEBUG',
'ebrisk#%d' % monitor.task_no, msg)
except Exception: # for `oq run`
print(msg)
yield calc_risk, numpy.concatenate(gmfs), param
nbytes = 0
gmfs = []
res = {}
if gmfs:
res.update(calc_risk(numpy.concatenate(gmfs), param, monitor))
if gmf_info:
res['gmf_info'] = numpy.array(gmf_info, gmf_info_dt)
yield res
def calc(request, calc_id):
"""
Get a JSON blob containing all of parameters for the given calculation
(specified by ``calc_id``). Also includes the current job status (
executing, complete, etc.).
"""
try:
info = logs.dbcmd('calc_info', calc_id)
if not utils.user_has_permission(request, info['user_name']):
return HttpResponseForbidden()
except dbapi.NotFound:
return HttpResponseNotFound()
return HttpResponse(content=json.dumps(info), content_type=JSON)
def smart_run(job_ini, oqparam, log_level, log_file, exports,
reuse_hazard, **params):
"""
Run calculations by storing their hazard checksum and reusing previous
calculations if requested.
"""
haz_checksum = readinput.get_checksum32(oqparam, hazard=True)
# retrieve an old calculation with the right checksum, if any
job = logs.dbcmd('get_job_from_checksum', haz_checksum)
reuse = reuse_hazard and job and os.path.exists(job.ds_calc_dir + '.hdf5')
# recompute the hazard and store the checksum
if not reuse:
hc_id = run_job(job_ini, log_level, log_file, exports, **params)
if job is None:
logs.dbcmd('add_checksum', hc_id, haz_checksum)
elif not reuse_hazard or not os.path.exists(job.ds_calc_dir + '.hdf5'):
logs.dbcmd('update_job_checksum', hc_id, haz_checksum)
else:
hc_id = job.id
logging.info('Reusing job #%d', job.id)
run_job(job_ini, log_level, log_file,
exports, hazard_calculation_id=hc_id, **params)
def abort(job_id):
"""
Abort the given job
"""
job = logs.dbcmd('get_job', job_id) # job_id can be negative
if job is None:
print('There is no job %d' % job_id)
return
elif job.status not in ('executing', 'running'):
print('Job %d is %s' % (job.id, job.status))
return
name = 'oq-job-%d' % job.id
for p in psutil.process_iter():
if p.name() == name:
try:
os.kill(p.pid, signal.SIGTERM)
logs.dbcmd('set_status', job.id, 'aborted')
except Exception as exc:
print(exc)
break
else: # no break
print('%d aborted' % job.id)
def importcalc(host, calc_id, username, password):
"""
Import a remote calculation into the local database
"""
logging.basicConfig(level=logging.INFO)
if '/' in host.split('//', 1)[1]:
sys.exit('Wrong host ending with /%s' % host.rsplit('/', 1)[1])
calc_url = '/'.join([host, 'v1/calc', str(calc_id)])
dbserver.ensure_on()
job = logs.dbcmd('get_job', calc_id)
if job is not None:
sys.exit('There is already a job #%d in the local db' % calc_id)
datadir = datastore.get_datadir()
session = login(host, username, password)
status = session.get('%s/status' % calc_url)
if 'Log in to an existing account' in status.text:
sys.exit('Could not login')
json = status.json()
if json["parent_id"]:
sys.exit('The job has a parent (#%(parent_id)d) and cannot be '
'downloaded' % json)
resp = session.get('%s/datastore' % calc_url, stream=True)
assert resp.status_code == 200, resp.status_code
fname = '%s/calc_%d.hdf5' % (datadir, calc_id)
down = 0
with open(fname, 'wb') as f:
logging.info('%s -> %s', calc_url, fname)
for chunk in resp.iter_content(CHUNKSIZE):
f.write(chunk)
down += len(chunk)
general.println('Downloaded {:,} bytes'.format(down))
print()
logs.dbcmd('import_job', calc_id, json['calculation_mode'],
json['description'], json['owner'], json['status'],
json['parent_id'], datadir)
with datastore.read(calc_id) as dstore:
engine.expose_outputs(dstore)
logging.info('Imported calculation %d successfully', calc_id)
def calc_oqparam(request, job_id):
"""
Return the calculation parameters as a JSON
"""
job = logs.dbcmd('get_job', int(job_id))
if job is None:
return HttpResponseNotFound()
if not utils.user_has_permission(request, job.user_name):
return HttpResponseForbidden()
with datastore.read(job.ds_calc_dir + '.hdf5') as ds:
oq = ds['oqparam']
return HttpResponse(content=json.dumps(vars(oq)), content_type=JSON)
def calc_oqparam(request, job_id):
"""
Return the calculation parameters as a JSON
"""
job = logs.dbcmd('get_job', int(job_id))
if job is None:
return HttpResponseNotFound()
if not utils.user_has_permission(request, job.user_name):
return HttpResponseForbidden()
with datastore.read(job.ds_calc_dir + '.hdf5') as ds:
oq = ds['oqparam']
return HttpResponse(content=json.dumps(vars(oq)), content_type=JSON)
def classical_split_filter(srcs, srcfilter, gsims, params, monitor):
"""
Split the given sources, filter the subsources and the compute the
PoEs. Yield back subtasks if the split sources contain more than
maxweight ruptures.
"""
# first check if we are sampling the sources
ss = int(os.environ.get('OQ_SAMPLE_SOURCES', 0))
if ss:
splits, stime = split_sources(srcs)
srcs = random_filtered_sources(splits, srcfilter, ss)
yield classical(srcs, srcfilter, gsims, params, monitor)
return
# NB: splitting all the sources improves the distribution significantly,
# compared to splitting only the big sources
with monitor("splitting/filtering sources"):
splits, _stime = split_sources(srcs)
sources = list(srcfilter.filter(splits))
if sources:
maxw = sum(src.weight for src in sources) / 5
if maxw < 1000:
maxw = 1000
elif maxw > params['max_weight']:
maxw = params['max_weight']
blocks = list(block_splitter(sources, maxw, weight))
nb = len(blocks)
msg = 'produced %d subtask(s) with max weight=%d' % (
nb - 1, max(b.weight for b in blocks))
if monitor.calc_id and nb > 1:
try:
logs.dbcmd('log', monitor.calc_id, datetime.utcnow(), 'DEBUG',
'classical_split_filter#%d' % monitor.task_no, msg)
except Exception:
# a foreign key error in case of `oq run` is expected
print(msg)
for block in blocks[:-1]:
yield classical, block, srcfilter, gsims, params
yield classical(blocks[-1], srcfilter, gsims, params, monitor)
def calc_abort(request, calc_id):
"""
Abort the given calculation, it is it running
"""
job = logs.dbcmd('get_job', calc_id)
if job is None:
message = {'error': 'Unknown job %s' % calc_id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
if job.status not in ('executing', 'running'):
message = {'error': 'Job %s is not running' % job.id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
user = utils.get_user_data(request)
info = logs.dbcmd('calc_info', calc_id)
allowed_users = user['group_members'] or [user['name']]
if user['acl_on'] and info['user_name'] not in allowed_users:
message = {
'error': ('User %s has no permission to abort job %s' %
(info['user_name'], job.id))
}
return HttpResponse(content=json.dumps(message),
content_type=JSON,
status=403)
if job.pid: # is a spawned job
try:
os.kill(job.pid, signal.SIGTERM)
except Exception as exc:
logging.error(exc)
else:
logging.warn('Aborting job %d, pid=%d', job.id, job.pid)
logs.dbcmd('set_status', job.id, 'aborted')
message = {'success': 'Killing job %d' % job.id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
message = {'error': 'PID for job %s not found' % job.id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
def job_from_file(job_ini, job_id, username, **kw):
"""
Create a full job profile from a job config file.
:param job_ini:
Path to a job.ini file
:param job_id:
ID of the created job
:param username:
The user who will own this job profile and all results
:param kw:
Extra parameters including `calculation_mode` and `exposure_file`
:returns:
an oqparam instance
"""
hc_id = kw.get('hazard_calculation_id')
try:
oq = readinput.get_oqparam(job_ini, hc_id=hc_id)
except Exception:
logs.dbcmd('finish', job_id, 'failed')
raise
if 'calculation_mode' in kw:
oq.calculation_mode = kw.pop('calculation_mode')
if 'description' in kw:
oq.description = kw.pop('description')
if 'exposure_file' in kw: # hack used in commands.engine
fnames = kw.pop('exposure_file').split()
if fnames:
oq.inputs['exposure'] = fnames
elif 'exposure' in oq.inputs:
del oq.inputs['exposure']
logs.dbcmd(
'update_job', job_id,
dict(calculation_mode=oq.calculation_mode,
description=oq.description,
user_name=username,
hazard_calculation_id=hc_id))
return oq
def submit_job(job_ini, username, hazard_job_id=None):
"""
Create a job object from the given job.ini file in the job directory
and run it in a new process. Returns the job ID and PID.
"""
job_id = logs.init('job')
oq = engine.job_from_file(job_ini,
job_id,
username,
hazard_calculation_id=hazard_job_id)
pik = pickle.dumps(oq, protocol=0) # human readable protocol
code = RUNCALC % dict(
job_id=job_id, hazard_job_id=hazard_job_id, pik=pik, username=username)
tmp_py = gettemp(code, suffix='.py')
# print(code, tmp_py) # useful when debugging
devnull = subprocess.DEVNULL
popen = subprocess.Popen([sys.executable, tmp_py],
stdin=devnull,
stdout=devnull,
stderr=devnull)
threading.Thread(target=popen.wait).start()
logs.dbcmd('update_job', job_id, {'pid': popen.pid})
return job_id, popen.pid
def calc_abort(request, calc_id):
"""
Abort the given calculation, it is it running
"""
job = logs.dbcmd('get_job', calc_id)
if job is None:
message = {'error': 'Unknown job %s' % calc_id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
if job.status not in ('submitted', 'executing'):
message = {'error': 'Job %s is not running' % job.id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
# only the owner or superusers can abort a calculation
if (job.user_name not in utils.get_valid_users(request)
and not utils.is_superuser(request)):
message = {
'error': ('User %s has no permission to abort job %s' %
(request.user, job.id))
}
return HttpResponse(content=json.dumps(message),
content_type=JSON,
status=403)
if job.pid: # is a spawned job
try:
os.kill(job.pid, signal.SIGINT)
except Exception as exc:
logging.error(exc)
else:
logging.warning('Aborting job %d, pid=%d', job.id, job.pid)
logs.dbcmd('set_status', job.id, 'aborted')
message = {'success': 'Killing job %d' % job.id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
message = {'error': 'PID for job %s not found' % job.id}
return HttpResponse(content=json.dumps(message), content_type=JSON)
def job_from_file(job_ini, job_id, username, **kw):
"""
Create a full job profile from a job config file.
:param job_ini:
Path to a job.ini file
:param job_id:
ID of the created job
:param username:
The user who will own this job profile and all results
:param kw:
Extra parameters including `calculation_mode` and `exposure_file`
:returns:
an oqparam instance
"""
hc_id = kw.get('hazard_calculation_id')
try:
oq = readinput.get_oqparam(job_ini, hc_id=hc_id)
except Exception:
logs.dbcmd('finish', job_id, 'failed')
raise
if 'calculation_mode' in kw:
oq.calculation_mode = kw.pop('calculation_mode')
if 'description' in kw:
oq.description = kw.pop('description')
if 'exposure_file' in kw: # hack used in commands.engine
fnames = kw.pop('exposure_file').split()
if fnames:
oq.inputs['exposure'] = fnames
elif 'exposure' in oq.inputs:
del oq.inputs['exposure']
logs.dbcmd('update_job', job_id,
dict(calculation_mode=oq.calculation_mode,
description=oq.description,
user_name=username,
hazard_calculation_id=hc_id))
return oq
def extract(request, calc_id, what):
"""
Wrapper over the `oq extract` command. If `setting.LOCKDOWN` is true
only calculations owned by the current user can be retrieved.
"""
job = logs.dbcmd('get_job', int(calc_id))
if job is None:
return HttpResponseNotFound()
if not utils.user_has_permission(request, job.user_name):
return HttpResponseForbidden()
try:
# read the data and save them on a temporary .npz file
with datastore.read(job.ds_calc_dir + '.hdf5') as ds:
fd, fname = tempfile.mkstemp(prefix=what.replace('/', '-'),
suffix='.npz')
os.close(fd)
n = len(request.path_info)
query_string = unquote_plus(request.get_full_path()[n:])
aw = _extract(ds, what + query_string)
a = {}
for key, val in vars(aw).items():
key = str(key) # can be a numpy.bytes_
if key.startswith('_'):
continue
elif isinstance(val, str):
# without this oq extract would fail
a[key] = numpy.array(val.encode('utf-8'))
elif isinstance(val, dict):
# this is hack: we are losing the values
a[key] = list(val)
else:
a[key] = val
numpy.savez_compressed(fname, **a)
except Exception as exc:
tb = ''.join(traceback.format_tb(exc.__traceback__))
return HttpResponse(content='%s: %s\n%s' %
(exc.__class__.__name__, exc, tb),
content_type='text/plain',
status=500)
# stream the data back
stream = FileWrapper(open(fname, 'rb'))
stream.close = lambda: (FileWrapper.close(stream), os.remove(fname))
response = FileResponse(stream, content_type='application/octet-stream')
response['Content-Disposition'] = ('attachment; filename=%s' %
os.path.basename(fname))
response['Content-Length'] = str(os.path.getsize(fname))
return response
def main(cmd, args=()):
"""
Run a database command
"""
if cmd in commands and len(args) != len(commands[cmd]):
sys.exit('Wrong number of arguments, expected %s, got %s' %
(commands[cmd], args))
elif (cmd not in commands and not cmd.upper().startswith('SELECT')
and config.dbserver.multi_user and getpass.getuser() != 'openquake'):
sys.exit('You have no permission to run %s' % cmd)
dbserver.ensure_on()
res = logs.dbcmd(cmd, *convert(args))
if hasattr(res, '_fields') and res.__class__.__name__ != 'Row':
print(rst_table(res))
else:
print(res)
def read(calc_id, username=None):
"""
:param calc_id: a calculation ID
:param username: if given, restrict the search to the user's calculations
:returns: the associated DataStore instance
"""
if isinstance(calc_id, str) or calc_id < 0 and not username:
# get the last calculation in the datastore of the current user
return datastore.read(calc_id)
job = logs.dbcmd('get_job', calc_id, username)
if job:
return datastore.read(job.ds_calc_dir + '.hdf5')
else:
# calc_id can be present in the datastore and not in the database:
# this happens if the calculation was run with `oq run`
return datastore.read(calc_id)
def read(calc_id, username=None):
"""
:param calc_id: a calculation ID
:param username: if given, restrict the search to the user's calculations
:returns: the associated DataStore instance
"""
if isinstance(calc_id, str) or calc_id < 0 and not username:
# get the last calculation in the datastore of the current user
return datastore.read(calc_id)
job = logs.dbcmd('get_job', calc_id, username)
if job:
return datastore.read(job.ds_calc_dir + '.hdf5')
else:
# calc_id can be present in the datastore and not in the database:
# this happens if the calculation was run with `oq run`
return datastore.read(calc_id)
def extract(request, calc_id, what):
"""
Wrapper over the `oq extract` command. If `setting.LOCKDOWN` is true
only calculations owned by the current user can be retrieved.
"""
job = logs.dbcmd('get_job', int(calc_id))
if job is None:
return HttpResponseNotFound()
if not utils.user_has_permission(request, job.user_name):
return HttpResponseForbidden()
try:
# read the data and save them on a temporary .npz file
with datastore.read(job.ds_calc_dir + '.hdf5') as ds:
fd, fname = tempfile.mkstemp(
prefix=what.replace('/', '-'), suffix='.npz')
os.close(fd)
n = len(request.path_info)
query_string = unquote_plus(request.get_full_path()[n:])
aw = _extract(ds, what + query_string)
a = {}
for key, val in vars(aw).items():
key = str(key) # can be a numpy.bytes_
if isinstance(val, str):
# without this oq extract would fail
a[key] = numpy.array(val.encode('utf-8'))
elif isinstance(val, dict):
# this is hack: we are losing the values
a[key] = list(val)
else:
a[key] = val
numpy.savez_compressed(fname, **a)
except Exception as exc:
tb = ''.join(traceback.format_tb(exc.__traceback__))
return HttpResponse(
content='%s: %s\n%s' % (exc.__class__.__name__, exc, tb),
content_type='text/plain', status=500)
# stream the data back
stream = FileWrapper(open(fname, 'rb'))
stream.close = lambda: (FileWrapper.close(stream), os.remove(fname))
response = FileResponse(stream, content_type='application/octet-stream')
response['Content-Disposition'] = (
'attachment; filename=%s' % os.path.basename(fname))
response['Content-Length'] = str(os.path.getsize(fname))
return response
def calc_list(request, id=None):
# view associated to the endpoints /v1/calc/list and /v1/calc/:id/status
"""
Get a list of calculations and report their id, status, calculation_mode,
is_running, description, and a url where more detailed information
can be accessed. This is called several times by the Javascript.
Responses are in JSON.
"""
base_url = _get_base_url(request)
# always filter calculation list unless user is a superuser
calc_data = logs.dbcmd('get_calcs', request.GET,
utils.get_valid_users(request),
not utils.is_superuser(request), id)
response_data = []
username = psutil.Process(os.getpid()).username()
for (hc_id, owner, status, calculation_mode, is_running, desc, pid,
parent_id, size_mb) in calc_data:
url = urlparse.urljoin(base_url, 'v1/calc/%d' % hc_id)
abortable = False
if is_running:
try:
if psutil.Process(pid).username() == username:
abortable = True
except psutil.NoSuchProcess:
pass
response_data.append(
dict(id=hc_id,
owner=owner,
calculation_mode=calculation_mode,
status=status,
is_running=bool(is_running),
description=desc,
url=url,
parent_id=parent_id,
abortable=abortable,
size_mb=size_mb))
# if id is specified the related dictionary is returned instead the list
if id is not None:
if not response_data:
return HttpResponseNotFound()
[response_data] = response_data
return HttpResponse(content=json.dumps(response_data), content_type=JSON)
def calc_remove(request, calc_id):
"""
Remove the calculation id
"""
user = utils.get_user_data(request)['name']
try:
message = logs.dbcmd('del_calc', calc_id, user)
except dbapi.NotFound:
return HttpResponseNotFound()
if isinstance(message, list): # list of removed files
return HttpResponse(content=json.dumps(message),
content_type=JSON,
status=200)
else: # FIXME: the error is not passed properly to the javascript
logging.error(message)
return HttpResponse(content=message,
content_type='text/plain',
status=500)
def run_calc(log):
"""
Run a calculation.
:param log:
LogContext of the current job
"""
register_signals()
setproctitle('oq-job-%d' % log.calc_id)
with log:
oqparam = log.get_oqparam()
calc = base.calculators(oqparam, log.calc_id)
logging.info('%s running %s [--hc=%s]',
getpass.getuser(),
calc.oqparam.inputs['job_ini'],
calc.oqparam.hazard_calculation_id)
logging.info('Using engine version %s', __version__)
msg = check_obsolete_version(oqparam.calculation_mode)
# NB: disabling the warning should be done only for users with
# an updated LTS version, but we are doing it for all users
# if msg:
# logging.warning(msg)
calc.from_engine = True
if config.zworkers['host_cores']:
set_concurrent_tasks_default(calc)
else:
logging.warning('Assuming %d %s workers',
parallel.Starmap.num_cores, OQ_DISTRIBUTE)
t0 = time.time()
calc.run()
logging.info('Exposing the outputs to the database')
expose_outputs(calc.datastore)
path = calc.datastore.filename
size = general.humansize(getsize(path))
logging.info('Stored %s on %s in %d seconds',
size, path, time.time() - t0)
calc.datastore.close()
for line in logs.dbcmd('list_outputs', log.calc_id, False):
general.safeprint(line)
# sanity check to make sure that the logging on file is working
if (log.log_file and log.log_file != os.devnull and
getsize(log.log_file) == 0):
logging.warning('The log file %s is empty!?' % log.log_file)
return calc
def extract(request, calc_id, what):
"""
Wrapper over the `oq extract` command. If setting.LOCKDOWN is true
only calculations owned by the current user can be retrieved.
"""
job = logs.dbcmd('get_job', int(calc_id))
if job is None:
return HttpResponseNotFound()
if not utils.user_has_permission(request, job.user_name):
return HttpResponseForbidden()
# read the data and save them on a temporary .pik file
with datastore.read(job.ds_calc_dir + '.hdf5') as ds:
fd, fname = tempfile.mkstemp(
prefix=what.replace('/', '-'), suffix='.npz')
os.close(fd)
n = len(request.path_info)
query_string = unquote_plus(request.get_full_path()[n:])
obj = _extract(ds, what + query_string)
if inspect.isgenerator(obj):
array, attrs = 0, {k: _array(v) for k, v in obj}
elif hasattr(obj, '__toh5__'):
array, attrs = obj.__toh5__()
else: # assume obj is an array
array, attrs = obj, {}
a = {}
for key, val in attrs.items():
if isinstance(key, bytes):
key = key.decode('utf-8')
if isinstance(val, str):
# without this oq extract would fail
a[key] = numpy.array(val.encode('utf-8'))
else:
a[key] = val
numpy.savez_compressed(fname, array=array, **a)
# stream the data back
stream = FileWrapper(open(fname, 'rb'))
stream.close = lambda: (FileWrapper.close(stream), os.remove(fname))
response = FileResponse(stream, content_type='application/octet-stream')
response['Content-Disposition'] = (
'attachment; filename=%s' % os.path.basename(fname))
response['Content-Length'] = str(os.path.getsize(fname))
return response
def db(cmd, args=()):
"""
Run a database command
"""
if cmd not in commands:
okcmds = '\n'.join(
'%s %s' % (name, repr(' '.join(args)) if args else '')
for name, args in sorted(commands.items()))
print('Invalid command "%s": choose one from\n%s' % (cmd, okcmds))
elif len(args) != len(commands[cmd]):
print('Wrong number of arguments, expected %s, got %s' % (
commands[cmd], args))
else:
dbserver.ensure_on()
res = logs.dbcmd(cmd, *convert(args))
if hasattr(res, '_fields') and res.__class__.__name__ != 'Row':
print(rst_table(res))
else:
print(res)
def calc_list(request, id=None):
# view associated to the endpoints /v1/calc/list and /v1/calc/:id/status
"""
Get a list of calculations and report their id, status, calculation_mode,
is_running, description, and a url where more detailed information
can be accessed. This is called several times by the Javascript.
Responses are in JSON.
"""
base_url = _get_base_url(request)
calc_data = logs.dbcmd('get_calcs', request.GET,
utils.get_valid_users(request),
utils.get_acl_on(request), id)
response_data = []
username = psutil.Process(os.getpid()).username()
for (hc_id, owner, status, calculation_mode, is_running, desc, pid,
parent_id, size_mb) in calc_data:
url = urlparse.urljoin(base_url, 'v1/calc/%d' % hc_id)
abortable = False
if is_running:
try:
if psutil.Process(pid).username() == username:
abortable = True
except psutil.NoSuchProcess:
pass
response_data.append(
dict(id=hc_id, owner=owner,
calculation_mode=calculation_mode, status=status,
is_running=bool(is_running), description=desc, url=url,
parent_id=parent_id, abortable=abortable, size_mb=size_mb))
# if id is specified the related dictionary is returned instead the list
if id is not None:
[response_data] = response_data
return HttpResponse(content=json.dumps(response_data),
content_type=JSON)
def calc_remove(request, calc_id):
"""
Remove the calculation id
"""
# Only the owner can remove a job
user = utils.get_user(request)
try:
message = logs.dbcmd('del_calc', calc_id, user)
except dbapi.NotFound:
return HttpResponseNotFound()
if 'success' in message:
return HttpResponse(content=json.dumps(message),
content_type=JSON, status=200)
elif 'error' in message:
logging.error(message['error'])
return HttpResponse(content=json.dumps(message),
content_type=JSON, status=403)
else:
# This is an untrapped server error
logging.error(message)
return HttpResponse(content=message,
content_type='text/plain', status=500)
def poll_queue(job_id, pid, poll_time):
"""
Check the queue of executing/submitted jobs and exit when there is
a free slot.
"""
if config.distribution.serialize_jobs:
first_time = True
while True:
jobs = logs.dbcmd(GET_JOBS)
failed = [job.id for job in jobs if not psutil.pid_exists(job.pid)]
if failed:
logs.dbcmd("UPDATE job SET status='failed', is_running=0 "
"WHERE id in (?X)", failed)
elif any(job.id < job_id for job in jobs):
if first_time:
logs.LOG.warn('Waiting for jobs %s', [j.id for j in jobs])
logs.dbcmd('update_job', job_id,
{'status': 'submitted', 'pid': pid})
first_time = False
time.sleep(poll_time)
else:
break
logs.dbcmd('update_job', job_id, {'status': 'executing', 'pid': _PID})
def web_engine_get_outputs(request, calc_id, **kwargs):
job = logs.dbcmd('get_job', calc_id)
size_mb = '?' if job.size_mb is None else '%.2f' % job.size_mb
return render(request, "engine/get_outputs.html",
dict(calc_id=calc_id, size_mb=size_mb))
