def smart_run(job_ini, oqparam, log_level, log_file, exports, reuse_hazard): """ Run calculations by storing their hazard checksum and reusing previous calculations if requested. """ haz_checksum = readinput.get_checksum32(oqparam, hazard=True) # retrieve an old calculation with the right checksum, if any job = logs.dbcmd('get_job_from_checksum', haz_checksum) reuse = reuse_hazard and job and os.path.exists(job.ds_calc_dir + '.hdf5') # recompute the hazard and store the checksum ebr = (oqparam.calculation_mode == 'event_based_risk' and 'gmfs' not in oqparam.inputs) if ebr: kw = dict(calculation_mode='event_based') if (oqparam.sites or 'sites' in oqparam.inputs or 'site_model' in oqparam.inputs): # remove exposure from the hazard kw['exposure_file'] = '' else: kw = {} if not reuse: hc_id = run_job(job_ini, log_level, log_file, exports, **kw) if job is None: logs.dbcmd('add_checksum', hc_id, haz_checksum) elif not reuse_hazard or not os.path.exists(job.ds_calc_dir + '.hdf5'): logs.dbcmd('update_job_checksum', hc_id, haz_checksum) if ebr: run_job(job_ini, log_level, log_file, exports, hazard_calculation_id=hc_id) else: hc_id = job.id logging.info('Reusing job #%d', job.id) run_job(job_ini, log_level, log_file, exports, hazard_calculation_id=hc_id)
def calc_abort(request, calc_id): """ Abort the given calculation, it is it running """ job = logs.dbcmd('get_job', calc_id) if job is None: message = {'error': 'Unknown job %s' % calc_id} return HttpResponse(content=json.dumps(message), content_type=JSON) if job.status not in ('submitted', 'executing'): message = {'error': 'Job %s is not running' % job.id} return HttpResponse(content=json.dumps(message), content_type=JSON) if not utils.user_has_permission(request, job.user_name): message = {'error': ('User %s has no permission to abort job %s' % (job.user_name, job.id))} return HttpResponse(content=json.dumps(message), content_type=JSON, status=403) if job.pid: # is a spawned job try: os.kill(job.pid, signal.SIGINT) except Exception as exc: logging.error(exc) else: logging.warning('Aborting job %d, pid=%d', job.id, job.pid) logs.dbcmd('set_status', job.id, 'aborted') message = {'success': 'Killing job %d' % job.id} return HttpResponse(content=json.dumps(message), content_type=JSON) message = {'error': 'PID for job %s not found' % job.id} return HttpResponse(content=json.dumps(message), content_type=JSON)
def dbserver(cmd, dbhostport=None, dbpath=os.path.expanduser(config.dbserver.file), foreground=False): """ start/stop/restart the database server, or return its status """ if config.dbserver.multi_user and getpass.getuser() != 'openquake': sys.exit('oq dbserver only works in single user mode') status = dbs.get_status() if cmd == 'status': print('dbserver ' + status) elif cmd == 'stop': if status == 'running': pid = logs.dbcmd('getpid') os.kill(pid, signal.SIGINT) # this is trapped by the DbServer else: print('dbserver already stopped') elif cmd == 'start': if status == 'not-running': dbs.run_server(dbpath, dbhostport) else: print('dbserver already running') elif cmd == 'restart': if status == 'running': pid = logs.dbcmd('getpid') os.kill(pid, signal.SIGINT) dbs.run_server(dbpath, dbhostport)
def abort(job_id): """ Abort the given job """ job = logs.dbcmd('get_job', job_id) # job_id can be negative if job is None: print('There is no job %d' % job_id) return elif job.status not in ('executing', 'running'): print('Job %d is %s' % (job.id, job.status)) return name = 'oq-job-%d' % job.id for p in psutil.process_iter(): if p.name() == name: try: os.kill(p.pid, signal.SIGINT) logs.dbcmd('set_status', job.id, 'aborted') print('Job %d aborted' % job.id) except Exception as exc: print(exc) break else: # no break # set job as failed if it is set as 'executing' or 'running' in the db # but the corresponding process is not running anymore logs.dbcmd('set_status', job.id, 'failed') print('Unable to find a process for job %d,' ' setting it as failed' % job.id)
def expose_outputs(dstore, owner=getpass.getuser(), status='complete'): """ Build a correspondence between the outputs in the datastore and the ones in the database. :param dstore: datastore """ oq = dstore['oqparam'] exportable = set(ekey[0] for ekey in export.export) calcmode = oq.calculation_mode dskeys = set(dstore) & exportable # exportable datastore keys dskeys.add('fullreport') rlzs = dstore['csm_info'].rlzs if len(rlzs) > 1: dskeys.add('realizations') if len(dstore['csm_info/sg_data']) > 1: # export sourcegroups.csv dskeys.add('sourcegroups') hdf5 = dstore.hdf5 if 'hcurves-stats' in hdf5 or 'hcurves-rlzs' in hdf5: if oq.hazard_stats() or oq.individual_curves or len(rlzs) == 1: dskeys.add('hcurves') if oq.uniform_hazard_spectra: dskeys.add('uhs') # export them if oq.hazard_maps: dskeys.add('hmaps') # export them if 'avg_losses-stats' in dstore or ( 'avg_losses-rlzs' in dstore and len(rlzs)): dskeys.add('avg_losses-stats') if 'curves-rlzs' in dstore and len(rlzs) == 1: dskeys.add('loss_curves-rlzs') if 'curves-stats' in dstore and len(rlzs) > 1: dskeys.add('loss_curves-stats') if oq.conditional_loss_poes: # expose loss_maps outputs if 'loss_curves-stats' in dstore: dskeys.add('loss_maps-stats') if 'all_loss_ratios' in dskeys: dskeys.remove('all_loss_ratios') # export only specific IDs if 'ruptures' in dskeys and 'scenario' in calcmode: exportable.remove('ruptures') # do not export, as requested by Vitor if 'rup_loss_table' in dskeys: # keep it hidden for the moment dskeys.remove('rup_loss_table') if 'hmaps' in dskeys and not oq.hazard_maps: dskeys.remove('hmaps') # do not export the hazard maps if logs.dbcmd('get_job', dstore.calc_id) is None: # the calculation has not been imported in the db yet logs.dbcmd('import_job', dstore.calc_id, oq.calculation_mode, oq.description + ' [parent]', owner, status, oq.hazard_calculation_id, dstore.datadir) keysize = [] for key in sorted(dskeys & exportable): try: size_mb = dstore.get_attr(key, 'nbytes') / MB except (KeyError, AttributeError): size_mb = None keysize.append((key, size_mb)) ds_size = os.path.getsize(dstore.filename) / MB logs.dbcmd('create_outputs', dstore.calc_id, keysize, ds_size)
def set_concurrent_tasks_default(job_id): """ Set the default for concurrent_tasks based on the number of available celery workers. """ stats = celery.task.control.inspect(timeout=1).stats() if not stats: logs.LOG.critical("No live compute nodes, aborting calculation") logs.dbcmd('finish', job_id, 'failed') sys.exit(1) ncores = sum(stats[k]['pool']['max-concurrency'] for k in stats) OqParam.concurrent_tasks.default = ncores * 3 logs.LOG.warn('Using %s, %d cores', ', '.join(sorted(stats)), ncores)
def importcalc(calc_id): """ Import a remote calculation into the local database. server, username and password must be specified in an openquake.cfg file. NB: calc_id can be a local pathname to a datastore not already present in the database: in that case it is imported in the db. """ dbserver.ensure_on() try: calc_id = int(calc_id) except ValueError: # assume calc_id is a pathname calc_id, datadir = datastore.extract_calc_id_datadir(calc_id) status = 'complete' remote = False else: remote = True job = logs.dbcmd('get_job', calc_id) if job is not None: sys.exit('There is already a job #%d in the local db' % calc_id) if remote: datadir = datastore.get_datadir() webex = WebExtractor(calc_id) status = webex.status['status'] hc_id = webex.oqparam.hazard_calculation_id if hc_id: sys.exit('The job has a parent (#%d) and cannot be ' 'downloaded' % hc_id) webex.dump('%s/calc_%d.hdf5' % (datadir, calc_id)) webex.close() with datastore.read(calc_id) as dstore: engine.expose_outputs(dstore, status=status) logging.info('Imported calculation %d successfully', calc_id)
def run_job(job_ini, log_level='info', log_file=None, exports='', username=getpass.getuser(), **kw): """ Run a job using the specified config file and other options. :param str job_ini: Path to calculation config (INI-style) files. :param str log_level: 'debug', 'info', 'warn', 'error', or 'critical' :param str log_file: Path to log file. :param exports: A comma-separated string of export types requested by the user. :param username: Name of the user running the job :param kw: Extra parameters like hazard_calculation_id and calculation_mode """ job_id = logs.init('job', getattr(logging, log_level.upper())) with logs.handle(job_id, log_level, log_file): job_ini = os.path.abspath(job_ini) oqparam = eng.job_from_file(job_ini, job_id, username, **kw) kw['username'] = username eng.run_calc(job_id, oqparam, exports, **kw) for line in logs.dbcmd('list_outputs', job_id, False): safeprint(line) return job_id
def reset(yes): """ Remove all the datastores and the database of the current user """ ok = yes or confirm('Do you really want to destroy all your data? (y/n) ') if not ok: return dbpath = os.path.realpath(os.path.expanduser(config.dbserver.file)) # user must be able to access and write the databse file to remove it if os.path.isfile(dbpath) and os.access(dbpath, os.W_OK): if dbserver.get_status() == 'running': if config.dbserver.multi_user: sys.exit('The oq dbserver must be stopped ' 'before proceeding') else: pid = logs.dbcmd('getpid') os.kill(pid, signal.SIGTERM) time.sleep(.5) # give time to stop assert dbserver.get_status() == 'not-running' print('dbserver stopped') try: os.remove(dbpath) print('Removed %s' % dbpath) except OSError as exc: print(exc, file=sys.stderr) # fast way of removing everything purge_all(fast=True) # datastore of the current user
def calc_datastore(request, job_id): """ Download a full datastore file. :param request: `django.http.HttpRequest` object. :param job_id: The id of the requested datastore :returns: A `django.http.HttpResponse` containing the content of the requested artifact, if present, else throws a 404 """ job = logs.dbcmd('get_job', int(job_id)) if job is None: return HttpResponseNotFound() if not utils.user_has_permission(request, job.user_name): return HttpResponseForbidden() fname = job.ds_calc_dir + '.hdf5' response = FileResponse( FileWrapper(open(fname, 'rb')), content_type=HDF5) response['Content-Disposition'] = ( 'attachment; filename=%s' % os.path.basename(fname)) response['Content-Length'] = str(os.path.getsize(fname)) return response
def make_report(isodate='today'): """ Build a HTML report with the computations performed at the given isodate. Return the name of the report, which is saved in the current directory. """ if isodate == 'today': isodate = date.today() else: isodate = date(*time.strptime(isodate, '%Y-%m-%d')[:3]) isodate1 = isodate + timedelta(1) # +1 day tag_ids = [] tag_status = [] tag_contents = [] # the fetcher returns an header which is stripped with [1:] jobs = dbcmd( 'fetch', ALL_JOBS, isodate.isoformat(), isodate1.isoformat()) page = '<h2>%d job(s) finished before midnight of %s</h2>' % ( len(jobs), isodate) for job_id, user, status, ds_calc in jobs: tag_ids.append(job_id) tag_status.append(status) [stats] = dbcmd('fetch', JOB_STATS, job_id) (job_id, user, start_time, stop_time, status, duration) = stats try: ds = read(job_id, datadir=os.path.dirname(ds_calc)) txt = view_fullreport('fullreport', ds) report = html_parts(txt) except Exception as exc: report = dict( html_title='Could not generate report: %s' % cgi.escape( str(exc), quote=True), fragment='') page = report['html_title'] page += html([stats._fields, stats]) page += report['fragment'] tag_contents.append(page) page = make_tabs(tag_ids, tag_status, tag_contents) + ( 'Report last updated: %s' % datetime.now()) fname = 'jobs-%s.html' % isodate with open(fname, 'w') as f: f.write(PAGE_TEMPLATE % page) return fname
def calc_log_size(request, calc_id): """ Get the current number of lines in the log """ try: response_data = logs.dbcmd('get_log_size', calc_id) except dbapi.NotFound: return HttpResponseNotFound() return HttpResponse(content=json.dumps(response_data), content_type=JSON)
def calc_results(request, calc_id): """ Get a summarized list of calculation results for a given ``calc_id``. Result is a JSON array of objects containing the following attributes: * id * name * type (hazard_curve, hazard_map, etc.) * url (the exact url where the full result can be accessed) """ # If the specified calculation doesn't exist OR is not yet complete, # throw back a 404. try: info = logs.dbcmd('calc_info', calc_id) if not utils.user_has_permission(request, info['user_name']): return HttpResponseForbidden() except dbapi.NotFound: return HttpResponseNotFound() base_url = _get_base_url(request) # NB: export_output has as keys the list (output_type, extension) # so this returns an ordered map output_type -> extensions such as # {'agg_loss_curve': ['xml', 'csv'], ...} output_types = groupby(export, lambda oe: oe[0], lambda oes: [e for o, e in oes]) results = logs.dbcmd('get_outputs', calc_id) if not results: return HttpResponseNotFound() response_data = [] for result in results: try: # output from the datastore rtype = result.ds_key # Catalina asked to remove the .txt outputs (used for the GMFs) outtypes = [ot for ot in output_types[rtype] if ot != 'txt'] except KeyError: continue # non-exportable outputs should not be shown url = urlparse.urljoin(base_url, 'v1/calc/result/%d' % result.id) datum = dict( id=result.id, name=result.display_name, type=rtype, outtypes=outtypes, url=url, size_mb=result.size_mb) response_data.append(datum) return HttpResponse(content=json.dumps(response_data))
def calc_traceback(request, calc_id): """ Get the traceback as a list of lines for a given ``calc_id``. """ # If the specified calculation doesn't exist throw back a 404. try: response_data = logs.dbcmd('get_traceback', calc_id) except dbapi.NotFound: return HttpResponseNotFound() return HttpResponse(content=json.dumps(response_data), content_type=JSON)
def submit_job(job_ini, username, hazard_job_id=None): """ Create a job object from the given job.ini file in the job directory and run it in a new process. Returns the job ID and PID. """ job_id = logs.init('job') oq = engine.job_from_file( job_ini, job_id, username, hazard_calculation_id=hazard_job_id) pik = pickle.dumps(oq, protocol=0) # human readable protocol code = RUNCALC % dict(job_id=job_id, hazard_job_id=hazard_job_id, pik=pik, username=username) tmp_py = gettemp(code, suffix='.py') # print(code, tmp_py) # useful when debugging devnull = subprocess.DEVNULL popen = subprocess.Popen([sys.executable, tmp_py], stdin=devnull, stdout=devnull, stderr=devnull) threading.Thread(target=popen.wait).start() logs.dbcmd('update_job', job_id, {'pid': popen.pid}) return job_id, popen.pid
def export_outputs(job_id, target_dir, export_types): # make it possible commands like `oq engine --eos -1 /tmp` datadir, dskeys = logs.dbcmd('get_results', job_id) if not dskeys: yield('Found nothing to export for job %s' % job_id) for dskey in dskeys: yield('Exporting %s...' % dskey) for line in export_output( dskey, job_id, datadir, target_dir, export_types): yield line
def purge_one(calc_id, user): """ Remove one calculation ID from the database and remove its datastore """ hdf5path = os.path.join(datastore.DATADIR, 'calc_%s.hdf5' % calc_id) err = dbcmd('del_calc', calc_id, user) if err: print(err) if os.path.exists(hdf5path): # not removed yet os.remove(hdf5path) print('Removed %s' % hdf5path)
def calc_log(request, calc_id, start, stop): """ Get a slice of the calculation log as a JSON list of rows """ start = start or 0 stop = stop or 0 try: response_data = logs.dbcmd('get_log_slice', calc_id, start, stop) except dbapi.NotFound: return HttpResponseNotFound() return HttpResponse(content=json.dumps(response_data), content_type=JSON)
def get_log_slice(request, calc_id, start, stop): """ Get a slice of the calculation log as a JSON list of rows """ start = start or 0 stop = stop or 0 try: response_data = logs.dbcmd('get_log_slice', calc_id, start, stop) except dbapi.NotFound: return HttpResponseNotFound() return HttpResponse(content=json.dumps(response_data), content_type=JSON)
def check_foreign(): """ Check if we the DbServer is the right one """ if not config.dbserver.multi_user: remote_server_path = logs.dbcmd('get_path') if different_paths(server_path, remote_server_path): return ('You are trying to contact a DbServer from another' ' instance (got %s, expected %s)\n' 'Check the configuration or stop the foreign' ' DbServer instance') % (remote_server_path, server_path)
def purge_one(calc_id, user): """ Remove one calculation ID from the database and remove its datastore """ filename = os.path.join(datadir, 'calc_%s.hdf5' % calc_id) err = dbcmd('del_calc', calc_id, user) if err: print(err) elif os.path.exists(filename): # not removed yet os.remove(filename) print('Removed %s' % filename)
def get_oqparam(request, job_id): """ Return the calculation parameters as a JSON """ try: job = logs.dbcmd('get_job', int(job_id), getpass.getuser()) except dbapi.NotFound: return HttpResponseNotFound() with datastore.read(job.ds_calc_dir + '.hdf5') as ds: oq = ds['oqparam'] return HttpResponse(content=json.dumps(vars(oq)), content_type=JSON)
def calc_info(request, calc_id): """ Get a JSON blob containing all of parameters for the given calculation (specified by ``calc_id``). Also includes the current job status ( executing, complete, etc.). """ try: info = logs.dbcmd('calc_info', calc_id) except dbapi.NotFound: return HttpResponseNotFound() return HttpResponse(content=json.dumps(info), content_type=JSON)
def check_foreign(): """ Check if we the DbServer is the right one """ if not config.dbserver.multi_user: remote_server_path = logs.dbcmd('get_path') if different_paths(server_path, remote_server_path): return('You are trying to contact a DbServer from another' ' instance (got %s, expected %s)\n' 'Check the configuration or stop the foreign' ' DbServer instance') % (remote_server_path, server_path)
def classical_split_filter(srcs, srcfilter, gsims, params, monitor): """ Split the given sources, filter the subsources and the compute the PoEs. Yield back subtasks if the split sources contain more than maxweight ruptures. """ # first check if we are sampling the sources ss = int(os.environ.get('OQ_SAMPLE_SOURCES', 0)) if ss: splits, stime = split_sources(srcs) srcs = random_filtered_sources(splits, srcfilter, ss) yield classical(srcs, srcfilter, gsims, params, monitor) return # NB: splitting all the sources improves the distribution significantly, # compared to splitting only the big sources with monitor("splitting/filtering sources"): splits, _stime = split_sources(srcs) sources = list(srcfilter.filter(splits)) if not sources: yield {'pmap': {}} return maxw = params['max_weight'] N = len(srcfilter.sitecol.complete) def weight(src): n = 10 * numpy.sqrt(len(src.indices) / N) return src.weight * params['rescale_weight'] * n blocks = list(block_splitter(sources, maxw, weight)) subtasks = len(blocks) - 1 for block in blocks[:-1]: yield classical, block, srcfilter, gsims, params if monitor.calc_id and subtasks: msg = 'produced %d subtask(s) with mean weight %d' % ( subtasks, numpy.mean([b.weight for b in blocks[:-1]])) try: logs.dbcmd('log', monitor.calc_id, datetime.utcnow(), 'DEBUG', 'classical_split_filter#%d' % monitor.task_no, msg) except Exception: # a foreign key error in case of `oq run` is expected print(msg) yield classical(blocks[-1], srcfilter, gsims, params, monitor)
def del_calculation(job_id, confirmed=False): """ Delete a calculation and all associated outputs. """ if logs.dbcmd('get_job', job_id) is None: print('There is no job %d' % job_id) return if confirmed or confirm( 'Are you sure you want to (abort and) delete this calculation and ' 'all associated outputs?\nThis action cannot be undone. (y/n): '): try: abort(job_id) resp = logs.dbcmd('del_calc', job_id, getpass.getuser()) except RuntimeError as err: safeprint(err) else: if 'success' in resp: print('Removed %d' % job_id) else: print(resp['error'])
def set_concurrent_tasks_default(calc): """ Look at the number of available workers and update the parameter OqParam.concurrent_tasks.default. Abort the calculations if no workers are available. Do nothing for trivial distributions. """ if OQ_DISTRIBUTE in 'no processpool': # do nothing num_workers = 0 if OQ_DISTRIBUTE == 'no' else parallel.CT // 2 logging.warning('Using %d cores on %s', num_workers, platform.node()) return num_workers = sum(total for host, running, total in parallel.workers_wait()) if num_workers == 0: logging.critical("No live compute nodes, aborting calculation") logs.dbcmd('finish', calc.datastore.calc_id, 'failed') sys.exit(1) parallel.CT = num_workers * 2 OqParam.concurrent_tasks.default = num_workers * 2 logging.warning('Using %d %s workers', num_workers, OQ_DISTRIBUTE)
def get_oqparam(request, job_id): """ Return the calculation parameters as a JSON """ user = utils.get_user_data(request) username = user['name'] if user['acl_on'] else None job = logs.dbcmd('get_job', int(job_id), username) if job is None: return HttpResponseNotFound() with datastore.read(job.ds_calc_dir + '.hdf5') as ds: oq = ds['oqparam'] return HttpResponse(content=json.dumps(vars(oq)), content_type=JSON)
def del_calculation(job_id, confirmed=False): """ Delete a calculation and all associated outputs. """ if logs.dbcmd('get_job', job_id) is None: print('There is no job %d' % job_id) return if confirmed or confirm( 'Are you sure you want to (abort and) delete this calculation and ' 'all associated outputs?\nThis action cannot be undone. (y/n): '): try: abort.func(job_id) resp = logs.dbcmd('del_calc', job_id, getpass.getuser()) except RuntimeError as err: safeprint(err) else: if 'success' in resp: print('Removed %d' % job_id) else: print(resp['error'])
def poll_queue(job_id, poll_time): """ Check the queue of executing/submitted jobs and exit when there is a free slot. """ offset = config.distribution.serialize_jobs - 1 if offset >= 0: first_time = True while True: running = logs.dbcmd(GET_JOBS) previous = [job for job in running if job.id < job_id - offset] if previous: if first_time: logs.dbcmd('update_job', job_id, {'status': 'submitted', 'pid': _PID}) first_time = False # the logging is not yet initialized, so use a print print('Waiting for jobs %s' % [p.id for p in previous]) time.sleep(poll_time) else: break
def main(calc_id: int, aggregate_by): """ Re-run the postprocessing after an event based risk calculation """ parent = util.read(calc_id) oqp = parent['oqparam'] aggby = aggregate_by.split(',') for tagname in aggby: if tagname not in oqp.aggregate_by: raise ValueError('%r not in %s' % (tagname, oqp.aggregate_by)) job_id = logs.init('job', level=logging.INFO) dic = dict( calculation_mode='reaggregate', description=oqp.description + '[aggregate_by=%s]' % aggregate_by, user_name=getpass.getuser(), is_running=1, status='executing', pid=os.getpid(), hazard_calculation_id=job_id) logs.dbcmd('update_job', job_id, dic) if os.environ.get('OQ_DISTRIBUTE') not in ('no', 'processpool'): os.environ['OQ_DISTRIBUTE'] = 'processpool' with logs.handle(job_id, logging.INFO): oqp.hazard_calculation_id = parent.calc_id parallel.Starmap.init() prc = PostRiskCalculator(oqp, job_id) try: prc.run(aggregate_by=aggby) engine.expose_outputs(prc.datastore) logs.dbcmd('finish', job_id, 'complete') except Exception: logs.dbcmd('finish', job_id, 'failed') finally: parallel.Starmap.shutdown()
def submit_job(request_files, ini, username, hc_id): """ Create a job object from the given files and run it in a new process. :returns: a job ID """ # build a LogContext object associated to a database job [job] = engine.create_jobs([ dict(calculation_mode='preclassical', description='Calculation waiting to start') ], config.distribution.log_level, None, username, hc_id) # store the request files and perform some validation try: job_ini = store(request_files, ini, job.calc_id) job.oqparam = oq = readinput.get_oqparam( job_ini, kw={'hazard_calculation_id': hc_id}) if oq.sensitivity_analysis: logs.dbcmd('set_status', job.calc_id, 'deleted') # hide it jobs = engine.create_jobs([job_ini], config.distribution.log_level, None, username, hc_id, True) else: dic = dict(calculation_mode=oq.calculation_mode, description=oq.description, hazard_calculation_id=hc_id) logs.dbcmd('update_job', job.calc_id, dic) jobs = [job] except Exception: tb = traceback.format_exc() logs.dbcmd('log', job.calc_id, datetime.utcnow(), 'CRITICAL', 'before starting', tb) logs.dbcmd('finish', job.calc_id, 'failed') raise custom_tmp = os.path.dirname(job_ini) submit_cmd = config.distribution.submit_cmd.split() big_job = oq.get_input_size() > int(config.distribution.min_input_size) if submit_cmd == ENGINE: # used for debugging for job in jobs: subprocess.Popen(submit_cmd + [save(job, custom_tmp)]) elif submit_cmd == KUBECTL and big_job: for job in jobs: with open(os.path.join(CWD, 'job.yaml')) as f: yaml = string.Template(f.read()).substitute( CALC_PIK=save(job, custom_tmp), CALC_NAME='calc%d' % job.calc_id) subprocess.run(submit_cmd, input=yaml.encode('ascii')) else: Process(target=engine.run_jobs, args=(jobs, )).start() return job.calc_id
def ebrisk(rupgetter, srcfilter, param, monitor): """ :param rupgetter: RuptureGetter with multiple ruptures :param srcfilter: a SourceFilter :param param: dictionary of parameters coming from oqparam :param monitor: a Monitor instance :returns: a dictionary with keys elt, alt, ... """ mon_rup = monitor('getting ruptures', measuremem=False) mon_haz = monitor('getting hazard', measuremem=False) gmfs = [] gmf_info = [] gg = getters.GmfGetter(rupgetter, srcfilter, param['oqparam'], param['amplifier']) nbytes = 0 for c in gg.gen_computers(mon_rup): with mon_haz: data, time_by_rup = c.compute_all(gg.min_iml, gg.rlzs_by_gsim) if len(data): gmfs.append(data) nbytes += data.nbytes gmf_info.append((c.ebrupture.id, mon_haz.task_no, len(c.sids), data.nbytes, mon_haz.dt)) if nbytes > param['ebrisk_maxsize']: msg = 'produced subtask' try: logs.dbcmd('log', monitor.calc_id, datetime.utcnow(), 'DEBUG', 'ebrisk#%d' % monitor.task_no, msg) except Exception: # for `oq run` print(msg) yield calc_risk, numpy.concatenate(gmfs), param nbytes = 0 gmfs = [] res = {} if gmfs: res.update(calc_risk(numpy.concatenate(gmfs), param, monitor)) if gmf_info: res['gmf_info'] = numpy.array(gmf_info, gmf_info_dt) yield res
def calc(request, calc_id): """ Get a JSON blob containing all of parameters for the given calculation (specified by ``calc_id``). Also includes the current job status ( executing, complete, etc.). """ try: info = logs.dbcmd('calc_info', calc_id) if not utils.user_has_permission(request, info['user_name']): return HttpResponseForbidden() except dbapi.NotFound: return HttpResponseNotFound() return HttpResponse(content=json.dumps(info), content_type=JSON)
def smart_run(job_ini, oqparam, log_level, log_file, exports, reuse_hazard, **params): """ Run calculations by storing their hazard checksum and reusing previous calculations if requested. """ haz_checksum = readinput.get_checksum32(oqparam, hazard=True) # retrieve an old calculation with the right checksum, if any job = logs.dbcmd('get_job_from_checksum', haz_checksum) reuse = reuse_hazard and job and os.path.exists(job.ds_calc_dir + '.hdf5') # recompute the hazard and store the checksum if not reuse: hc_id = run_job(job_ini, log_level, log_file, exports, **params) if job is None: logs.dbcmd('add_checksum', hc_id, haz_checksum) elif not reuse_hazard or not os.path.exists(job.ds_calc_dir + '.hdf5'): logs.dbcmd('update_job_checksum', hc_id, haz_checksum) else: hc_id = job.id logging.info('Reusing job #%d', job.id) run_job(job_ini, log_level, log_file, exports, hazard_calculation_id=hc_id, **params)
def abort(job_id): """ Abort the given job """ job = logs.dbcmd('get_job', job_id) # job_id can be negative if job is None: print('There is no job %d' % job_id) return elif job.status not in ('executing', 'running'): print('Job %d is %s' % (job.id, job.status)) return name = 'oq-job-%d' % job.id for p in psutil.process_iter(): if p.name() == name: try: os.kill(p.pid, signal.SIGTERM) logs.dbcmd('set_status', job.id, 'aborted') except Exception as exc: print(exc) break else: # no break print('%d aborted' % job.id)
def importcalc(host, calc_id, username, password): """ Import a remote calculation into the local database """ logging.basicConfig(level=logging.INFO) if '/' in host.split('//', 1)[1]: sys.exit('Wrong host ending with /%s' % host.rsplit('/', 1)[1]) calc_url = '/'.join([host, 'v1/calc', str(calc_id)]) dbserver.ensure_on() job = logs.dbcmd('get_job', calc_id) if job is not None: sys.exit('There is already a job #%d in the local db' % calc_id) datadir = datastore.get_datadir() session = login(host, username, password) status = session.get('%s/status' % calc_url) if 'Log in to an existing account' in status.text: sys.exit('Could not login') json = status.json() if json["parent_id"]: sys.exit('The job has a parent (#%(parent_id)d) and cannot be ' 'downloaded' % json) resp = session.get('%s/datastore' % calc_url, stream=True) assert resp.status_code == 200, resp.status_code fname = '%s/calc_%d.hdf5' % (datadir, calc_id) down = 0 with open(fname, 'wb') as f: logging.info('%s -> %s', calc_url, fname) for chunk in resp.iter_content(CHUNKSIZE): f.write(chunk) down += len(chunk) general.println('Downloaded {:,} bytes'.format(down)) print() logs.dbcmd('import_job', calc_id, json['calculation_mode'], json['description'], json['owner'], json['status'], json['parent_id'], datadir) with datastore.read(calc_id) as dstore: engine.expose_outputs(dstore) logging.info('Imported calculation %d successfully', calc_id)
def calc_oqparam(request, job_id): """ Return the calculation parameters as a JSON """ job = logs.dbcmd('get_job', int(job_id)) if job is None: return HttpResponseNotFound() if not utils.user_has_permission(request, job.user_name): return HttpResponseForbidden() with datastore.read(job.ds_calc_dir + '.hdf5') as ds: oq = ds['oqparam'] return HttpResponse(content=json.dumps(vars(oq)), content_type=JSON)
def classical_split_filter(srcs, srcfilter, gsims, params, monitor): """ Split the given sources, filter the subsources and the compute the PoEs. Yield back subtasks if the split sources contain more than maxweight ruptures. """ # first check if we are sampling the sources ss = int(os.environ.get('OQ_SAMPLE_SOURCES', 0)) if ss: splits, stime = split_sources(srcs) srcs = random_filtered_sources(splits, srcfilter, ss) yield classical(srcs, srcfilter, gsims, params, monitor) return # NB: splitting all the sources improves the distribution significantly, # compared to splitting only the big sources with monitor("splitting/filtering sources"): splits, _stime = split_sources(srcs) sources = list(srcfilter.filter(splits)) if sources: maxw = sum(src.weight for src in sources) / 5 if maxw < 1000: maxw = 1000 elif maxw > params['max_weight']: maxw = params['max_weight'] blocks = list(block_splitter(sources, maxw, weight)) nb = len(blocks) msg = 'produced %d subtask(s) with max weight=%d' % ( nb - 1, max(b.weight for b in blocks)) if monitor.calc_id and nb > 1: try: logs.dbcmd('log', monitor.calc_id, datetime.utcnow(), 'DEBUG', 'classical_split_filter#%d' % monitor.task_no, msg) except Exception: # a foreign key error in case of `oq run` is expected print(msg) for block in blocks[:-1]: yield classical, block, srcfilter, gsims, params yield classical(blocks[-1], srcfilter, gsims, params, monitor)
def calc_abort(request, calc_id): """ Abort the given calculation, it is it running """ job = logs.dbcmd('get_job', calc_id) if job is None: message = {'error': 'Unknown job %s' % calc_id} return HttpResponse(content=json.dumps(message), content_type=JSON) if job.status not in ('executing', 'running'): message = {'error': 'Job %s is not running' % job.id} return HttpResponse(content=json.dumps(message), content_type=JSON) user = utils.get_user_data(request) info = logs.dbcmd('calc_info', calc_id) allowed_users = user['group_members'] or [user['name']] if user['acl_on'] and info['user_name'] not in allowed_users: message = { 'error': ('User %s has no permission to abort job %s' % (info['user_name'], job.id)) } return HttpResponse(content=json.dumps(message), content_type=JSON, status=403) if job.pid: # is a spawned job try: os.kill(job.pid, signal.SIGTERM) except Exception as exc: logging.error(exc) else: logging.warn('Aborting job %d, pid=%d', job.id, job.pid) logs.dbcmd('set_status', job.id, 'aborted') message = {'success': 'Killing job %d' % job.id} return HttpResponse(content=json.dumps(message), content_type=JSON) message = {'error': 'PID for job %s not found' % job.id} return HttpResponse(content=json.dumps(message), content_type=JSON)
def job_from_file(job_ini, job_id, username, **kw): """ Create a full job profile from a job config file. :param job_ini: Path to a job.ini file :param job_id: ID of the created job :param username: The user who will own this job profile and all results :param kw: Extra parameters including `calculation_mode` and `exposure_file` :returns: an oqparam instance """ hc_id = kw.get('hazard_calculation_id') try: oq = readinput.get_oqparam(job_ini, hc_id=hc_id) except Exception: logs.dbcmd('finish', job_id, 'failed') raise if 'calculation_mode' in kw: oq.calculation_mode = kw.pop('calculation_mode') if 'description' in kw: oq.description = kw.pop('description') if 'exposure_file' in kw: # hack used in commands.engine fnames = kw.pop('exposure_file').split() if fnames: oq.inputs['exposure'] = fnames elif 'exposure' in oq.inputs: del oq.inputs['exposure'] logs.dbcmd( 'update_job', job_id, dict(calculation_mode=oq.calculation_mode, description=oq.description, user_name=username, hazard_calculation_id=hc_id)) return oq
def submit_job(job_ini, username, hazard_job_id=None): """ Create a job object from the given job.ini file in the job directory and run it in a new process. Returns the job ID and PID. """ job_id = logs.init('job') oq = engine.job_from_file(job_ini, job_id, username, hazard_calculation_id=hazard_job_id) pik = pickle.dumps(oq, protocol=0) # human readable protocol code = RUNCALC % dict( job_id=job_id, hazard_job_id=hazard_job_id, pik=pik, username=username) tmp_py = gettemp(code, suffix='.py') # print(code, tmp_py) # useful when debugging devnull = subprocess.DEVNULL popen = subprocess.Popen([sys.executable, tmp_py], stdin=devnull, stdout=devnull, stderr=devnull) threading.Thread(target=popen.wait).start() logs.dbcmd('update_job', job_id, {'pid': popen.pid}) return job_id, popen.pid
def calc_abort(request, calc_id): """ Abort the given calculation, it is it running """ job = logs.dbcmd('get_job', calc_id) if job is None: message = {'error': 'Unknown job %s' % calc_id} return HttpResponse(content=json.dumps(message), content_type=JSON) if job.status not in ('submitted', 'executing'): message = {'error': 'Job %s is not running' % job.id} return HttpResponse(content=json.dumps(message), content_type=JSON) # only the owner or superusers can abort a calculation if (job.user_name not in utils.get_valid_users(request) and not utils.is_superuser(request)): message = { 'error': ('User %s has no permission to abort job %s' % (request.user, job.id)) } return HttpResponse(content=json.dumps(message), content_type=JSON, status=403) if job.pid: # is a spawned job try: os.kill(job.pid, signal.SIGINT) except Exception as exc: logging.error(exc) else: logging.warning('Aborting job %d, pid=%d', job.id, job.pid) logs.dbcmd('set_status', job.id, 'aborted') message = {'success': 'Killing job %d' % job.id} return HttpResponse(content=json.dumps(message), content_type=JSON) message = {'error': 'PID for job %s not found' % job.id} return HttpResponse(content=json.dumps(message), content_type=JSON)
def job_from_file(job_ini, job_id, username, **kw): """ Create a full job profile from a job config file. :param job_ini: Path to a job.ini file :param job_id: ID of the created job :param username: The user who will own this job profile and all results :param kw: Extra parameters including `calculation_mode` and `exposure_file` :returns: an oqparam instance """ hc_id = kw.get('hazard_calculation_id') try: oq = readinput.get_oqparam(job_ini, hc_id=hc_id) except Exception: logs.dbcmd('finish', job_id, 'failed') raise if 'calculation_mode' in kw: oq.calculation_mode = kw.pop('calculation_mode') if 'description' in kw: oq.description = kw.pop('description') if 'exposure_file' in kw: # hack used in commands.engine fnames = kw.pop('exposure_file').split() if fnames: oq.inputs['exposure'] = fnames elif 'exposure' in oq.inputs: del oq.inputs['exposure'] logs.dbcmd('update_job', job_id, dict(calculation_mode=oq.calculation_mode, description=oq.description, user_name=username, hazard_calculation_id=hc_id)) return oq
def extract(request, calc_id, what): """ Wrapper over the `oq extract` command. If `setting.LOCKDOWN` is true only calculations owned by the current user can be retrieved. """ job = logs.dbcmd('get_job', int(calc_id)) if job is None: return HttpResponseNotFound() if not utils.user_has_permission(request, job.user_name): return HttpResponseForbidden() try: # read the data and save them on a temporary .npz file with datastore.read(job.ds_calc_dir + '.hdf5') as ds: fd, fname = tempfile.mkstemp(prefix=what.replace('/', '-'), suffix='.npz') os.close(fd) n = len(request.path_info) query_string = unquote_plus(request.get_full_path()[n:]) aw = _extract(ds, what + query_string) a = {} for key, val in vars(aw).items(): key = str(key) # can be a numpy.bytes_ if key.startswith('_'): continue elif isinstance(val, str): # without this oq extract would fail a[key] = numpy.array(val.encode('utf-8')) elif isinstance(val, dict): # this is hack: we are losing the values a[key] = list(val) else: a[key] = val numpy.savez_compressed(fname, **a) except Exception as exc: tb = ''.join(traceback.format_tb(exc.__traceback__)) return HttpResponse(content='%s: %s\n%s' % (exc.__class__.__name__, exc, tb), content_type='text/plain', status=500) # stream the data back stream = FileWrapper(open(fname, 'rb')) stream.close = lambda: (FileWrapper.close(stream), os.remove(fname)) response = FileResponse(stream, content_type='application/octet-stream') response['Content-Disposition'] = ('attachment; filename=%s' % os.path.basename(fname)) response['Content-Length'] = str(os.path.getsize(fname)) return response
def main(cmd, args=()): """ Run a database command """ if cmd in commands and len(args) != len(commands[cmd]): sys.exit('Wrong number of arguments, expected %s, got %s' % (commands[cmd], args)) elif (cmd not in commands and not cmd.upper().startswith('SELECT') and config.dbserver.multi_user and getpass.getuser() != 'openquake'): sys.exit('You have no permission to run %s' % cmd) dbserver.ensure_on() res = logs.dbcmd(cmd, *convert(args)) if hasattr(res, '_fields') and res.__class__.__name__ != 'Row': print(rst_table(res)) else: print(res)
def read(calc_id, username=None): """ :param calc_id: a calculation ID :param username: if given, restrict the search to the user's calculations :returns: the associated DataStore instance """ if isinstance(calc_id, str) or calc_id < 0 and not username: # get the last calculation in the datastore of the current user return datastore.read(calc_id) job = logs.dbcmd('get_job', calc_id, username) if job: return datastore.read(job.ds_calc_dir + '.hdf5') else: # calc_id can be present in the datastore and not in the database: # this happens if the calculation was run with `oq run` return datastore.read(calc_id)
def extract(request, calc_id, what): """ Wrapper over the `oq extract` command. If `setting.LOCKDOWN` is true only calculations owned by the current user can be retrieved. """ job = logs.dbcmd('get_job', int(calc_id)) if job is None: return HttpResponseNotFound() if not utils.user_has_permission(request, job.user_name): return HttpResponseForbidden() try: # read the data and save them on a temporary .npz file with datastore.read(job.ds_calc_dir + '.hdf5') as ds: fd, fname = tempfile.mkstemp( prefix=what.replace('/', '-'), suffix='.npz') os.close(fd) n = len(request.path_info) query_string = unquote_plus(request.get_full_path()[n:]) aw = _extract(ds, what + query_string) a = {} for key, val in vars(aw).items(): key = str(key) # can be a numpy.bytes_ if isinstance(val, str): # without this oq extract would fail a[key] = numpy.array(val.encode('utf-8')) elif isinstance(val, dict): # this is hack: we are losing the values a[key] = list(val) else: a[key] = val numpy.savez_compressed(fname, **a) except Exception as exc: tb = ''.join(traceback.format_tb(exc.__traceback__)) return HttpResponse( content='%s: %s\n%s' % (exc.__class__.__name__, exc, tb), content_type='text/plain', status=500) # stream the data back stream = FileWrapper(open(fname, 'rb')) stream.close = lambda: (FileWrapper.close(stream), os.remove(fname)) response = FileResponse(stream, content_type='application/octet-stream') response['Content-Disposition'] = ( 'attachment; filename=%s' % os.path.basename(fname)) response['Content-Length'] = str(os.path.getsize(fname)) return response
def calc_list(request, id=None): # view associated to the endpoints /v1/calc/list and /v1/calc/:id/status """ Get a list of calculations and report their id, status, calculation_mode, is_running, description, and a url where more detailed information can be accessed. This is called several times by the Javascript. Responses are in JSON. """ base_url = _get_base_url(request) # always filter calculation list unless user is a superuser calc_data = logs.dbcmd('get_calcs', request.GET, utils.get_valid_users(request), not utils.is_superuser(request), id) response_data = [] username = psutil.Process(os.getpid()).username() for (hc_id, owner, status, calculation_mode, is_running, desc, pid, parent_id, size_mb) in calc_data: url = urlparse.urljoin(base_url, 'v1/calc/%d' % hc_id) abortable = False if is_running: try: if psutil.Process(pid).username() == username: abortable = True except psutil.NoSuchProcess: pass response_data.append( dict(id=hc_id, owner=owner, calculation_mode=calculation_mode, status=status, is_running=bool(is_running), description=desc, url=url, parent_id=parent_id, abortable=abortable, size_mb=size_mb)) # if id is specified the related dictionary is returned instead the list if id is not None: if not response_data: return HttpResponseNotFound() [response_data] = response_data return HttpResponse(content=json.dumps(response_data), content_type=JSON)
def calc_remove(request, calc_id): """ Remove the calculation id """ user = utils.get_user_data(request)['name'] try: message = logs.dbcmd('del_calc', calc_id, user) except dbapi.NotFound: return HttpResponseNotFound() if isinstance(message, list): # list of removed files return HttpResponse(content=json.dumps(message), content_type=JSON, status=200) else: # FIXME: the error is not passed properly to the javascript logging.error(message) return HttpResponse(content=message, content_type='text/plain', status=500)
def run_calc(log): """ Run a calculation. :param log: LogContext of the current job """ register_signals() setproctitle('oq-job-%d' % log.calc_id) with log: oqparam = log.get_oqparam() calc = base.calculators(oqparam, log.calc_id) logging.info('%s running %s [--hc=%s]', getpass.getuser(), calc.oqparam.inputs['job_ini'], calc.oqparam.hazard_calculation_id) logging.info('Using engine version %s', __version__) msg = check_obsolete_version(oqparam.calculation_mode) # NB: disabling the warning should be done only for users with # an updated LTS version, but we are doing it for all users # if msg: # logging.warning(msg) calc.from_engine = True if config.zworkers['host_cores']: set_concurrent_tasks_default(calc) else: logging.warning('Assuming %d %s workers', parallel.Starmap.num_cores, OQ_DISTRIBUTE) t0 = time.time() calc.run() logging.info('Exposing the outputs to the database') expose_outputs(calc.datastore) path = calc.datastore.filename size = general.humansize(getsize(path)) logging.info('Stored %s on %s in %d seconds', size, path, time.time() - t0) calc.datastore.close() for line in logs.dbcmd('list_outputs', log.calc_id, False): general.safeprint(line) # sanity check to make sure that the logging on file is working if (log.log_file and log.log_file != os.devnull and getsize(log.log_file) == 0): logging.warning('The log file %s is empty!?' % log.log_file) return calc
def extract(request, calc_id, what): """ Wrapper over the `oq extract` command. If setting.LOCKDOWN is true only calculations owned by the current user can be retrieved. """ job = logs.dbcmd('get_job', int(calc_id)) if job is None: return HttpResponseNotFound() if not utils.user_has_permission(request, job.user_name): return HttpResponseForbidden() # read the data and save them on a temporary .pik file with datastore.read(job.ds_calc_dir + '.hdf5') as ds: fd, fname = tempfile.mkstemp( prefix=what.replace('/', '-'), suffix='.npz') os.close(fd) n = len(request.path_info) query_string = unquote_plus(request.get_full_path()[n:]) obj = _extract(ds, what + query_string) if inspect.isgenerator(obj): array, attrs = 0, {k: _array(v) for k, v in obj} elif hasattr(obj, '__toh5__'): array, attrs = obj.__toh5__() else: # assume obj is an array array, attrs = obj, {} a = {} for key, val in attrs.items(): if isinstance(key, bytes): key = key.decode('utf-8') if isinstance(val, str): # without this oq extract would fail a[key] = numpy.array(val.encode('utf-8')) else: a[key] = val numpy.savez_compressed(fname, array=array, **a) # stream the data back stream = FileWrapper(open(fname, 'rb')) stream.close = lambda: (FileWrapper.close(stream), os.remove(fname)) response = FileResponse(stream, content_type='application/octet-stream') response['Content-Disposition'] = ( 'attachment; filename=%s' % os.path.basename(fname)) response['Content-Length'] = str(os.path.getsize(fname)) return response
def db(cmd, args=()): """ Run a database command """ if cmd not in commands: okcmds = '\n'.join( '%s %s' % (name, repr(' '.join(args)) if args else '') for name, args in sorted(commands.items())) print('Invalid command "%s": choose one from\n%s' % (cmd, okcmds)) elif len(args) != len(commands[cmd]): print('Wrong number of arguments, expected %s, got %s' % ( commands[cmd], args)) else: dbserver.ensure_on() res = logs.dbcmd(cmd, *convert(args)) if hasattr(res, '_fields') and res.__class__.__name__ != 'Row': print(rst_table(res)) else: print(res)
def calc_list(request, id=None): # view associated to the endpoints /v1/calc/list and /v1/calc/:id/status """ Get a list of calculations and report their id, status, calculation_mode, is_running, description, and a url where more detailed information can be accessed. This is called several times by the Javascript. Responses are in JSON. """ base_url = _get_base_url(request) calc_data = logs.dbcmd('get_calcs', request.GET, utils.get_valid_users(request), utils.get_acl_on(request), id) response_data = [] username = psutil.Process(os.getpid()).username() for (hc_id, owner, status, calculation_mode, is_running, desc, pid, parent_id, size_mb) in calc_data: url = urlparse.urljoin(base_url, 'v1/calc/%d' % hc_id) abortable = False if is_running: try: if psutil.Process(pid).username() == username: abortable = True except psutil.NoSuchProcess: pass response_data.append( dict(id=hc_id, owner=owner, calculation_mode=calculation_mode, status=status, is_running=bool(is_running), description=desc, url=url, parent_id=parent_id, abortable=abortable, size_mb=size_mb)) # if id is specified the related dictionary is returned instead the list if id is not None: [response_data] = response_data return HttpResponse(content=json.dumps(response_data), content_type=JSON)
def calc_remove(request, calc_id): """ Remove the calculation id """ # Only the owner can remove a job user = utils.get_user(request) try: message = logs.dbcmd('del_calc', calc_id, user) except dbapi.NotFound: return HttpResponseNotFound() if 'success' in message: return HttpResponse(content=json.dumps(message), content_type=JSON, status=200) elif 'error' in message: logging.error(message['error']) return HttpResponse(content=json.dumps(message), content_type=JSON, status=403) else: # This is an untrapped server error logging.error(message) return HttpResponse(content=message, content_type='text/plain', status=500)
def poll_queue(job_id, pid, poll_time): """ Check the queue of executing/submitted jobs and exit when there is a free slot. """ if config.distribution.serialize_jobs: first_time = True while True: jobs = logs.dbcmd(GET_JOBS) failed = [job.id for job in jobs if not psutil.pid_exists(job.pid)] if failed: logs.dbcmd("UPDATE job SET status='failed', is_running=0 " "WHERE id in (?X)", failed) elif any(job.id < job_id for job in jobs): if first_time: logs.LOG.warn('Waiting for jobs %s', [j.id for j in jobs]) logs.dbcmd('update_job', job_id, {'status': 'submitted', 'pid': pid}) first_time = False time.sleep(poll_time) else: break logs.dbcmd('update_job', job_id, {'status': 'executing', 'pid': _PID})
def web_engine_get_outputs(request, calc_id, **kwargs): job = logs.dbcmd('get_job', calc_id) size_mb = '?' if job.size_mb is None else '%.2f' % job.size_mb return render(request, "engine/get_outputs.html", dict(calc_id=calc_id, size_mb=size_mb))