def zip(what, archive_zip='', risk_file=''): logging.basicConfig(level=logging.INFO) if os.path.isdir(what): oqzip.zip_all(what) elif what.endswith('.xml') and '<logicTree' in open(what).read(512): # hack to see if the NRML file is of kind logicTree oqzip.zip_source_model(what, archive_zip) elif what.endswith('.xml') and '<exposureModel' in open(what).read(512): # hack to see if the NRML file is of kind exposureModel oqzip.zip_exposure(what, archive_zip) elif what.endswith('.ini'): # a job.ini oqzip.zip_job(what, archive_zip, risk_file) else: sys.exit('Cannot zip %s' % what)
def zip(what, archive_zip='', risk_file=''): """ Zip into an archive one or two job.ini files with all related files """ if os.path.isdir(what): oqzip.zip_all(what) elif what.endswith('.xml') and '<logicTree' in open(what).read(512): # hack to see if the NRML file is of kind logicTree oqzip.zip_source_model(what, archive_zip) elif what.endswith('.xml') and '<exposureModel' in open(what).read(512): # hack to see if the NRML file is of kind exposureModel oqzip.zip_exposure(what, archive_zip) elif what.endswith('.ini'): # a job.ini oqzip.zip_job(what, archive_zip, risk_file) else: sys.exit('Cannot zip %s' % what)
def main(zipfile): """Post a zipfile to the WebUI""" sess = requests.Session() if config.webapi.username: login_url = '%s/accounts/ajax_login/' % config.webapi.server logging.info('POST %s', login_url) resp = sess.post(login_url, data=dict(username=config.webapi.username, password=config.webapi.password)) if resp.status_code != 200: raise WebAPIError(resp.text) if zipfile.endswith('.ini'): # not a zip file yet archive = zipfile[:-3] + 'zip' oqzip.zip_job(zipfile, archive) zipfile = archive resp = sess.post("%s/v1/calc/run" % config.webapi.server, {}, files=dict(archive=open(zipfile, 'rb'))) print(json.loads(resp.text))
def run_calc(job_id, oqparam, exports, hazard_calculation_id=None, **kw): """ Run a calculation. :param job_id: ID of the current job :param oqparam: :class:`openquake.commonlib.oqvalidation.OqParam` instance :param exports: A comma-separated string of export types. """ register_signals() setproctitle('oq-job-%d' % job_id) calc = base.calculators(oqparam, calc_id=job_id) logging.info('%s running %s [--hc=%s]', getpass.getuser(), calc.oqparam.inputs['job_ini'], calc.oqparam.hazard_calculation_id) logging.info('Using engine version %s', __version__) msg = check_obsolete_version(oqparam.calculation_mode) if msg: logs.LOG.warn(msg) calc.from_engine = True tb = 'None\n' try: if not oqparam.hazard_calculation_id: if 'input_zip' in oqparam.inputs: # starting from an archive with open(oqparam.inputs['input_zip'], 'rb') as arch: data = numpy.array(arch.read()) else: logs.LOG.info('Zipping the input files') bio = io.BytesIO() oqzip.zip_job(oqparam.inputs['job_ini'], bio, (), oqparam, logging.debug) data = numpy.array(bio.getvalue()) del bio calc.datastore['input/zip'] = data calc.datastore.set_attrs('input/zip', nbytes=data.nbytes) del data # save memory poll_queue(job_id, _PID, poll_time=15) if OQ_DISTRIBUTE.endswith('pool'): logs.LOG.warning('Using %d cores on %s', parallel.cpu_count, platform.node()) if OQ_DISTRIBUTE == 'zmq': logs.dbcmd('zmq_start') # start zworkers logs.dbcmd('zmq_wait') # wait for them to go up if OQ_DISTRIBUTE.startswith(('celery', 'zmq')): set_concurrent_tasks_default(job_id) t0 = time.time() calc.run(exports=exports, hazard_calculation_id=hazard_calculation_id, close=False, **kw) logs.LOG.info('Exposing the outputs to the database') expose_outputs(calc.datastore) duration = time.time() - t0 records = views.performance_view(calc.datastore, add_calc_id=False) logs.dbcmd('save_performance', job_id, records) calc.datastore.close() logs.LOG.info('Calculation %d finished correctly in %d seconds', job_id, duration) logs.dbcmd('finish', job_id, 'complete') except BaseException as exc: if isinstance(exc, MasterKilled): msg = 'aborted' else: msg = 'failed' tb = traceback.format_exc() try: logs.LOG.critical(tb) logs.dbcmd('finish', job_id, msg) except BaseException: # an OperationalError may always happen sys.stderr.write(tb) raise finally: # if there was an error in the calculation, this part may fail; # in such a situation, we simply log the cleanup error without # taking further action, so that the real error can propagate if OQ_DISTRIBUTE == 'zmq': # stop zworkers logs.dbcmd('zmq_stop') try: if OQ_DISTRIBUTE.startswith('celery'): celery_cleanup(TERMINATE) except BaseException: # log the finalization error only if there is no real error if tb == 'None\n': logs.LOG.error('finalizing', exc_info=True) return calc
def run_calc(job_id, oqparam, exports, hazard_calculation_id=None, **kw): """ Run a calculation. :param job_id: ID of the current job :param oqparam: :class:`openquake.commonlib.oqvalidation.OqParam` instance :param exports: A comma-separated string of export types. """ register_signals() setproctitle('oq-job-%d' % job_id) calc = base.calculators(oqparam, calc_id=job_id) logging.info('%s running %s [--hc=%s]', getpass.getuser(), calc.oqparam.inputs['job_ini'], calc.oqparam.hazard_calculation_id) logging.info('Using engine version %s', __version__) msg = check_obsolete_version(oqparam.calculation_mode) if msg: logs.LOG.warn(msg) if OQ_DISTRIBUTE.startswith(('celery', 'zmq')): set_concurrent_tasks_default(job_id) calc.from_engine = True tb = 'None\n' try: if not oqparam.hazard_calculation_id: if 'input_zip' in oqparam.inputs: # starting from an archive with open(oqparam.inputs['input_zip'], 'rb') as arch: data = numpy.array(arch.read()) else: logs.LOG.info('zipping the input files') bio = io.BytesIO() oqzip.zip_job(oqparam.inputs['job_ini'], bio, (), oqparam, logging.debug) data = numpy.array(bio.getvalue()) del bio calc.datastore['input/zip'] = data calc.datastore.set_attrs('input/zip', nbytes=data.nbytes) del data # save memory poll_queue(job_id, _PID, poll_time=15) t0 = time.time() calc.run(exports=exports, hazard_calculation_id=hazard_calculation_id, close=False, **kw) logs.LOG.info('Exposing the outputs to the database') expose_outputs(calc.datastore) duration = time.time() - t0 calc._monitor.flush() records = views.performance_view(calc.datastore) logs.dbcmd('save_performance', job_id, records) calc.datastore.close() logs.LOG.info('Calculation %d finished correctly in %d seconds', job_id, duration) logs.dbcmd('finish', job_id, 'complete') except BaseException as exc: if isinstance(exc, MasterKilled): msg = 'aborted' else: msg = 'failed' tb = traceback.format_exc() try: logs.LOG.critical(tb) logs.dbcmd('finish', job_id, msg) except BaseException: # an OperationalError may always happen sys.stderr.write(tb) raise finally: # if there was an error in the calculation, this part may fail; # in such a situation, we simply log the cleanup error without # taking further action, so that the real error can propagate try: if OQ_DISTRIBUTE.startswith('celery'): celery_cleanup(TERMINATE) except BaseException: # log the finalization error only if there is no real error if tb == 'None\n': logs.LOG.error('finalizing', exc_info=True) return calc