def test_supervised(self):
# Due to the way the executor/supervisor process/forking logic is
# defined, we can't really test the supervisor part of the workflow;
# we can only test the job executor.
mm = self.mm
with mm:
mm['get_job'].return_value = self.job
mm['get_calc'].return_value = self.calc_class
mm['fork'].return_value = 0
engine.run_calc(self.job, 'debug', 'oq.log', ['geojson'], 'hazard',
supervised=True)
# Check the intermediate function calls and the flow of data:
self.assertEqual(1, mm['get_calc'].call_count)
self.assertEqual((('hazard', 'classical'), {}),
mm['get_calc'].call_args)
self.assertEqual(1, mm['job_stats'].call_count)
self.assertEqual(((self.job, ), {}), mm['job_stats'].call_args)
self.assertEqual(1, mm['job_exec'].call_count)
self.assertEqual(
((self.job, 'debug', ['geojson'], 'hazard',
self.calc_instance),
{}),
mm['job_exec'].call_args
)
def test_unsupervised(self):
mm = self.mm
with mm:
mm['get_job'].return_value = self.job
mm['get_calc'].return_value = self.calc_class
engine.run_calc(self.job, 'debug', 'oq.log', ['geojson'], 'hazard',
supervised=False)
# Check the intermediate function calls and the flow of data:
self.assertEqual(1, mm['get_calc'].call_count)
self.assertEqual((('hazard', 'classical'), {}),
mm['get_calc'].call_args)
self.assertEqual(1, mm['job_stats'].call_count)
self.assertEqual(((self.job, ), {}), mm['job_stats'].call_args)
self.assertEqual(1, mm['job_exec'].call_count)
self.assertEqual(
((self.job, 'debug', ['geojson'], 'hazard', self.calc_instance),
{}),
mm['job_exec'].call_args
)
self.assertEqual(1, mm['cleanup'].call_count)
self.assertEqual(((1984, ), {}), mm['cleanup'].call_args)
self.assertEqual(1, mm['get_job'].call_count)
self.assertEqual(((1984, ), {}), mm['get_job'].call_args)
def test_unsupervised(self):
mm = self.mm
with mm:
mm['get_calc'].return_value = self.calc_class
engine.run_calc(self.job, 'debug', 'oq.log', ['geojson'], 'hazard')
self.assertEqual(1, mm['save_job_stats'].call_count)
# Check the intermediate function calls and the flow of data:
self.assertEqual(1, mm['get_calc'].call_count)
self.assertEqual((('hazard', 'classical'), {}),
mm['get_calc'].call_args)
self.assertEqual(1, mm['job_stats'].call_count)
self.assertEqual(((self.job, ), {}), mm['job_stats'].call_args)
self.assertEqual(1, mm['job_exec'].call_count)
self.assertEqual(
((self.job, 'debug', ['geojson'], 'hazard', self.calc_instance),
{}),
mm['job_exec'].call_args
)
self.assertEqual(1, mm['cleanup'].call_count)
self.assertEqual(((self.job, ), {'terminate': engine.TERMINATE}),
mm['cleanup'].call_args)
def run_job(cfg, exports=None, hazard_calculation_id=None,
hazard_output_id=None, **params):
"""
Given the path to a job config file and a hazard_calculation_id
or a output, run the job.
"""
if exports is None:
exports = []
job = get_job(cfg, hazard_calculation_id=hazard_calculation_id,
hazard_output_id=hazard_output_id)
job.is_running = True
job.save()
logfile = os.path.join(tempfile.gettempdir(), 'qatest.log')
job_type = 'risk' if (
hazard_calculation_id or hazard_output_id) else 'hazard'
# update calculation parameters
for name, value in params.iteritems():
setattr(job.calculation, name, value)
job.calculation.save()
engine.run_calc(job, 'error', logfile, exports, job_type)
return job
def test_supervised(self):
# Due to the way the executor/supervisor process/forking logic is
# defined, we can't really test the supervisor part of the workflow;
# we can only test the job executor.
mm = self.mm
with mm:
mm['get_job'].return_value = self.job
mm['get_calc'].return_value = self.calc_class
mm['fork'].return_value = 0
engine.run_calc(self.job, 'debug', 'oq.log', ['geojson'], 'hazard',
supervised=True)
# Check the intermediate function calls and the flow of data:
self.assertEqual(1, mm['get_calc'].call_count)
self.assertEqual((('hazard', 'classical'), {}),
mm['get_calc'].call_args)
self.assertEqual(1, mm['job_stats'].call_count)
self.assertEqual(((self.job, ), {}), mm['job_stats'].call_args)
self.assertEqual(1, mm['job_exec'].call_count)
self.assertEqual(
((self.job, 'debug', ['geojson'], 'hazard',
self.calc_instance),
{}),
mm['job_exec'].call_args
)
def run_job(job_ini,
log_level='info',
log_file=None,
exports='',
username=getpass.getuser(),
**kw):
"""
Run a job using the specified config file and other options.
:param str job_ini:
Path to calculation config (INI-style) files.
:param str log_level:
'debug', 'info', 'warn', 'error', or 'critical'
:param str log_file:
Path to log file.
:param exports:
A comma-separated string of export types requested by the user.
:param username:
Name of the user running the job
:param kw:
Extra parameters like hazard_calculation_id and calculation_mode
"""
job_id = logs.init('job', getattr(logging, log_level.upper()))
with logs.handle(job_id, log_level, log_file):
job_ini = os.path.abspath(job_ini)
oqparam = eng.job_from_file(job_ini, job_id, username, **kw)
kw['username'] = username
eng.run_calc(job_id, oqparam, exports, **kw)
for line in logs.dbcmd('list_outputs', job_id, False):
safeprint(line)
return job_id
def run_job(job_ini, log_level='info', log_file=None, exports='',
username=getpass.getuser(), **kw):
"""
Run a job using the specified config file and other options.
:param str job_ini:
Path to calculation config (INI-style) files.
:param str log_level:
'debug', 'info', 'warn', 'error', or 'critical'
:param str log_file:
Path to log file.
:param exports:
A comma-separated string of export types requested by the user.
:param username:
Name of the user running the job
:param kw:
Extra parameters like hazard_calculation_id and calculation_mode
"""
job_id = logs.init('job', getattr(logging, log_level.upper()))
with logs.handle(job_id, log_level, log_file):
job_ini = os.path.abspath(job_ini)
oqparam = eng.job_from_file(job_ini, job_id, username, **kw)
kw['username'] = username
eng.run_calc(job_id, oqparam, exports, **kw)
for line in logs.dbcmd('list_outputs', job_id, False):
safeprint(line)
return job_id
def run_job(cfg_file, log_level='info', log_file=None, exports='',
hazard_calculation_id=None, username=getpass.getuser(), **kw):
"""
Run a job using the specified config file and other options.
:param str cfg_file:
Path to calculation config (INI-style) files.
:param str log_level:
'debug', 'info', 'warn', 'error', or 'critical'
:param str log_file:
Path to log file.
:param exports:
A comma-separated string of export types requested by the user.
:param hazard_calculation_id:
ID of the previous calculation or None
:param username:
Name of the user running the job
"""
# if the master dies, automatically kill the workers
job_ini = os.path.abspath(cfg_file)
job_id, oqparam = eng.job_from_file(
job_ini, username, hazard_calculation_id)
kw['username'] = username
eng.run_calc(job_id, oqparam, log_level, log_file, exports,
hazard_calculation_id=hazard_calculation_id, **kw)
for line in logs.dbcmd('list_outputs', job_id, False):
safeprint(line)
return job_id
def test(self):
cfg = helpers.get_data_path('event_based_hazard/job.ini')
job_id, oq = actions.job_from_file(cfg, 'test_user')
with tempfile.NamedTemporaryFile() as temp:
with self.assertRaises(ZeroDivisionError), mock.patch(
'openquake.engine.engine._do_run_calc', lambda *args: 1/0):
engine.run_calc(job_id, oq, 'info', temp.name, exports=[])
logged = open(temp.name).read()
# make sure the real error has been logged
self.assertIn('integer division or modulo by zero', logged)
def test(self):
cfg = helpers.get_data_path('event_based_hazard/job.ini')
job_id, oq = actions.job_from_file(cfg, 'test_user')
with tempfile.NamedTemporaryFile() as temp:
with self.assertRaises(ZeroDivisionError), mock.patch(
'openquake.engine.engine._do_run_calc',
lambda *args: 1 / 0):
engine.run_calc(job_id, oq, 'info', temp.name, exports=[])
logged = open(temp.name).read()
# make sure the real error has been logged
self.assertIn('integer division or modulo by zero', logged)
def run_calc(job_type, calc_id, calc_dir,
callback_url=None, foreign_calc_id=None,
dbname="platform", log_file=None):
"""
Run a calculation given the calculation ID. It is assumed that the
entire calculation profile is already loaded into the oq-engine database
and is ready to execute. This function never fails; errors are trapped
but not logged since the engine already logs them.
:param job_type:
'hazard' or 'risk'
:param calc_id:
the calculation id on the engine
:param calc_dir:
the directory with the input files
:param callback_url:
the URL to call at the end of the calculation
:param foreign_calc_id:
the calculation id on the platform
:param dbname:
the platform database name
"""
if job_type == 'hazard':
job = oqe_models.OqJob.objects.get(hazard_calculation=calc_id)
else:
job = oqe_models.OqJob.objects.get(risk_calculation=calc_id)
update_calculation(callback_url, status="started", engine_id=calc_id)
exports = []
progress_handler = ProgressHandler(callback_url, job.calculation)
logging.root.addHandler(progress_handler)
try:
engine.run_calc(job, DEFAULT_LOG_LEVEL, log_file, exports, job_type)
except: # catch the errors before task spawning
# do not log the errors, since the engine already does that
exctype, exc, tb = sys.exc_info()
einfo = ''.join(traceback.format_tb(tb))
einfo += '%s: %s' % (exctype.__name__, exc)
update_calculation(callback_url, status="failed", einfo=einfo)
return
finally:
logging.root.removeHandler(progress_handler)
shutil.rmtree(calc_dir)
# If requested to, signal job completion and trigger a migration of
# results.
if not None in (callback_url, foreign_calc_id):
_trigger_migration(job, callback_url, foreign_calc_id, dbname)
def run_job(cfg_file, log_level, log_file, exports='',
hazard_calculation_id=None):
"""
Run a job using the specified config file and other options.
:param str cfg_file:
Path to calculation config (INI-style) files.
:param str log_level:
'debug', 'info', 'warn', 'error', or 'critical'
:param str log_file:
Path to log file.
:param exports:
A comma-separated string of export types requested by the user.
:param hazard_calculation_id:
ID of the previous calculation or None
"""
# if the master dies, automatically kill the workers
concurrent_futures_process_monkeypatch()
job_ini = os.path.abspath(cfg_file)
job_id, oqparam = eng.job_from_file(
job_ini, getpass.getuser(), hazard_calculation_id)
calc = eng.run_calc(job_id, oqparam, log_level, log_file, exports,
hazard_calculation_id=hazard_calculation_id)
calc.monitor.flush()
for line in logs.dbcmd('list_outputs', job_id, False):
print(line)
return job_id
def run_calc(job_id,
oqparam,
calc_dir,
log_file=None,
hazard_calculation_id=None):
"""
Run a calculation given the calculation ID. It is assumed that the
entire calculation profile is already loaded into the oq-engine database
and is ready to execute. This function never fails; errors are trapped
but not logged since the engine already logs them.
:param job_id:
the job ID
:param calc_dir:
the directory with the input files
:param log_file:
the name of the log file
:param hazard_calculation_id:
the previous calculation, if any
"""
try:
calc = engine.run_calc(job_id, oqparam, DEFAULT_LOG_LEVEL, log_file,
'', hazard_calculation_id)
except: # catch the errors before task spawning
# do not log the errors, since the engine already does that
exctype, exc, tb = sys.exc_info()
einfo = ''.join(traceback.format_tb(tb))
einfo += '%s: %s' % (exctype.__name__, exc)
raise
calc.datastore.close()
shutil.rmtree(calc_dir)
def run_calc(
job_id, oqparam, calc_dir, log_file=None, hazard_calculation_id=None):
"""
Run a calculation given the calculation ID. It is assumed that the
entire calculation profile is already loaded into the oq-engine database
and is ready to execute. This function never fails; errors are trapped
but not logged since the engine already logs them.
:param job_id:
the job ID
:param calc_dir:
the directory with the input files
:param log_file:
the name of the log file
:param hazard_calculation_id:
the previous calculation, if any
"""
try:
calc = engine.run_calc(job_id, oqparam, DEFAULT_LOG_LEVEL, log_file,
'', hazard_calculation_id)
except: # catch the errors before task spawning
# do not log the errors, since the engine already does that
exctype, exc, tb = sys.exc_info()
einfo = ''.join(traceback.format_tb(tb))
einfo += '%s: %s' % (exctype.__name__, exc)
raise
calc.datastore.close()
shutil.rmtree(calc_dir)
def test(self):
cfg = helpers.get_data_path('event_based_hazard/job.ini')
job = engine.job_from_file(cfg, 'test_user')
with tempfile.NamedTemporaryFile() as temp:
with self.assertRaises(ZeroDivisionError), mock.patch(
'openquake.engine.engine._do_run_calc', lambda *args: 1/0
), mock.patch('openquake.engine.engine.cleanup_after_job',
lambda job: None):
engine.run_calc(job, 'info', temp.name, exports=[])
logged = open(temp.name).read()
# make sure the real error has been logged
self.assertIn('integer division or modulo by zero', logged)
# also check the spurious cleanup error
self.assertIn('TypeError: <lambda>() got an unexpected keyword '
"argument 'terminate'", logged)
def run_calc(job_id,
calc_dir,
callback_url=None,
foreign_calc_id=None,
dbname="platform",
log_file=None):
"""
Run a calculation given the calculation ID. It is assumed that the
entire calculation profile is already loaded into the oq-engine database
and is ready to execute. This function never fails; errors are trapped
but not logged since the engine already logs them.
:param job_id:
the ID of the job on the engine
:param calc_dir:
the directory with the input files
:param callback_url:
the URL to call at the end of the calculation
:param foreign_calc_id:
the calculation id on the platform
:param dbname:
the platform database name
"""
job = oqe_models.OqJob.objects.get(pk=job_id)
update_calculation(callback_url, status="started", engine_id=job_id)
progress_handler = ProgressHandler(callback_url, job)
logging.root.addHandler(progress_handler)
try:
engine.run_calc(job, DEFAULT_LOG_LEVEL, log_file, 'xml,geojson,csv')
except: # catch the errors before task spawning
# do not log the errors, since the engine already does that
exctype, exc, tb = sys.exc_info()
einfo = ''.join(traceback.format_tb(tb))
einfo += '%s: %s' % (exctype.__name__, exc)
update_calculation(callback_url, status="failed", einfo=einfo)
raise
finally:
logging.root.removeHandler(progress_handler)
shutil.rmtree(calc_dir)
# If requested to, signal job completion and trigger a migration of
# results.
if not None in (callback_url, foreign_calc_id):
_trigger_migration(job, callback_url, foreign_calc_id, dbname)
def run_job(cfg, exports=None, hazard_calculation_id=None,
hazard_output_id=None, **params):
"""
Given the path to a job config file and a hazard_calculation_id
or a output, run the job.
"""
if exports is None:
exports = []
job = get_job(cfg, hazard_calculation_id=hazard_calculation_id,
hazard_output_id=hazard_output_id, **params)
job.is_running = True
job.save()
logfile = os.path.join(tempfile.gettempdir(), 'qatest.log')
engine.run_calc(job, 'error', logfile, exports, job.job_type)
return job
def run_risk_calc(calc_id, migration_callback_url=None, owner_user=None,
results_url=None):
"""
Run a risk calculation given the calculation ID. It is assumed that the
entire calculation profile is already loaded into the oq-engine database
and is ready to execute.
"""
job = oqe_models.OqJob.objects.get(risk_calculation=calc_id)
exports = []
# TODO: Log to file somewhere. But where?
log_file = None
# NOTE: Supervision MUST be turned off, or else the celeryd cluster
# handling this task will leak processes!!!
engine.run_calc(job, DEFAULT_LOG_LEVEL, log_file, exports, 'risk',
supervised=False)
# If requested to, signal job completion and trigger a migration of
# results.
if not None in (migration_callback_url, owner_user, results_url):
_trigger_migration(migration_callback_url, owner_user, results_url)
def run_job(cfg, exports='xml,csv', hazard_calculation_id=None, **params):
"""
Given the path to a job config file and a hazard_calculation_id
or a output, run the job.
:returns: a calculator object
"""
job_id, oqparam = actions.job_from_file(
cfg, 'openquake', 'error', [], hazard_calculation_id, **params)
logfile = os.path.join(tempfile.gettempdir(), 'qatest.log')
return engine.run_calc(job_id, oqparam, 'error', logfile, exports)
def run_job(cfg, exports='xml,csv', hazard_calculation_id=None, **params):
"""
Given the path to a job config file and a hazard_calculation_id
or a output, run the job.
:returns: a calculator object
"""
job_id, oqparam = engine.job_from_file(cfg, 'openquake', 'error', [],
hazard_calculation_id, **params)
logfile = os.path.join(tempfile.gettempdir(), 'qatest.log')
return engine.run_calc(job_id, oqparam, 'error', logfile, exports)
def run_job(cfg, exports='xml,csv', hazard_calculation_id=None,
hazard_output_id=None, **params):
"""
Given the path to a job config file and a hazard_calculation_id
or a output, run the job.
:returns: a calculator object
"""
job = get_job(cfg, hazard_calculation_id=hazard_calculation_id,
hazard_output_id=hazard_output_id, **params)
job.is_running = True
job.save()
logfile = os.path.join(tempfile.gettempdir(), 'qatest.log')
return engine.run_calc(job, 'error', logfile, exports)
def run_job(cfg, exports='xml,csv', hazard_calculation_id=None,
hazard_output_id=None, **params):
"""
Given the path to a job config file and a hazard_calculation_id
or a output, run the job.
:returns: a calculator object
"""
job = get_job(cfg, hazard_calculation_id=hazard_calculation_id,
hazard_output_id=hazard_output_id, **params)
job.is_running = True
job.save()
logfile = os.path.join(tempfile.gettempdir(), 'qatest.log')
return engine.run_calc(job, 'error', logfile, exports)
def run_calc(job_id, calc_dir,
callback_url=None, foreign_calc_id=None,
dbname="platform", log_file=None):
"""
Run a calculation given the calculation ID. It is assumed that the
entire calculation profile is already loaded into the oq-engine database
and is ready to execute. This function never fails; errors are trapped
but not logged since the engine already logs them.
:param job_id:
the ID of the job on the engine
:param calc_dir:
the directory with the input files
:param callback_url:
the URL to call at the end of the calculation
:param foreign_calc_id:
the calculation id on the platform
:param dbname:
the platform database name
"""
job = oqe_models.OqJob.objects.get(pk=job_id)
update_calculation(callback_url, status="started", engine_id=job_id)
progress_handler = ProgressHandler(callback_url, job)
logging.root.addHandler(progress_handler)
try:
calc = engine.run_calc(job, DEFAULT_LOG_LEVEL, log_file, exports='')
except: # catch the errors before task spawning
# do not log the errors, since the engine already does that
exctype, exc, tb = sys.exc_info()
einfo = ''.join(traceback.format_tb(tb))
einfo += '%s: %s' % (exctype.__name__, exc)
update_calculation(callback_url, status="failed", einfo=einfo)
raise
finally:
logging.root.removeHandler(progress_handler)
if hasattr(calc, 'datastore'):
calc.datastore.close()
shutil.rmtree(calc_dir)
def run_jobs(job_inis, log_level='info', log_file=None, exports='',
username=getpass.getuser(), **kw):
"""
Run jobs using the specified config file and other options.
:param str job_inis:
A list of paths to .ini files.
:param str log_level:
'debug', 'info', 'warn', 'error', or 'critical'
:param str log_file:
Path to log file.
:param exports:
A comma-separated string of export types requested by the user.
:param username:
Name of the user running the job
:param kw:
Extra parameters like hazard_calculation_id and calculation_mode
"""
dist = parallel.oq_distribute()
jobparams = []
for job_ini in job_inis:
# NB: the logs must be initialized BEFORE everything
job_id = logs.init('job', getattr(logging, log_level.upper()))
with logs.handle(job_id, log_level, log_file):
oqparam = eng.job_from_file(os.path.abspath(job_ini), job_id,
username, **kw)
if (not jobparams and 'csm_cache' not in kw
and 'hazard_calculation_id' not in kw):
kw['hazard_calculation_id'] = job_id
jobparams.append((job_id, oqparam))
jobarray = len(jobparams) > 1 and 'csm_cache' in kw
try:
eng.poll_queue(job_id, poll_time=15)
# wait for an empty slot or a CTRL-C
except BaseException:
# the job aborted even before starting
for job_id, oqparam in jobparams:
logs.dbcmd('finish', job_id, 'aborted')
return jobparams
else:
for job_id, oqparam in jobparams:
dic = {'status': 'executing', 'pid': eng._PID}
if jobarray:
dic['hazard_calculation_id'] = jobparams[0][0]
logs.dbcmd('update_job', job_id, dic)
try:
if dist == 'zmq' and config.zworkers['host_cores']:
logging.info('Asking the DbServer to start the workers')
logs.dbcmd('zmq_start') # start the zworkers
logs.dbcmd('zmq_wait') # wait for them to go up
allargs = [(job_id, oqparam, exports, log_level, log_file)
for job_id, oqparam in jobparams]
if jobarray:
with start_many(eng.run_calc, allargs):
pass
else:
for args in allargs:
eng.run_calc(*args)
finally:
if dist == 'zmq' and config.zworkers['host_cores']:
logging.info('Stopping the zworkers')
logs.dbcmd('zmq_stop')
elif dist.startswith('celery'):
eng.celery_cleanup(config.distribution.terminate_workers_on_revoke)
return jobparams