def test(self):
# check that if risk models are provided, then the ``points to
# compute`` and the imls are got from there
username = helpers.default_user()
job = engine.prepare_job(username)
cfg = helpers.get_data_path('classical_job-sd-imt.ini')
params = engine.parse_config(open(cfg, 'r'))
haz_calc = engine.create_calculation(models.HazardCalculation, params)
haz_calc = models.HazardCalculation.objects.get(id=haz_calc.id)
job.hazard_calculation = haz_calc
job.is_running = True
job.save()
base_path = ('openquake.engine.calculators.hazard.classical.core'
'.ClassicalHazardCalculator')
init_src_patch = helpers.patch('%s.%s' %
(base_path, 'initialize_sources'))
init_sm_patch = helpers.patch('%s.%s' %
(base_path, 'initialize_site_model'))
init_rlz_patch = helpers.patch('%s.%s' %
(base_path, 'initialize_realizations'))
record_stats_patch = helpers.patch('%s.%s' %
(base_path, 'record_init_stats'))
init_pr_data_patch = helpers.patch('%s.%s' %
(base_path, 'initialize_pr_data'))
patches = (init_src_patch, init_sm_patch, init_rlz_patch,
record_stats_patch, init_pr_data_patch)
mocks = [p.start() for p in patches]
get_calculator_class(
'hazard',
job.hazard_calculation.calculation_mode)(job).pre_execute()
self.assertEqual([(1.0, -1.0), (0.0, 0.0)],
[(point.latitude, point.longitude)
for point in haz_calc.points_to_compute()])
self.assertEqual(['PGA'], haz_calc.get_imts())
self.assertEqual(3,
haz_calc.oqjob.exposuremodel.exposuredata_set.count())
for i, m in enumerate(mocks):
m.stop()
patches[i].stop()
return job
def test(self):
# check that if risk models are provided, then the ``points to
# compute`` and the imls are got from there
username = helpers.default_user().user_name
job = engine.prepare_job(username)
cfg = helpers.get_data_path('classical_job-sd-imt.ini')
params, files = engine.parse_config(open(cfg, 'r'))
haz_calc = engine.create_hazard_calculation(
job.owner.user_name, params, files)
haz_calc = models.HazardCalculation.objects.get(id=haz_calc.id)
job.hazard_calculation = haz_calc
job.is_running = True
job.save()
base_path = ('openquake.engine.calculators.hazard.classical.core'
'.ClassicalHazardCalculator')
init_src_patch = helpers.patch(
'%s.%s' % (base_path, 'initialize_sources'))
init_sm_patch = helpers.patch(
'%s.%s' % (base_path, 'initialize_site_model'))
init_rlz_patch = helpers.patch(
'%s.%s' % (base_path, 'initialize_realizations'))
record_stats_patch = helpers.patch(
'%s.%s' % (base_path, 'record_init_stats'))
init_pr_data_patch = helpers.patch(
'%s.%s' % (base_path, 'initialize_pr_data'))
patches = (init_src_patch, init_sm_patch, init_rlz_patch,
record_stats_patch, init_pr_data_patch)
mocks = [p.start() for p in patches]
get_calculator_class(
'hazard',
job.hazard_calculation.calculation_mode)(job).pre_execute()
self.assertEqual([(1.0, -1.0), (0.0, 0.0)],
[(point.latitude, point.longitude)
for point in haz_calc.points_to_compute()])
self.assertEqual(['PGA'], haz_calc.get_imts())
self.assertEqual(3, haz_calc.exposure_model.exposuredata_set.count())
for i, m in enumerate(mocks):
m.stop()
patches[i].stop()
return job
def run_hazard_job(cfg, exports=None):
"""
Given the path to job config file, run the job and assert that it was
successful. If this assertion passes, return the completed job.
:param str cfg:
Path to a job config file.
:param list exports:
A list of export format types. Currently only 'xml' is supported.
:returns:
The completed :class:`~openquake.engine.db.models.OqJob`.
"""
if exports is None:
exports = []
job = get_hazard_job(cfg)
job.is_running = True
job.save()
models.JobStats.objects.create(oq_job=job)
calc_mode = job.hazard_calculation.calculation_mode
calc = get_calculator_class('hazard', calc_mode)(job)
completed_job = engine2._do_run_calc(job, exports, calc, 'hazard')
job.is_running = False
job.save()
return completed_job
def test(self):
# check that if risk models are provided, then the ``points to
# compute`` and the imls are got from there
username = helpers.default_user()
job = engine.prepare_job(username)
cfg = helpers.get_data_path("classical_job-sd-imt.ini")
params = vars(readini.parse_config(open(cfg)))
del params["hazard_calculation_id"]
del params["hazard_output_id"]
haz_calc = engine.create_calculation(models.HazardCalculation, params)
haz_calc = models.HazardCalculation.objects.get(id=haz_calc.id)
job.hazard_calculation = haz_calc
job.is_running = True
job.save()
calc = get_calculator_class("hazard", job.hazard_calculation.calculation_mode)(job)
calc.parse_risk_models()
self.assertEqual(
[(1.0, -1.0), (0.0, 0.0)], [(point.latitude, point.longitude) for point in haz_calc.points_to_compute()]
)
self.assertEqual(["PGA"], haz_calc.get_imts())
self.assertEqual(3, haz_calc.oqjob.exposuremodel.exposuredata_set.count())
return job
def run_calc(job, log_level, log_file, exports, job_type):
"""
Run a calculation.
:param job:
:class:`openquake.engine.db.model.OqJob` instance which references a
valid :class:`openquake.engine.db.models.RiskCalculation` or
:class:`openquake.engine.db.models.HazardCalculation`.
:param str log_level:
The desired logging level. Valid choices are 'debug', 'info',
'progress', 'warn', 'error', and 'critical'.
:param str log_file:
Complete path (including file name) to file where logs will be written.
If `None`, logging will just be printed to standard output.
:param list exports:
A (potentially empty) list of export targets. Currently only "xml" is
supported.
:param str job_type:
'hazard' or 'risk'
"""
calc_mode = getattr(job, '%s_calculation' % job_type).calculation_mode
calculator = get_calculator_class(job_type, calc_mode)(job)
calc = job.calculation
# initialize log handlers
handler = (LogFileHandler(job_type, calc, log_file) if log_file
else LogStreamHandler(job_type, calc))
logging.root.addHandler(handler)
try:
with job_stats(job): # run the job
logs.set_level(log_level)
_do_run_calc(job, exports, calculator, job_type)
finally:
logging.root.removeHandler(handler)
return job
def run_hazard_job(cfg, exports=None):
"""
Given the path to job config file, run the job and assert that it was
successful. If this assertion passes, return the completed job.
:param str cfg:
Path to a job config file.
:param list exports:
A list of export format types. Currently only 'xml' is supported.
:returns:
The completed :class:`~openquake.engine.db.models.OqJob`.
"""
if exports is None:
exports = []
job = get_hazard_job(cfg)
job.is_running = True
job.save()
models.JobStats.objects.create(oq_job=job)
hc = job.hazard_calculation
calc = get_calculator_class('hazard', hc.calculation_mode)(job)
try:
logs.init_logs_amqp_send(
level='ERROR', calc_domain='hazard', calc_id=hc.id)
engine._do_run_calc(job, exports, calc, 'hazard')
finally:
job.is_running = False
job.calc = calc
job.save()
return job
def run_risk_job(cfg, exports=None, hazard_calculation_id=None,
hazard_output_id=None):
"""
Given the path to a risk job config file and a hazard_calculation_id
or a output, run the job.
"""
if exports is None:
exports = []
# You can't specify both a hazard output and hazard calculation
# Pick one
assert not (hazard_calculation_id is not None
and hazard_output_id is not None)
job = get_risk_job(cfg, hazard_calculation_id=hazard_calculation_id,
hazard_output_id=hazard_output_id)
job.is_running = True
job.save()
models.JobStats.objects.create(oq_job=job)
rc = job.risk_calculation
calc = get_calculator_class('risk', rc.calculation_mode)(job)
logs.init_logs_amqp_send(level='ERROR', calc_domain='risk', calc_id=rc.id)
completed_job = engine._do_run_calc(job, exports, calc, 'risk')
job.is_running = False
job.save()
return completed_job
def setUpClass(cls):
cfg = helpers.get_data_path(
'calculators/hazard/classical/haz_map_test_job.ini')
job = helpers.get_hazard_job(cfg)
hc = job.hazard_calculation
cls.calc = get_calculator_class('hazard', hc.calculation_mode)(job)
cls.calc.initialize_site_model()
assert len(hc.site_collection) == 2, len(hc.site_collection)
def setUpClass(cls):
cfg = helpers.get_data_path("calculators/hazard/classical/haz_map_test_job.ini")
job = helpers.get_job(cfg)
models.JobStats.objects.create(oq_job=job)
hc = job.hazard_calculation
cls.calc = get_calculator_class("hazard", hc.calculation_mode)(job)
cls.calc.initialize_site_model()
assert len(hc.site_collection) == 2, len(hc.site_collection)
def setUpClass(cls):
cfg = helpers.get_data_path(
'calculators/hazard/classical/haz_map_test_job.ini')
job = helpers.get_job(cfg)
models.JobStats.objects.create(oq_job=job)
hc = job.get_oqparam()
cls.calc = get_calculator_class('hazard', hc.calculation_mode)(job)
cls.calc.initialize_site_collection()
num_sites = len(cls.calc.site_collection)
assert num_sites == 2, num_sites
def run_calc(job, log_level, log_file, exports, job_type,
supervised=True, progress_handler=None):
"""
Run a calculation.
:param job:
:class:`openquake.engine.db.model.OqJob` instance which references a
valid :class:`openquake.engine.db.models.RiskCalculation` or
:class:`openquake.engine.db.models.HazardCalculation`.
:param str log_level:
The desired logging level. Valid choices are 'debug', 'info',
'progress', 'warn', 'error', and 'critical'.
:param str log_file:
Complete path (including file name) to file where logs will be written.
If `None`, logging will just be printed to standard output.
:param list exports:
A (potentially empty) list of export targets. Currently only "xml" is
supported.
:param str job_type:
'hazard' or 'risk'
:param bool supervised:
Defaults to `True`. If `True`, run OpenQuake with a supervisor process,
which monitors the job executor process and collects log messages.
:param callable progress_handler:
a callback getting the progress of the calculation and the calculation
object
"""
calc_mode = getattr(job, '%s_calculation' % job_type).calculation_mode
calc = get_calculator_class(job_type, calc_mode)(job)
if progress_handler is not None:
calc.register_progress_handler(progress_handler)
# Create job stats, which implicitly records the start time for the job
_create_job_stats(job)
# Closing all db connections to make sure they're not shared between
# supervisor and job executor processes.
# Otherwise, if one of them closes the connection it immediately
# becomes unavailable for others.
if supervised:
django_db.close_connection()
job_pid = os.fork()
if not job_pid:
# calculation executor process
try:
with job_stats(job):
_job_exec(job, log_level, exports, job_type, calc)
except Exception, ex:
logs.LOG.critical("Calculation failed with exception: '%s'"
% str(ex))
raise
finally:
def test_check_limits_event_based(self):
# this is a based on a demo with 2 realizations, 5 ses,
# 2 imt and 121 sites
cfg = helpers.get_data_path("event_based_hazard/job.ini")
job = helpers.get_job(cfg)
models.JobStats.objects.create(oq_job=job)
hc = job.hazard_calculation
calc = get_calculator_class("hazard", hc.calculation_mode)(job)
input_weight, output_weight = calc.pre_execute()
self.assertEqual(input_weight, 1352.75)
self.assertAlmostEqual(output_weight, 12.1)
def run_calc(job, log_level, log_file, exports, job_type):
"""
Run a calculation.
:param job:
:class:`openquake.engine.db.model.OqJob` instance which references a
valid :class:`openquake.engine.db.models.RiskCalculation` or
:class:`openquake.engine.db.models.HazardCalculation`.
:param str log_level:
The desired logging level. Valid choices are 'debug', 'info',
'progress', 'warn', 'error', and 'critical'.
:param str log_file:
Complete path (including file name) to file where logs will be written.
If `None`, logging will just be printed to standard output.
:param list exports:
A (potentially empty) list of export targets. Currently only "xml" is
supported.
:param calc:
Calculator object, which must implement the interface of
:class:`openquake.engine.calculators.base.Calculator`.
:param str job_type:
'hazard' or 'risk'
"""
calc_mode = getattr(job, '%s_calculation' % job_type).calculation_mode
calc = get_calculator_class(job_type, calc_mode)(job)
# Create job stats, which implicitly records the start time for the job
models.JobStats.objects.create(oq_job=job)
# Closing all db connections to make sure they're not shared between
# supervisor and job executor processes.
# Otherwise, if one of them closes the connection it immediately becomes
# unavailable for others.
close_connection()
job_pid = os.fork()
if not job_pid:
# calculation executor process
try:
logs.init_logs_amqp_send(level=log_level, calc_domain=job_type,
calc_id=job.calculation.id)
# run the job
job.is_running = True
job.save()
kvs.mark_job_as_current(job.id)
_do_run_calc(job, exports, calc, job_type)
except Exception, ex:
logs.LOG.critical("Calculation failed with exception: '%s'"
% str(ex))
raise
finally:
def validate(job, job_type, params, exports):
"""
Validate a job of type 'hazard' or 'risk' by instantiating its
form class with the given files and exports.
:param job:
an instance of :class:`openquake.engine.db.models.OqJob`
:param str job_type:
"hazard" or "risk"
:param dict params:
The raw dictionary of parameters parsed from the config file.
:param exports:
a list of export types
:returns:
an error message if the form is invalid, None otherwise.
"""
calculation = getattr(job, '%s_calculation' % job_type)
calc_mode = calculation.calculation_mode
calculator_cls = get_calculator_class(job_type, calc_mode)
formname = calculator_cls.__name__.replace('Calculator', 'Form')
try:
form_class = globals()[formname]
except KeyError:
return 'Could not find form class for "%s"' % calc_mode
files = set(params['inputs'])
form = form_class(instance=calculation, files=files, exports=exports)
# Check for superfluous params and raise warnings:
params_copy = params.copy()
# There are a couple of parameters we can ignore.
# `calculation_mode` is supplied in every config file, but is validated in
# a special way; therefore, we don't declare it on the forms.
# The `base_path` is extracted from the directory containing the config
# file; it's not a real param.
# `hazard_output_id` and `hazard_calculation_id` are supplied via command
# line args.
for p in ('calculation_mode', 'base_path', 'hazard_output_id',
'hazard_calculation_id'):
if p in params_copy:
params_copy.pop(p)
for param in set(params_copy.keys()).difference(set(form._meta.fields)):
msg = "Unknown parameter '%s' for calculation mode '%s'. Ignoring."
msg %= (param, calc_mode)
warnings.warn(msg, RuntimeWarning)
if not form.is_valid():
return 'Job configuration is not valid. Errors: %s' % dict(form.errors)
def validate(job, job_type, params, exports):
"""
Validate a job of type 'hazard' or 'risk' by instantiating its
form class with the given files and exports.
:param job:
an instance of :class:`openquake.engine.db.models.OqJob`
:param str job_type:
"hazard" or "risk"
:param dict params:
The raw dictionary of parameters parsed from the config file.
:param exports:
a list of export types
:returns:
an error message if the form is invalid, None otherwise.
"""
calculation = getattr(job, '%s_calculation' % job_type)
calc_mode = calculation.calculation_mode
calculator_cls = get_calculator_class(job_type, calc_mode)
formname = calculator_cls.__name__.replace('Calculator', 'Form')
try:
form_class = globals()[formname]
except KeyError:
return 'Could not find form class for "%s"' % calc_mode
files = set(params['inputs'])
form = form_class(instance=calculation, files=files, exports=exports)
# Check for superfluous params and raise warnings:
params_copy = params.copy()
# There are a couple of parameters we can ignore.
# `calculation_mode` is supplied in every config file, but is validated in
# a special way; therefore, we don't declare it on the forms.
# The `base_path` is extracted from the directory containing the config
# file; it's not a real param.
# `hazard_output_id` and `hazard_calculation_id` are supplied via command
# line args.
for p in ('calculation_mode', 'base_path', 'hazard_output_id',
'hazard_calculation_id'):
if p in params_copy:
params_copy.pop(p)
for param in set(params_copy.keys()).difference(set(form._meta.fields)):
msg = "Unknown parameter '%s' for calculation mode '%s'. Ignoring."
msg %= (param, calc_mode)
warnings.warn(msg, RuntimeWarning)
if not form.is_valid():
return 'Job configuration is not valid. Errors: %s' % dict(form.errors)
def run_calc(job, log_level, log_file, exports, job_type, supervised=True):
"""
Run a calculation.
:param job:
:class:`openquake.engine.db.model.OqJob` instance which references a
valid :class:`openquake.engine.db.models.RiskCalculation` or
:class:`openquake.engine.db.models.HazardCalculation`.
:param str log_level:
The desired logging level. Valid choices are 'debug', 'info',
'progress', 'warn', 'error', and 'critical'.
:param str log_file:
Complete path (including file name) to file where logs will be written.
If `None`, logging will just be printed to standard output.
:param list exports:
A (potentially empty) list of export targets. Currently only "xml" is
supported.
:param str job_type:
'hazard' or 'risk'
:param bool supervised:
Defaults to `True`. If `True`, run OpenQuake with a supervisor process,
which monitors the job executor process and collects log messages.
"""
calc_mode = getattr(job, '%s_calculation' % job_type).calculation_mode
calc = get_calculator_class(job_type, calc_mode)(job)
# Create job stats, which implicitly records the start time for the job
_create_job_stats(job)
# Closing all db connections to make sure they're not shared between
# supervisor and job executor processes.
# Otherwise, if one of them closes the connection it immediately becomes
# unavailable for others.
if supervised:
django_db.close_connection()
job_pid = os.fork()
if not job_pid:
# calculation executor process
try:
_job_exec(job, log_level, exports, job_type, calc)
except Exception, ex:
logs.LOG.critical("Calculation failed with exception: '%s'" %
str(ex))
raise
finally:
def test(self):
# check that if risk models are provided, then the sites
# and the imls are got from there
cfg = helpers.get_data_path('classical_job-sd-imt.ini')
job = engine.job_from_file(cfg, helpers.default_user())
job.is_running = True
job.save()
haz_calc = job.get_oqparam()
calc = get_calculator_class('hazard', haz_calc.calculation_mode)(job)
calc.parse_risk_models()
self.assertEqual(['PGA'],
list(calc.hc.intensity_measure_types_and_levels))
self.assertEqual(3, calc.job.exposuremodel.exposuredata_set.count())
return job
def test_check_limits_classical(self):
# this is a based on a demo with 3 realizations, 2 sites and 4 rlzs
cfg = helpers.get_data_path("calculators/hazard/classical/haz_map_test_job.ini")
job = helpers.get_job(cfg)
models.JobStats.objects.create(oq_job=job)
hc = job.hazard_calculation
calc = get_calculator_class("hazard", hc.calculation_mode)(job)
input_weight, output_weight = calc.pre_execute()
self.assertEqual(input_weight, 225)
self.assertEqual(output_weight, 24)
calc.max_input_weight = 1
with self.assertRaises(general.InputWeightLimit):
calc.check_limits(input_weight, output_weight)
calc.max_input_weight = 1000
calc.max_output_weight = 1
with self.assertRaises(general.OutputWeightLimit):
calc.check_limits(input_weight, output_weight)
def run_calc(job, log_level, log_file, exports, job_type,
progress_handler=None):
"""
Run a calculation.
:param job:
:class:`openquake.engine.db.model.OqJob` instance which references a
valid :class:`openquake.engine.db.models.RiskCalculation` or
:class:`openquake.engine.db.models.HazardCalculation`.
:param str log_level:
The desired logging level. Valid choices are 'debug', 'info',
'progress', 'warn', 'error', and 'critical'.
:param str log_file:
Complete path (including file name) to file where logs will be written.
If `None`, logging will just be printed to standard output.
:param list exports:
A (potentially empty) list of export targets. Currently only "xml" is
supported.
:param str job_type:
'hazard' or 'risk'
:param callable progress_handler:
a callback getting the progress of the calculation and the calculation
object
"""
calc_mode = getattr(job, '%s_calculation' % job_type).calculation_mode
calc = get_calculator_class(job_type, calc_mode)(job)
if progress_handler is not None:
calc.register_progress_handler(progress_handler)
# Create job stats, which implicitly records the start time for the job
_create_job_stats(job)
calc_id = job.calculation.id
calc_domain = 'hazard' if job.hazard_calculation else 'risk'
start_logging(calc_id, calc_domain, log_file)
try:
with job_stats(job):
_job_exec(job, log_level, exports, job_type, calc)
except Exception, ex:
logs.LOG.critical("Calculation failed with exception: '%s'", ex)
raise
def validate(job, job_type, files, exports):
"""
Validate a job of type 'hazard' or 'risk' by instantiating its
form class with the given files and exports.
:param job: an instance of :class:`openquake.engine.db.models.OqJob`
:param str job_type: "hazard" or "risk"
:param dict files: {fname: :class:`openquake.engine.db.models.Input` obj}
:param exports: a list of export types
:returns: an error message if the form is invalid, None otherwise.
"""
calculation = getattr(job, '%s_calculation' % job_type)
calc_mode = calculation.calculation_mode
calculator_cls = get_calculator_class(job_type, calc_mode)
formname = calculator_cls.__name__.replace('Calculator', 'Form')
try:
form_class = globals()[formname]
except KeyError:
return 'Could not find form class for "%s"' % calc_mode
form = form_class(instance=calculation, files=files, exports=exports)
if not form.is_valid():
return 'Job configuration is not valid. Errors: %s' % dict(form.errors)
