def test(self): # check that if risk models are provided, then the ``points to # compute`` and the imls are got from there username = helpers.default_user() job = engine.prepare_job(username) cfg = helpers.get_data_path('classical_job-sd-imt.ini') params = engine.parse_config(open(cfg, 'r')) haz_calc = engine.create_calculation(models.HazardCalculation, params) haz_calc = models.HazardCalculation.objects.get(id=haz_calc.id) job.hazard_calculation = haz_calc job.is_running = True job.save() base_path = ('openquake.engine.calculators.hazard.classical.core' '.ClassicalHazardCalculator') init_src_patch = helpers.patch('%s.%s' % (base_path, 'initialize_sources')) init_sm_patch = helpers.patch('%s.%s' % (base_path, 'initialize_site_model')) init_rlz_patch = helpers.patch('%s.%s' % (base_path, 'initialize_realizations')) record_stats_patch = helpers.patch('%s.%s' % (base_path, 'record_init_stats')) init_pr_data_patch = helpers.patch('%s.%s' % (base_path, 'initialize_pr_data')) patches = (init_src_patch, init_sm_patch, init_rlz_patch, record_stats_patch, init_pr_data_patch) mocks = [p.start() for p in patches] get_calculator_class( 'hazard', job.hazard_calculation.calculation_mode)(job).pre_execute() self.assertEqual([(1.0, -1.0), (0.0, 0.0)], [(point.latitude, point.longitude) for point in haz_calc.points_to_compute()]) self.assertEqual(['PGA'], haz_calc.get_imts()) self.assertEqual(3, haz_calc.oqjob.exposuremodel.exposuredata_set.count()) for i, m in enumerate(mocks): m.stop() patches[i].stop() return job
def test(self): # check that if risk models are provided, then the ``points to # compute`` and the imls are got from there username = helpers.default_user().user_name job = engine.prepare_job(username) cfg = helpers.get_data_path('classical_job-sd-imt.ini') params, files = engine.parse_config(open(cfg, 'r')) haz_calc = engine.create_hazard_calculation( job.owner.user_name, params, files) haz_calc = models.HazardCalculation.objects.get(id=haz_calc.id) job.hazard_calculation = haz_calc job.is_running = True job.save() base_path = ('openquake.engine.calculators.hazard.classical.core' '.ClassicalHazardCalculator') init_src_patch = helpers.patch( '%s.%s' % (base_path, 'initialize_sources')) init_sm_patch = helpers.patch( '%s.%s' % (base_path, 'initialize_site_model')) init_rlz_patch = helpers.patch( '%s.%s' % (base_path, 'initialize_realizations')) record_stats_patch = helpers.patch( '%s.%s' % (base_path, 'record_init_stats')) init_pr_data_patch = helpers.patch( '%s.%s' % (base_path, 'initialize_pr_data')) patches = (init_src_patch, init_sm_patch, init_rlz_patch, record_stats_patch, init_pr_data_patch) mocks = [p.start() for p in patches] get_calculator_class( 'hazard', job.hazard_calculation.calculation_mode)(job).pre_execute() self.assertEqual([(1.0, -1.0), (0.0, 0.0)], [(point.latitude, point.longitude) for point in haz_calc.points_to_compute()]) self.assertEqual(['PGA'], haz_calc.get_imts()) self.assertEqual(3, haz_calc.exposure_model.exposuredata_set.count()) for i, m in enumerate(mocks): m.stop() patches[i].stop() return job
def run_hazard_job(cfg, exports=None): """ Given the path to job config file, run the job and assert that it was successful. If this assertion passes, return the completed job. :param str cfg: Path to a job config file. :param list exports: A list of export format types. Currently only 'xml' is supported. :returns: The completed :class:`~openquake.engine.db.models.OqJob`. """ if exports is None: exports = [] job = get_hazard_job(cfg) job.is_running = True job.save() models.JobStats.objects.create(oq_job=job) calc_mode = job.hazard_calculation.calculation_mode calc = get_calculator_class('hazard', calc_mode)(job) completed_job = engine2._do_run_calc(job, exports, calc, 'hazard') job.is_running = False job.save() return completed_job
def test(self): # check that if risk models are provided, then the ``points to # compute`` and the imls are got from there username = helpers.default_user() job = engine.prepare_job(username) cfg = helpers.get_data_path("classical_job-sd-imt.ini") params = vars(readini.parse_config(open(cfg))) del params["hazard_calculation_id"] del params["hazard_output_id"] haz_calc = engine.create_calculation(models.HazardCalculation, params) haz_calc = models.HazardCalculation.objects.get(id=haz_calc.id) job.hazard_calculation = haz_calc job.is_running = True job.save() calc = get_calculator_class("hazard", job.hazard_calculation.calculation_mode)(job) calc.parse_risk_models() self.assertEqual( [(1.0, -1.0), (0.0, 0.0)], [(point.latitude, point.longitude) for point in haz_calc.points_to_compute()] ) self.assertEqual(["PGA"], haz_calc.get_imts()) self.assertEqual(3, haz_calc.oqjob.exposuremodel.exposuredata_set.count()) return job
def run_calc(job, log_level, log_file, exports, job_type): """ Run a calculation. :param job: :class:`openquake.engine.db.model.OqJob` instance which references a valid :class:`openquake.engine.db.models.RiskCalculation` or :class:`openquake.engine.db.models.HazardCalculation`. :param str log_level: The desired logging level. Valid choices are 'debug', 'info', 'progress', 'warn', 'error', and 'critical'. :param str log_file: Complete path (including file name) to file where logs will be written. If `None`, logging will just be printed to standard output. :param list exports: A (potentially empty) list of export targets. Currently only "xml" is supported. :param str job_type: 'hazard' or 'risk' """ calc_mode = getattr(job, '%s_calculation' % job_type).calculation_mode calculator = get_calculator_class(job_type, calc_mode)(job) calc = job.calculation # initialize log handlers handler = (LogFileHandler(job_type, calc, log_file) if log_file else LogStreamHandler(job_type, calc)) logging.root.addHandler(handler) try: with job_stats(job): # run the job logs.set_level(log_level) _do_run_calc(job, exports, calculator, job_type) finally: logging.root.removeHandler(handler) return job
def run_hazard_job(cfg, exports=None): """ Given the path to job config file, run the job and assert that it was successful. If this assertion passes, return the completed job. :param str cfg: Path to a job config file. :param list exports: A list of export format types. Currently only 'xml' is supported. :returns: The completed :class:`~openquake.engine.db.models.OqJob`. """ if exports is None: exports = [] job = get_hazard_job(cfg) job.is_running = True job.save() models.JobStats.objects.create(oq_job=job) hc = job.hazard_calculation calc = get_calculator_class('hazard', hc.calculation_mode)(job) try: logs.init_logs_amqp_send( level='ERROR', calc_domain='hazard', calc_id=hc.id) engine._do_run_calc(job, exports, calc, 'hazard') finally: job.is_running = False job.calc = calc job.save() return job
def run_risk_job(cfg, exports=None, hazard_calculation_id=None, hazard_output_id=None): """ Given the path to a risk job config file and a hazard_calculation_id or a output, run the job. """ if exports is None: exports = [] # You can't specify both a hazard output and hazard calculation # Pick one assert not (hazard_calculation_id is not None and hazard_output_id is not None) job = get_risk_job(cfg, hazard_calculation_id=hazard_calculation_id, hazard_output_id=hazard_output_id) job.is_running = True job.save() models.JobStats.objects.create(oq_job=job) rc = job.risk_calculation calc = get_calculator_class('risk', rc.calculation_mode)(job) logs.init_logs_amqp_send(level='ERROR', calc_domain='risk', calc_id=rc.id) completed_job = engine._do_run_calc(job, exports, calc, 'risk') job.is_running = False job.save() return completed_job
def setUpClass(cls): cfg = helpers.get_data_path( 'calculators/hazard/classical/haz_map_test_job.ini') job = helpers.get_hazard_job(cfg) hc = job.hazard_calculation cls.calc = get_calculator_class('hazard', hc.calculation_mode)(job) cls.calc.initialize_site_model() assert len(hc.site_collection) == 2, len(hc.site_collection)
def setUpClass(cls): cfg = helpers.get_data_path("calculators/hazard/classical/haz_map_test_job.ini") job = helpers.get_job(cfg) models.JobStats.objects.create(oq_job=job) hc = job.hazard_calculation cls.calc = get_calculator_class("hazard", hc.calculation_mode)(job) cls.calc.initialize_site_model() assert len(hc.site_collection) == 2, len(hc.site_collection)
def setUpClass(cls): cfg = helpers.get_data_path( 'calculators/hazard/classical/haz_map_test_job.ini') job = helpers.get_job(cfg) models.JobStats.objects.create(oq_job=job) hc = job.get_oqparam() cls.calc = get_calculator_class('hazard', hc.calculation_mode)(job) cls.calc.initialize_site_collection() num_sites = len(cls.calc.site_collection) assert num_sites == 2, num_sites
def run_calc(job, log_level, log_file, exports, job_type, supervised=True, progress_handler=None): """ Run a calculation. :param job: :class:`openquake.engine.db.model.OqJob` instance which references a valid :class:`openquake.engine.db.models.RiskCalculation` or :class:`openquake.engine.db.models.HazardCalculation`. :param str log_level: The desired logging level. Valid choices are 'debug', 'info', 'progress', 'warn', 'error', and 'critical'. :param str log_file: Complete path (including file name) to file where logs will be written. If `None`, logging will just be printed to standard output. :param list exports: A (potentially empty) list of export targets. Currently only "xml" is supported. :param str job_type: 'hazard' or 'risk' :param bool supervised: Defaults to `True`. If `True`, run OpenQuake with a supervisor process, which monitors the job executor process and collects log messages. :param callable progress_handler: a callback getting the progress of the calculation and the calculation object """ calc_mode = getattr(job, '%s_calculation' % job_type).calculation_mode calc = get_calculator_class(job_type, calc_mode)(job) if progress_handler is not None: calc.register_progress_handler(progress_handler) # Create job stats, which implicitly records the start time for the job _create_job_stats(job) # Closing all db connections to make sure they're not shared between # supervisor and job executor processes. # Otherwise, if one of them closes the connection it immediately # becomes unavailable for others. if supervised: django_db.close_connection() job_pid = os.fork() if not job_pid: # calculation executor process try: with job_stats(job): _job_exec(job, log_level, exports, job_type, calc) except Exception, ex: logs.LOG.critical("Calculation failed with exception: '%s'" % str(ex)) raise finally:
def test_check_limits_event_based(self): # this is a based on a demo with 2 realizations, 5 ses, # 2 imt and 121 sites cfg = helpers.get_data_path("event_based_hazard/job.ini") job = helpers.get_job(cfg) models.JobStats.objects.create(oq_job=job) hc = job.hazard_calculation calc = get_calculator_class("hazard", hc.calculation_mode)(job) input_weight, output_weight = calc.pre_execute() self.assertEqual(input_weight, 1352.75) self.assertAlmostEqual(output_weight, 12.1)
def run_calc(job, log_level, log_file, exports, job_type): """ Run a calculation. :param job: :class:`openquake.engine.db.model.OqJob` instance which references a valid :class:`openquake.engine.db.models.RiskCalculation` or :class:`openquake.engine.db.models.HazardCalculation`. :param str log_level: The desired logging level. Valid choices are 'debug', 'info', 'progress', 'warn', 'error', and 'critical'. :param str log_file: Complete path (including file name) to file where logs will be written. If `None`, logging will just be printed to standard output. :param list exports: A (potentially empty) list of export targets. Currently only "xml" is supported. :param calc: Calculator object, which must implement the interface of :class:`openquake.engine.calculators.base.Calculator`. :param str job_type: 'hazard' or 'risk' """ calc_mode = getattr(job, '%s_calculation' % job_type).calculation_mode calc = get_calculator_class(job_type, calc_mode)(job) # Create job stats, which implicitly records the start time for the job models.JobStats.objects.create(oq_job=job) # Closing all db connections to make sure they're not shared between # supervisor and job executor processes. # Otherwise, if one of them closes the connection it immediately becomes # unavailable for others. close_connection() job_pid = os.fork() if not job_pid: # calculation executor process try: logs.init_logs_amqp_send(level=log_level, calc_domain=job_type, calc_id=job.calculation.id) # run the job job.is_running = True job.save() kvs.mark_job_as_current(job.id) _do_run_calc(job, exports, calc, job_type) except Exception, ex: logs.LOG.critical("Calculation failed with exception: '%s'" % str(ex)) raise finally:
def validate(job, job_type, params, exports): """ Validate a job of type 'hazard' or 'risk' by instantiating its form class with the given files and exports. :param job: an instance of :class:`openquake.engine.db.models.OqJob` :param str job_type: "hazard" or "risk" :param dict params: The raw dictionary of parameters parsed from the config file. :param exports: a list of export types :returns: an error message if the form is invalid, None otherwise. """ calculation = getattr(job, '%s_calculation' % job_type) calc_mode = calculation.calculation_mode calculator_cls = get_calculator_class(job_type, calc_mode) formname = calculator_cls.__name__.replace('Calculator', 'Form') try: form_class = globals()[formname] except KeyError: return 'Could not find form class for "%s"' % calc_mode files = set(params['inputs']) form = form_class(instance=calculation, files=files, exports=exports) # Check for superfluous params and raise warnings: params_copy = params.copy() # There are a couple of parameters we can ignore. # `calculation_mode` is supplied in every config file, but is validated in # a special way; therefore, we don't declare it on the forms. # The `base_path` is extracted from the directory containing the config # file; it's not a real param. # `hazard_output_id` and `hazard_calculation_id` are supplied via command # line args. for p in ('calculation_mode', 'base_path', 'hazard_output_id', 'hazard_calculation_id'): if p in params_copy: params_copy.pop(p) for param in set(params_copy.keys()).difference(set(form._meta.fields)): msg = "Unknown parameter '%s' for calculation mode '%s'. Ignoring." msg %= (param, calc_mode) warnings.warn(msg, RuntimeWarning) if not form.is_valid(): return 'Job configuration is not valid. Errors: %s' % dict(form.errors)
def run_calc(job, log_level, log_file, exports, job_type, supervised=True): """ Run a calculation. :param job: :class:`openquake.engine.db.model.OqJob` instance which references a valid :class:`openquake.engine.db.models.RiskCalculation` or :class:`openquake.engine.db.models.HazardCalculation`. :param str log_level: The desired logging level. Valid choices are 'debug', 'info', 'progress', 'warn', 'error', and 'critical'. :param str log_file: Complete path (including file name) to file where logs will be written. If `None`, logging will just be printed to standard output. :param list exports: A (potentially empty) list of export targets. Currently only "xml" is supported. :param str job_type: 'hazard' or 'risk' :param bool supervised: Defaults to `True`. If `True`, run OpenQuake with a supervisor process, which monitors the job executor process and collects log messages. """ calc_mode = getattr(job, '%s_calculation' % job_type).calculation_mode calc = get_calculator_class(job_type, calc_mode)(job) # Create job stats, which implicitly records the start time for the job _create_job_stats(job) # Closing all db connections to make sure they're not shared between # supervisor and job executor processes. # Otherwise, if one of them closes the connection it immediately becomes # unavailable for others. if supervised: django_db.close_connection() job_pid = os.fork() if not job_pid: # calculation executor process try: _job_exec(job, log_level, exports, job_type, calc) except Exception, ex: logs.LOG.critical("Calculation failed with exception: '%s'" % str(ex)) raise finally:
def test(self): # check that if risk models are provided, then the sites # and the imls are got from there cfg = helpers.get_data_path('classical_job-sd-imt.ini') job = engine.job_from_file(cfg, helpers.default_user()) job.is_running = True job.save() haz_calc = job.get_oqparam() calc = get_calculator_class('hazard', haz_calc.calculation_mode)(job) calc.parse_risk_models() self.assertEqual(['PGA'], list(calc.hc.intensity_measure_types_and_levels)) self.assertEqual(3, calc.job.exposuremodel.exposuredata_set.count()) return job
def test_check_limits_classical(self): # this is a based on a demo with 3 realizations, 2 sites and 4 rlzs cfg = helpers.get_data_path("calculators/hazard/classical/haz_map_test_job.ini") job = helpers.get_job(cfg) models.JobStats.objects.create(oq_job=job) hc = job.hazard_calculation calc = get_calculator_class("hazard", hc.calculation_mode)(job) input_weight, output_weight = calc.pre_execute() self.assertEqual(input_weight, 225) self.assertEqual(output_weight, 24) calc.max_input_weight = 1 with self.assertRaises(general.InputWeightLimit): calc.check_limits(input_weight, output_weight) calc.max_input_weight = 1000 calc.max_output_weight = 1 with self.assertRaises(general.OutputWeightLimit): calc.check_limits(input_weight, output_weight)
def run_calc(job, log_level, log_file, exports, job_type, progress_handler=None): """ Run a calculation. :param job: :class:`openquake.engine.db.model.OqJob` instance which references a valid :class:`openquake.engine.db.models.RiskCalculation` or :class:`openquake.engine.db.models.HazardCalculation`. :param str log_level: The desired logging level. Valid choices are 'debug', 'info', 'progress', 'warn', 'error', and 'critical'. :param str log_file: Complete path (including file name) to file where logs will be written. If `None`, logging will just be printed to standard output. :param list exports: A (potentially empty) list of export targets. Currently only "xml" is supported. :param str job_type: 'hazard' or 'risk' :param callable progress_handler: a callback getting the progress of the calculation and the calculation object """ calc_mode = getattr(job, '%s_calculation' % job_type).calculation_mode calc = get_calculator_class(job_type, calc_mode)(job) if progress_handler is not None: calc.register_progress_handler(progress_handler) # Create job stats, which implicitly records the start time for the job _create_job_stats(job) calc_id = job.calculation.id calc_domain = 'hazard' if job.hazard_calculation else 'risk' start_logging(calc_id, calc_domain, log_file) try: with job_stats(job): _job_exec(job, log_level, exports, job_type, calc) except Exception, ex: logs.LOG.critical("Calculation failed with exception: '%s'", ex) raise
def validate(job, job_type, files, exports): """ Validate a job of type 'hazard' or 'risk' by instantiating its form class with the given files and exports. :param job: an instance of :class:`openquake.engine.db.models.OqJob` :param str job_type: "hazard" or "risk" :param dict files: {fname: :class:`openquake.engine.db.models.Input` obj} :param exports: a list of export types :returns: an error message if the form is invalid, None otherwise. """ calculation = getattr(job, '%s_calculation' % job_type) calc_mode = calculation.calculation_mode calculator_cls = get_calculator_class(job_type, calc_mode) formname = calculator_cls.__name__.replace('Calculator', 'Form') try: form_class = globals()[formname] except KeyError: return 'Could not find form class for "%s"' % calc_mode form = form_class(instance=calculation, files=files, exports=exports) if not form.is_valid(): return 'Job configuration is not valid. Errors: %s' % dict(form.errors)