def test_parse_config_with_files(self): temp_dir = tempfile.mkdtemp() site_model_input = general.writetmp(dir=temp_dir, content="foo") job_config = general.writetmp(dir=temp_dir, content=""" [general] calculation_mode = classical [site] sites = 0 0 site_model_file = %s maximum_distance=1 truncation_level=0 random_seed=0 """ % site_model_input) try: exp_base_path = os.path.dirname(job_config) expected_params = { 'base_path': exp_base_path, 'calculation_mode': 'classical', 'hazard_calculation_id': None, 'hazard_output_id': None, 'truncation_level': 0.0, 'random_seed': 0, 'maximum_distance': 1.0, 'inputs': {'site_model': site_model_input}, 'sites': [(0.0, 0.0)], } params = vars(readini.parse_config(open(job_config))) self.assertEqual(expected_params, params) self.assertEqual(['site_model'], params['inputs'].keys()) self.assertEqual([site_model_input], params['inputs'].values()) finally: shutil.rmtree(temp_dir)
def job_from_file(cfg_file_path, username, log_level, exports, hazard_output_id=None, hazard_calculation_id=None): """ Create a full job profile from a job config file. :param str cfg_file_path: Path to the job.ini. :param str username: The user who will own this job profile and all results. :param str log_level: Desired log level. :param exports: List of desired export types. :param int hazard_output_id: ID of a hazard output to use as input to this calculation. Specify this xor ``hazard_calculation_id``. :param int hazard_calculation_id: ID of a complete hazard calculation to use as input to this calculation. Specify this xor ``hazard_output_id``. :returns: :class:`openquake.engine.db.models.OqJob` object :raises: `RuntimeError` if the input job configuration is not valid """ # create the job job = prepare_job(user_name=username, log_level=log_level) # read calculation params and create the calculation profile params = readini.parse_config(open(cfg_file_path, 'r')) missing = set(params['inputs']) - INPUT_TYPES if missing: raise ValueError( 'The parameters %s in the .ini file does ' 'not correspond to a valid input type' % ', '.join(missing)) if hazard_output_id is None and hazard_calculation_id is None: # this is a hazard calculation, not a risk one job.hazard_calculation = create_calculation( models.HazardCalculation, params) job.save() # validate and raise an error if there are any problems error_message = validate(job, 'hazard', params, exports) if error_message: raise RuntimeError(error_message) return job # otherwise run a risk calculation params.update(dict(hazard_output_id=hazard_output_id, hazard_calculation_id=hazard_calculation_id)) calculation = create_calculation(models.RiskCalculation, params) job.risk_calculation = calculation job.save() # validate and raise an error if there are any problems error_message = validate(job, 'risk', params, exports) if error_message: raise RuntimeError(error_message) return job
def test_parse_config_no_files(self): # sections are there just for documentation # when we parse the file, we ignore these source = StringIO.StringIO(""" [general] CALCULATION_MODE = classical_risk region = 1 1, 2 2, 3 3 [foo] bar = baz """) # Add a 'name' to make this look like a real file: source.name = 'path/to/some/job.ini' exp_base_path = os.path.dirname( os.path.join(os.path.abspath('.'), source.name)) expected_params = { 'base_path': exp_base_path, 'calculation_mode': 'classical_risk', 'hazard_calculation_id': None, 'hazard_output_id': 42, 'region': [(1.0, 1.0), (2.0, 2.0), (3.0, 3.0)], 'inputs': {}, } params = vars(readini.parse_config(source, hazard_output_id=42)) self.assertEqual(expected_params, params)
def test(self): # check that if risk models are provided, then the ``points to # compute`` and the imls are got from there username = helpers.default_user() job = engine.prepare_job(username) cfg = helpers.get_data_path("classical_job-sd-imt.ini") params = vars(readini.parse_config(open(cfg))) del params["hazard_calculation_id"] del params["hazard_output_id"] haz_calc = engine.create_calculation(models.HazardCalculation, params) haz_calc = models.HazardCalculation.objects.get(id=haz_calc.id) job.hazard_calculation = haz_calc job.is_running = True job.save() calc = get_calculator_class("hazard", job.hazard_calculation.calculation_mode)(job) calc.parse_risk_models() self.assertEqual( [(1.0, -1.0), (0.0, 0.0)], [(point.latitude, point.longitude) for point in haz_calc.points_to_compute()] ) self.assertEqual(["PGA"], haz_calc.get_imts()) self.assertEqual(3, haz_calc.oqjob.exposuremodel.exposuredata_set.count()) return job
def get_job(cfg, username="******", hazard_calculation_id=None, hazard_output_id=None, **extras): """ Given a path to a config file and a hazard_calculation_id (or, alternatively, a hazard_output_id, create a :class:`openquake.engine.db.models.OqJob` object for a risk calculation. """ if hazard_calculation_id is None and hazard_output_id is None: return engine.job_from_file(cfg, username, 'error', [], **extras) job = engine.prepare_job(username) oqparam = readini.parse_config( open(cfg), hazard_calculation_id, hazard_output_id) params = vars(oqparam) if hazard_calculation_id is None: params['hazard_calculation_id'] = models.Output.objects.get( pk=hazard_output_id).oq_job.id # we are removing intensity_measure_types_and_levels because it is not # a field of RiskCalculation; this ugliness will disappear when # RiskCalculation will be removed del params['intensity_measure_types_and_levels'] job.save_params(params) risk_calc = engine.create_calculation(models.RiskCalculation, params) risk_calc = models.RiskCalculation.objects.get(id=risk_calc.id) job.risk_calculation = risk_calc job.save() return job
def test_parse_config_with_sites_csv(self): sites_csv = general.writetmp(content='1.0,2.1\n3.0,4.1\n5.0,6.1') try: source = StringIO.StringIO(""" [general] calculation_mode = classical [geometry] sites_csv = %s [misc] maximum_distance=0 truncation_level=3 random_seed=5 """ % sites_csv) source.name = 'path/to/some/job.ini' exp_base_path = os.path.dirname( os.path.join(os.path.abspath('.'), source.name)) expected_params = { 'base_path': exp_base_path, 'sites': 'MULTIPOINT(1.0 2.1, 3.0 4.1, 5.0 6.1)', 'calculation_mode': 'classical', 'truncation_level': '3', 'random_seed': '5', 'maximum_distance': '0', 'inputs': {}, } params = readini.parse_config(source) self.assertEqual(expected_params, params) finally: os.unlink(sites_csv)
def test_parse_config_no_files(self): # sections are there just for documentation # when we parse the file, we ignore these source = StringIO.StringIO(""" [general] CALCULATION_MODE = classical region = 1 1 2 2 3 3 [foo] bar = baz """) # Add a 'name' to make this look like a real file: source.name = 'path/to/some/job.ini' exp_base_path = os.path.dirname( os.path.join(os.path.abspath('.'), source.name)) expected_params = { 'base_path': exp_base_path, 'calculation_mode': 'classical', 'region': '1 1 2 2 3 3', 'bar': 'baz', 'inputs': {}, } params = readini.parse_config(source) self.assertEqual(expected_params, params)
def get_job(cfg, username="******", hazard_calculation_id=None, hazard_output_id=None): """ Given a path to a config file and a hazard_calculation_id (or, alternatively, a hazard_output_id, create a :class:`openquake.engine.db.models.OqJob` object for a risk calculation. """ if hazard_calculation_id is None and hazard_output_id is None: return engine.job_from_file(cfg, username, 'error', []) job = engine.prepare_job(username) params = vars(readini.parse_config( open(cfg), hazard_calculation_id, hazard_output_id)) risk_calc = engine.create_calculation(models.RiskCalculation, params) risk_calc = models.RiskCalculation.objects.get(id=risk_calc.id) job.risk_calculation = risk_calc job.save() return job
def test_parse_config_with_sites_csv(self): sites_csv = general.writetmp(content='1.0,2.1\n3.0,4.1\n5.0,6.1') try: source = StringIO.StringIO(""" [general] calculation_mode = classical [geometry] sites_csv = %s [misc] maximum_distance=1 truncation_level=3 random_seed=5 [site_params] reference_vs30_type = measured reference_vs30_value = 600.0 reference_depth_to_2pt5km_per_sec = 5.0 reference_depth_to_1pt0km_per_sec = 100.0 intensity_measure_types = PGA """ % sites_csv) source.name = 'path/to/some/job.ini' exp_base_path = os.path.dirname( os.path.join(os.path.abspath('.'), source.name)) expected_params = { 'base_path': exp_base_path, 'calculation_mode': 'classical', 'hazard_calculation_id': None, 'hazard_output_id': None, 'truncation_level': 3.0, 'random_seed': 5, 'maximum_distance': 1.0, 'inputs': {'site': sites_csv}, 'reference_depth_to_1pt0km_per_sec': 100.0, 'reference_depth_to_2pt5km_per_sec': 5.0, 'reference_vs30_type': 'measured', 'reference_vs30_value': 600.0, 'intensity_measure_types_and_levels': {'PGA': None}, } params = vars(readini.parse_config(source)) self.assertEqual(expected_params, params) finally: os.unlink(sites_csv)
def test_parse_config_with_sites_csv(self): sites_csv = general.writetmp(content='1.0,2.1\n3.0,4.1\n5.0,6.1') try: source = StringIO.StringIO(""" [general] calculation_mode = classical [geometry] sites_csv = %s [misc] maximum_distance=1 truncation_level=3 random_seed=5 [site_params] reference_vs30_type = measured reference_vs30_value = 600.0 reference_depth_to_2pt5km_per_sec = 5.0 reference_depth_to_1pt0km_per_sec = 100.0 """ % sites_csv) source.name = 'path/to/some/job.ini' exp_base_path = os.path.dirname( os.path.join(os.path.abspath('.'), source.name)) expected_params = { 'base_path': exp_base_path, 'sites': [(1.0, 2.1), (3.0, 4.1), (5.0, 6.1)], 'calculation_mode': 'classical', 'hazard_calculation_id': None, 'hazard_output_id': None, 'truncation_level': 3.0, 'random_seed': 5, 'maximum_distance': 1.0, 'inputs': {}, 'reference_depth_to_1pt0km_per_sec': 100.0, 'reference_depth_to_2pt5km_per_sec': 5.0, 'reference_vs30_type': 'measured', 'reference_vs30_value': 600.0, } params = vars(readini.parse_config(source)) self.assertEqual(expected_params, params) finally: os.unlink(sites_csv)
def test_parse_config_with_files(self): temp_dir = tempfile.mkdtemp() site_model_input = general.writetmp(dir=temp_dir, content="foo") job_config = general.writetmp(dir=temp_dir, content=""" [general] calculation_mode = classical [site] sites = 0 0 site_model_file = %s maximum_distance=1 truncation_level=0 random_seed=0 """ % site_model_input) try: exp_base_path = os.path.dirname(job_config) expected_params = { 'base_path': exp_base_path, 'calculation_mode': 'classical', 'hazard_calculation_id': None, 'hazard_output_id': None, 'truncation_level': 0.0, 'random_seed': 0, 'maximum_distance': 1.0, 'inputs': { 'site_model': site_model_input }, 'sites': [(0.0, 0.0)], } params = vars(readini.parse_config(open(job_config))) self.assertEqual(expected_params, params) self.assertEqual(['site_model'], params['inputs'].keys()) self.assertEqual([site_model_input], params['inputs'].values()) finally: shutil.rmtree(temp_dir)
def job_from_file(cfg_file_path, username, log_level="info", exports=(), hazard_output_id=None, hazard_job_id=None): """ Create a full job profile from a job config file. :param str cfg_file_path: Path to the job.ini. :param str username: The user who will own this job profile and all results. :param str log_level: Desired log level. :param exports: List of desired export types. :param int hazard_output_id: ID of a hazard output to use as input to this calculation. Specify this xor ``hazard_calculation_id``. :param int hazard_job_id: ID of a complete hazard job to use as input to this calculation. Specify this xor ``hazard_output_id``. :returns: :class:`openquake.engine.db.models.OqJob` object :raises: `RuntimeError` if the input job configuration is not valid """ # determine the previous hazard job, if any if hazard_job_id: haz_job = models.OqJob.objects.get(pk=hazard_job_id) elif hazard_output_id: # extract the hazard job from the hazard_output_id haz_job = models.Output.objects.get(pk=hazard_output_id).oq_job else: haz_job = None # no previous hazard job if haz_job: assert haz_job.job_type == "hazard", haz_job # create the current job job = prepare_job(user_name=username, log_level=log_level) # read calculation params and create the calculation profile oqparam = readini.parse_config( open(cfg_file_path), haz_job.hazard_calculation.id if haz_job and not hazard_output_id else None, hazard_output_id, ) missing = set(oqparam.inputs) - INPUT_TYPES if missing: raise ValueError( "The parameters %s in the .ini file does " "not correspond to a valid input type" % ", ".join(missing) ) params = vars(oqparam).copy() job.save_params(params) if hazard_output_id is None and hazard_job_id is None: # this is a hazard calculation, not a risk one del params["hazard_calculation_id"] del params["hazard_output_id"] job.hazard_calculation = create_calculation(models.HazardCalculation, params) job.save() return job calculation = create_calculation(models.RiskCalculation, params) job.risk_calculation = calculation job.save() return job
def get_fake_risk_job(risk_cfg, hazard_cfg, output_type="curve", username="******"): """ Takes in input the paths to a risk job config file and a hazard job config file. Creates fake hazard outputs suitable to be used by a risk calculation and then creates a :class:`openquake.engine.db.models.OqJob` object for a risk calculation. It also returns the input files referenced by the risk config file. :param output_type: gmf, gmf_scenario, or curve """ hazard_job = get_job(hazard_cfg, username) hc = hazard_job.hazard_calculation lt_model = models.LtSourceModel.objects.create( hazard_calculation=hazard_job.hazard_calculation, ordinal=1, sm_lt_path="test_sm") rlz = models.LtRealization.objects.create( lt_model=lt_model, ordinal=1, weight=1, gsim_lt_path="test_gsim") if output_type == "curve": models.HazardCurve.objects.create( lt_realization=rlz, output=models.Output.objects.create_output( hazard_job, "Test Hazard output", "hazard_curve_multi"), investigation_time=hc.investigation_time) hazard_output = models.HazardCurve.objects.create( lt_realization=rlz, output=models.Output.objects.create_output( hazard_job, "Test Hazard output", "hazard_curve"), investigation_time=hc.investigation_time, imt="PGA", imls=[0.1, 0.2, 0.3]) for point in ["POINT(-1.01 1.01)", "POINT(0.9 1.01)", "POINT(0.01 0.01)", "POINT(0.9 0.9)"]: models.HazardSite.objects.create( hazard_calculation=hc, location=point) models.HazardCurveData.objects.create( hazard_curve=hazard_output, poes=[0.1, 0.2, 0.3], location="%s" % point) elif output_type == "gmf_scenario": hazard_output = models.Gmf.objects.create( output=models.Output.objects.create_output( hazard_job, "Test gmf scenario output", "gmf_scenario")) models.SESCollection.objects.create( output=models.Output.objects.create_output( hazard_job, "Test SES Collection", "ses"), lt_model=None, ordinal=0) site_ids = hazard_job.hazard_calculation.save_sites( [(15.48, 38.0900001), (15.565, 38.17), (15.481, 38.25)]) for site_id in site_ids: models.GmfData.objects.create( gmf=hazard_output, task_no=0, imt="PGA", site_id=site_id, gmvs=[0.1, 0.2, 0.3], rupture_ids=[0, 1, 2]) elif output_type in ("ses", "gmf"): hazard_output = create_gmf_data_records(hazard_job, rlz)[0].gmf else: raise RuntimeError('Unexpected output_type: %s' % output_type) hazard_job.status = "complete" hazard_job.save() job = engine.prepare_job(username) params = vars(readini.parse_config(open(risk_cfg), hazard_output_id=hazard_output.output.id)) risk_calc = engine.create_calculation(models.RiskCalculation, params) job.risk_calculation = risk_calc job.save() # reload risk calculation to have all the types converted properly job.risk_calculation = models.RiskCalculation.objects.get(id=risk_calc.id) return job, set(params['inputs'])