def test_parse_config_with_files_no_force_inputs(self): site_model_input = helpers.touch(content="foo") source = StringIO.StringIO(""" [general] calculation_mode = classical [site] site_model_file = %s maximum_distance=0 truncation_level=0 random_seed=0 """ % site_model_input) # Add a 'name' to make this look like a real file: source.name = 'path/to/some/job.ini' exp_base_path = os.path.dirname( os.path.join(os.path.abspath('.'), source.name)) expected_params = { 'base_path': exp_base_path, 'force_inputs': False, 'calculation_mode': 'classical', 'truncation_level': '0', 'random_seed': '0', 'maximum_distance': '0' } # Run first with force_inputs=True to create the new Input. params, expected_files = engine2.parse_config( source, force_inputs=True) # In order for us to reuse the existing input, we need to associate # each input with a successful job. job = engine2.prepare_job(getpass.getuser()) job.hazard_calculation = engine2.create_hazard_calculation( job.owner, params, expected_files.values()) job.status = 'complete' job.save() # Now run twice with force_inputs=False (the default). source.seek(0) params1, files1 = engine2.parse_config(source) source.seek(0) params2, files2 = engine2.parse_config(source) # Check the params just for sanity. self.assertEqual(expected_params, params1) self.assertEqual(expected_params, params2) # Finally, check that the Input returned by the latest 2 calls matches # the input we created above. self.assertEqual(len(expected_files), len(files1)) self.assertEqual(len(expected_files), len(files2)) self.assertEqual( expected_files['site_model_file'].id, files1['site_model_file'].id) self.assertEqual( expected_files['site_model_file'].id, files2['site_model_file'].id)
def test_parse_config_no_files(self): # sections are there just for documentation # when we parse the file, we ignore these source = StringIO.StringIO(""" [general] CALCULATION_MODE = classical region = 1 1 2 2 3 3 [foo] bar = baz """) # Add a 'name' to make this look like a real file: source.name = 'path/to/some/job.ini' exp_base_path = os.path.dirname( os.path.join(os.path.abspath('.'), source.name)) expected_params = { 'base_path': exp_base_path, 'force_inputs': False, 'calculation_mode': 'classical', 'region': '1 1 2 2 3 3', 'bar': 'baz', } params, _ = engine2.parse_config(source) self.assertEqual(expected_params, params)
def test_read_and_validate_hazard_config(self): cfg = helpers.demo_file('simple_fault_demo_hazard/job.ini') job = engine2.prepare_job(getpass.getuser()) params, files = engine2.parse_config(open(cfg, 'r')) calculation = engine2.create_hazard_calculation( job.owner, params, files.values()) form = validation.ClassicalHazardForm( instance=calculation, files=files ) self.assertTrue(form.is_valid())
def test_a_few_inputs(self): cfg = helpers.demo_file('simple_fault_demo_hazard/job.ini') params, files = engine2.parse_config(open(cfg, 'r'), force_inputs=True) owner = helpers.default_user() hc = engine2.create_hazard_calculation(owner, params, files.values()) expected_ids = sorted([x.id for x in files.values()]) inputs = models.inputs4hcalc(hc.id) actual_ids = sorted([x.id for x in inputs]) self.assertEqual(expected_ids, actual_ids)
def test_with_input_type(self): cfg = helpers.demo_file("simple_fault_demo_hazard/job.ini") params, files = engine2.parse_config(open(cfg, "r"), force_inputs=True) owner = helpers.default_user() hc = engine2.create_hazard_calculation(owner, params, files.values()) # It should only be 1 id, actually. expected_ids = [x.id for x in files.values() if x.input_type == "lt_source"] inputs = models.inputs4hcalc(hc.id, input_type="lt_source") actual_ids = sorted([x.id for x in inputs]) self.assertEqual(expected_ids, actual_ids)
def get_hazard_job(cfg, username=None): """ Given a path to a config file, create a :class:`openquake.engine.db.models.OqJob` object for a hazard calculation. """ username = username if username is not None else default_user().user_name job = engine2.prepare_job(username) params, files = engine2.parse_config(open(cfg, 'r'), force_inputs=True) haz_calc = engine2.create_hazard_calculation( job.owner, params, files.values()) haz_calc = models.HazardCalculation.objects.get(id=haz_calc.id) job.hazard_calculation = haz_calc job.save() return job
def test_parse_config_with_files_force_inputs(self): site_model_input = helpers.touch(content="site model") sm_lt_input = helpers.touch(content="source model logic tree") gsim_lt_input = helpers.touch(content="gsim logic tree") source = StringIO.StringIO(""" [hazard_or_whatever] calculation_mode = classical gsim_logic_tree_file = %s source_model_logic_tree_file = %s site_model_file = %s not_a_valid_xml = foo.xml """ % (gsim_lt_input, sm_lt_input, site_model_input)) # Add a 'name' to make this look like a real file: source.name = 'path/to/some/job.ini' exp_base_path = os.path.dirname( os.path.join(os.path.abspath('.'), source.name)) expected_params = { 'base_path': exp_base_path, 'force_inputs': True, 'calculation_mode': 'classical', 'not_a_valid_xml': 'foo.xml', } params, files = engine2.parse_config(source, force_inputs=True) expected_files = { 'site_model_file': models.Input.objects.filter( input_type='site_model').latest('id'), 'source_model_logic_tree_file': models.Input.objects.filter( input_type='source_model_logic_tree').latest('id'), 'gsim_logic_tree_file': models.Input.objects.filter( input_type='gsim_logic_tree').latest('id'), } self.assertEqual(expected_params, params) self.assertEqual(len(expected_files), len(files)) for key in expected_files: self.assertEqual(expected_files[key].id, files[key].id)
def get_risk_job(risk_demo, hazard_demo, output_type="curve", username=None): """ Takes in input the paths (relative to the demos directory) to a risk and hazard demo file config, respectively. Creates the hazard outputs suitable to be used by a risk calculation and then creates a :class:`openquake.engine.db.models.OqJob` object for a risk calculation. It also returns the input files referenced by the risk config file. :param output_type: gmf or curve """ username = username if username is not None else default_user().user_name hazard_cfg = demo_file(hazard_demo) hazard_job = get_hazard_job(hazard_cfg, username) hc = hazard_job.hazard_calculation risk_cfg = demo_file(risk_demo) rlz = models.LtRealization.objects.create( hazard_calculation=hazard_job.hazard_calculation, ordinal=1, seed=1, weight=None, sm_lt_path="test_sm", gsim_lt_path="test_gsim", is_complete=False, total_items=1, completed_items=1) if output_type == "curve": hazard_output = models.HazardCurveData.objects.create( hazard_curve=models.HazardCurve.objects.create( lt_realization=rlz, output=models.Output.objects.create_output( hazard_job, "Test Hazard output", "hazard_curve"), investigation_time=hc.investigation_time, imt="PGA", imls=[0.1, 0.2, 0.3]), poes=[0.1, 0.2, 0.3], location="POINT(1 1)") else: hazard_output = models.Gmf.objects.create( gmf_set=models.GmfSet.objects.create( gmf_collection=models.GmfCollection.objects.create( output=models.Output.objects.create_output( hazard_job, "Test Hazard output", "gmf"), lt_realization=rlz, complete_logic_tree_gmf=False), investigation_time=hc.investigation_time, ses_ordinal=1, complete_logic_tree_gmf=False), imt="PGA", gmvs=[0.1, 0.2, 0.3], result_grp_ordinal=1, location="POINT(1 1)") hazard_job.status = "complete" hazard_job.save() job = engine2.prepare_job(username) params, files = engine2.parse_config( open(risk_cfg, 'r'), force_inputs=True) if output_type == "curve": params.update( dict(hazard_output_id=hazard_output.hazard_curve.output.id)) else: output = hazard_output.gmf_set.gmf_collection.output params.update(dict(hazard_output_id=output.id)) risk_calc = engine2.create_risk_calculation( job.owner, params, files.values()) risk_calc = models.RiskCalculation.objects.get(id=risk_calc.id) job.risk_calculation = risk_calc job.save() return job, files
def get_risk_job(risk_cfg, hazard_cfg, output_type="curve", username=None): """ Takes in input the paths to a risk job config file and a hazard job config file. Creates fake hazard outputs suitable to be used by a risk calculation and then creates a :class:`openquake.engine.db.models.OqJob` object for a risk calculation. It also returns the input files referenced by the risk config file. :param output_type: gmf, gmf_scenario, or curve """ username = username if username is not None else default_user().user_name hazard_job = get_hazard_job(hazard_cfg, username) hc = hazard_job.hazard_calculation rlz = models.LtRealization.objects.create( hazard_calculation=hazard_job.hazard_calculation, ordinal=1, seed=1, weight=None, sm_lt_path="test_sm", gsim_lt_path="test_gsim", is_complete=False, total_items=1, completed_items=1) if output_type == "curve": hazard_output = models.HazardCurve.objects.create( lt_realization=rlz, output=models.Output.objects.create_output( hazard_job, "Test Hazard output", "hazard_curve"), investigation_time=hc.investigation_time, imt="PGA", imls=[0.1, 0.2, 0.3]) for point in ["POINT(-1.01 1.01)", "POINT(0.9 1.01)", "POINT(0.01 0.01)", "POINT(0.9 0.9)"]: models.HazardCurveData.objects.create( hazard_curve=hazard_output, poes=[0.1, 0.2, 0.3], location="%s" % point) elif output_type == "gmf_scenario": output = models.Output.objects.create_output( hazard_job, "Test Hazard output", "gmf_scenario") for point in ["POINT(15.48 38.0900001)", "POINT(15.565 38.17)", "POINT(15.481 38.25)"]: hazard_output = models.GmfScenario.objects.create( output=output, imt="PGA", location=point, gmvs=[0.1, 0.2, 0.3]) else: rupture_ids = get_rupture_ids(hazard_job, hc, rlz, 3) hazard_output = models.GmfCollection.objects.create( output=models.Output.objects.create_output( hazard_job, "Test Hazard output", "gmf"), lt_realization=rlz, complete_logic_tree_gmf=False) gmf_set = models.GmfSet.objects.create( gmf_collection=hazard_output, investigation_time=hc.investigation_time, ses_ordinal=1, complete_logic_tree_gmf=False) for point in ["POINT(15.310 38.225)", "POINT(15.71 37.225)", "POINT(15.48 38.091)", "POINT(15.565 38.17)", "POINT(15.481 38.25)"]: models.Gmf.objects.create( gmf_set=gmf_set, imt="PGA", gmvs=[0.1, 0.2, 0.3], rupture_ids=rupture_ids, result_grp_ordinal=1, location=point) hazard_job.status = "complete" hazard_job.save() job = engine2.prepare_job(username) params, files = engine2.parse_config( open(risk_cfg, 'r'), force_inputs=True) params.update(dict(hazard_output_id=hazard_output.output.id)) risk_calc = engine2.create_risk_calculation( job.owner, params, files.values()) risk_calc = models.RiskCalculation.objects.get(id=risk_calc.id) job.risk_calculation = risk_calc job.save() return job, files