def test_run_calc_with_description(self): # Test importing and running a job with a config containing the # optional DESCRIPTION parameter. description = 'Classical PSHA hazard test description' orig_cfg_path = demo_file('PeerTestSet1Case2/config.gem') mod_cfg_path = os.path.join(demo_file('PeerTestSet1Case2'), 'modified_config.gem') # Use ConfigParser to add the DESCRIPTION param to an existing config # profile and write a new temporary config file: cfg_parser = ConfigParser.ConfigParser() cfg_parser.readfp(open(orig_cfg_path, 'r')) cfg_parser.set('general', 'DESCRIPTION', description) cfg_parser.write(open(mod_cfg_path, 'w')) run_job(mod_cfg_path) job = OqJob.objects.latest('id') job_profile = job.profile() self.assertEqual(description, job_profile.description) self.assertEqual(description, job.description) # Clean up the temporary config file: os.unlink(mod_cfg_path)
def test_uhs_output_type_xml(self): # Run a calculation with --output-type=xml and check that the expected # result files are created in the right location. # This location is based on parameters in the UHS config file: results_target_dir = demo_file('uhs/computed_output') # clear the target dir from previous demo/test runs shutil.rmtree(results_target_dir) expected_export_files = [ os.path.join(results_target_dir, 'uhs_poe:0.1.hdf5'), os.path.join(results_target_dir, 'uhs_poe:0.02.hdf5'), os.path.join(results_target_dir, 'uhs.xml'), ] for f in expected_export_files: self.assertFalse(os.path.exists(f)) uhs_cfg = demo_file('uhs/config.gem') try: ret_code = run_job(uhs_cfg, ['--output-type=xml']) self.assertEqual(0, ret_code) # Check that all the output files were created: for f in expected_export_files: self.assertTrue(os.path.exists(f)) finally: shutil.rmtree(results_target_dir)
def test_scenario_risk_sample_based(self): # This QA test is a longer-running test of the Scenario Risk # calculator. # The vulnerabiilty model has non-zero Coefficients of Variation and # therefore exercises the 'sample-based' path through the calculator. # This test is configured to produce 1000 ground motion fields at each # location of interest (in the test above, only 10 are produced). # Since we're seeding the random epsilon sampling, we can consistently # reproduce all result values. # When these values are compared to the results computed by a similar # config which takes the 'mean-based' path (with CoVs = 0), we expect # the following: # All of the mean values in the 'sample-based' results should be with # 5%, + or -, of the 'mean-based' results. # The standard deviation values of the 'sample-based' results should # simply be greater than those produced with the 'mean-based' method. # For comparison, mean and stddev values for the region were computed # with 1000 GMFs using the mean-based approach. These values (rounded # to 2 decimal places) are: mb_mean_loss = 1233.26 mb_stddev_loss = 443.63 # Loss map for the mean-based approach: mb_loss_map = [ dict(asset='a3', pos='15.48 38.25', mean=200.54874638, stddev=94.2302991022), dict(asset='a2', pos='15.56 38.17', mean=510.821363253, stddev=259.964152622), dict(asset='a1', pos='15.48 38.09', mean=521.885458891, stddev=244.825980356), ] # Sanity checks are done. Let's do this. scen_cfg = helpers.demo_file( 'scenario_risk/config_sample-based_qa.gem') result = helpers.run_job(scen_cfg, ['--output-type=xml'], check_output=True) job = OqJob.objects.latest('id') self.assertEqual('succeeded', job.status) expected_loss_map_file = helpers.demo_file( 'scenario_risk/computed_output/loss-map-%s.xml' % job.id) self.assertTrue(os.path.exists(expected_loss_map_file)) loss_map = helpers.loss_map_result_from_file(expected_loss_map_file) self._verify_loss_map_within_range(sorted(mb_loss_map), sorted(loss_map), 0.05) exp_mean_loss, exp_stddev_loss = helpers.mean_stddev_from_result_line( result) self.assertAlmostEqual(mb_mean_loss, exp_mean_loss, delta=mb_mean_loss * 0.05) self.assertTrue(exp_stddev_loss > mb_stddev_loss)
def test_scenario_risk(self): # This test exercises the 'mean-based' path through the Scenario Risk # calculator. There is no random sampling done here so the results are # 100% predictable. scen_cfg = helpers.demo_file('scenario_risk/config.gem') exp_mean_loss = 1053.09 exp_stddev_loss = 246.62 expected_loss_map = [ dict(asset='a3', pos='15.48 38.25', mean=180.717534009275, stddev=92.2122644809969), dict(asset='a2', pos='15.56 38.17', mean=432.225448142534, stddev=186.864456949986), dict(asset='a1', pos='15.48 38.09', mean=440.147078317589, stddev=182.615976701858), ] result = helpers.run_job(scen_cfg, ['--output-type=xml'], check_output=True) job = OqJob.objects.latest('id') self.assertEqual('succeeded', job.status) expected_loss_map_file = helpers.demo_file( 'scenario_risk/computed_output/loss-map-%s.xml' % job.id) self.assertTrue(os.path.exists(expected_loss_map_file)) self._verify_loss_map(expected_loss_map_file, expected_loss_map) # We expected the shell output to look something like the following # two lines: # Mean region loss value: 1053.09 # Standard deviation region loss value: 246.62 # split on newline and filter out empty lines result = [line for line in result.split('\n') if len(line) > 0] # we expect 2 lines; 1 for mean, 1 for stddev self.assertEqual(2, len(result)) actual_mean = float(result[0].split()[-1]) actual_stddev = float(result[1].split()[-1]) self.assertAlmostEqual(exp_mean_loss, actual_mean, places=self.TOTAL_LOSS_PRECISION) self.assertAlmostEqual(exp_stddev_loss, actual_stddev, places=self.TOTAL_LOSS_PRECISION)
def _create_job_profiles(self, user_name): uhs_cfg = helpers.demo_file('uhs/config.gem') job = engine.prepare_job() self.uhs_jp, _, _ = engine.import_job_profile(uhs_cfg, job, user_name=user_name) cpsha_cfg = helpers.demo_file('classical_psha_based_risk/config.gem') job = engine.prepare_job() self.cpsha_jp, _, _ = engine.import_job_profile(cpsha_cfg, job, user_name=user_name)
def setUp(self): self.job, _ = helpers.get_risk_job( demo_file('scenario_risk/job.ini'), demo_file('scenario_hazard/job.ini'), output_type="gmf_scenario") self.calculator = scenario.ScenarioRiskCalculator(self.job) self.calculator.pre_execute() self.job.is_running = True self.job.status = 'executing' self.job.save()
def setUp(self): self.job, _ = helpers.get_risk_job( demo_file('event_based_risk/job.ini'), demo_file('event_based_hazard/job.ini'), output_type="gmf") self.calculator = event_based.EventBasedRiskCalculator(self.job) self.calculator.pre_execute() self.job.is_running = True self.job.status = 'executing' self.job.save()
def test_a_few_inputs(self): job, files = helpers.get_risk_job( demo_file('classical_psha_based_risk/job.ini'), demo_file('simple_fault_demo_hazard/job.ini')) rc = job.risk_calculation expected_ids = sorted([x.id for x in files.values()]) inputs = models.inputs4rcalc(rc.id) actual_ids = sorted([x.id for x in inputs]) self.assertEqual(expected_ids, actual_ids)
def test_hazard_map_test(self): helpers.run_job(helpers.demo_file( os.path.join("HazardMapTest", "config.gem"))) self.job = models.OqCalculation.objects.latest("id") path = helpers.demo_file(os.path.join("HazardMapTest", "expected_results", "meanHazardMap0.1.dat")) expected_map = load_expected_map(path) poe = 0.1 statistic_type = "mean" verify_hazmap_results(self, self.job, expected_map, poe, statistic_type)
def setUp(self): job, _ = helpers.get_risk_job( demo_file('classical_psha_based_risk/job.ini'), demo_file('simple_fault_demo_hazard/job.ini') ) self.compulsory_arguments = dict( calculation_mode="classical", lrem_steps_per_interval=5) self.other_args = dict( owner=helpers.default_user(), region_constraint=( 'POLYGON((-122.0 38.113, -122.114 38.113, -122.57 38.111, ' '-122.0 38.113))'), hazard_output=job.risk_calculation.hazard_output)
def test_hazard_map_test(self): helpers.run_job(helpers.demo_file( os.path.join("HazardMapTest", "config.gem"))) self.job = models.OqJob.objects.latest("id") path = helpers.demo_file(os.path.join("HazardMapTest", "expected_results", "meanHazardMap0.1.dat")) expected_map = load_expected_map(path) poe = 0.1 statistic_type = "mean" verify_hazmap_results(self, self.job, expected_map, poe, statistic_type)
def test_with_input_type(self): job, files = helpers.get_risk_job( demo_file('classical_psha_based_risk/job.ini'), demo_file('simple_fault_demo_hazard/job.ini')) rc = job.risk_calculation # It should only be 1 id, actually. expected_ids = [x.id for x in files.values() if x.input_type == 'exposure'] inputs = models.inputs4rcalc(rc.id, input_type='exposure') actual_ids = sorted([x.id for x in inputs]) self.assertEqual(expected_ids, actual_ids)
def test_hazard_input_on_exposure_sites(self): cfg = helpers.demo_file("scenario_risk/config_hzr_exposure.gem") ret_code = helpers.run_job(cfg, ["--output-type=xml"]) self.assertEquals(0, ret_code) self.assertEqual("succeeded", OqJob.objects.latest("id").status)
def test_calculator_for_task(self): """Load up a sample calculation (into the db and cache) and make sure we can instantiate the correct calculator for a given calculation id. """ from openquake.calculators.hazard.classical.core import ( ClassicalHazardCalculator) job = engine.prepare_job() job_profile, params, sections = engine.import_job_profile( demo_file('simple_fault_demo_hazard/config.gem'), job) job_ctxt = engine.JobContext(params, job.id, oq_job_profile=job_profile, oq_job=job) job_ctxt.to_kvs() with patch('openquake.utils.tasks.get_running_job') as grc_mock: # Loading of the JobContext is done by # `get_running_job`, which is covered by other tests. # So, we just want to make sure that it's called here. grc_mock.return_value = job_ctxt calculator = tasks.calculator_for_task(job.id, 'hazard') self.assertTrue(isinstance(calculator, ClassicalHazardCalculator)) self.assertEqual(1, grc_mock.call_count)
def test_bcr_risk_export(self): # Tests that outputs of a risk classical calculation are # exported target_dir = tempfile.mkdtemp() try: cfg = helpers.demo_file('classical_bcr/job.ini') # run the calculation to create something to export retcode = helpers.run_risk_job_sp(cfg, self.hazard_id, silence=True) self.assertEqual(0, retcode) job = models.OqJob.objects.latest('id') outputs = export_core.get_outputs(job.id) expected_outputs = 1 # 1 bcr distribution self.assertEqual(expected_outputs, len(outputs)) # Export the loss curves: distribution = outputs.filter(output_type='bcr_distribution')[0] rc_files = risk.export(distribution.id, target_dir) self.assertEqual(1, len(rc_files)) for f in rc_files: self._test_exported_file(f) finally: shutil.rmtree(target_dir)
def hazard_id(self): job = helpers.get_hazard_job( helpers.demo_file("scenario_hazard/job.ini")) job.hazard_calculation = models.HazardCalculation.objects.create( owner=job.hazard_calculation.owner, truncation_level=job.hazard_calculation.truncation_level, maximum_distance=job.hazard_calculation.maximum_distance, intensity_measure_types_and_levels=( job.hazard_calculation.intensity_measure_types_and_levels), calculation_mode="scenario") job.save() output = models.Output.objects.create_output( job, "Test Hazard output", "gmf_scenario") fname = os.path.join(os.path.dirname(__file__), 'gmf_scenario.csv') with open(fname, 'rb') as csvfile: gmfreader = csv.reader(csvfile, delimiter=',') locations = gmfreader.next() arr = numpy.array([map(float, row) for row in gmfreader]) for i, gmvs in enumerate(arr.transpose()): models.GmfScenario.objects.create( output=output, imt="MMI", gmvs=gmvs, result_grp_ordinal=1, location="POINT(%s)" % locations[i]) return output.id
def test_parameterize_sites_with_site_model(self): job_ctxt = helpers.prepare_job_context( helpers.demo_file( 'simple_fault_demo_hazard/config_with_site_model.gem')) calc = classical.ClassicalHazardCalculator(job_ctxt) calc.initialize() # This tests to ensure that the `initialize` implementation for this # calculator properly stores the site model in the DB. # NOTE: If this test ever breaks, it's probably because the # ClassicalHazardCalculator is no longer calling the `initalize` code # in its super class (BaseHazardCalculator). site_model = hazard_general.get_site_model(job_ctxt.oq_job.id) self.assertIsNotNone(site_model) set_params_patch = helpers.patch( 'openquake.calculators.hazard.general.set_java_site_parameters') closest_data_patch = helpers.patch( 'openquake.calculators.hazard.general.get_closest_site_model_data') sp_mock = set_params_patch.start() cd_mock = closest_data_patch.start() try: calc.parameterize_sites(job_ctxt.sites_to_compute()) exp_call_count = len(job_ctxt.sites_to_compute()) self.assertEqual(exp_call_count, sp_mock.call_count) self.assertEqual(exp_call_count, cd_mock.call_count) finally: # tear down the patches set_params_patch.stop() closest_data_patch.stop()
def test_parameterize_sites_no_site_model(self): job_ctxt = helpers.prepare_job_context( helpers.demo_file('simple_fault_demo_hazard/config.gem')) calc = classical.ClassicalHazardCalculator(job_ctxt) jsites = calc.parameterize_sites(job_ctxt.sites_to_compute()) # expected params: jp = job_ctxt.oq_job_profile exp_sadigh = job_params.REVERSE_ENUM_MAP[jp.sadigh_site_type] exp_vs30 = jp.reference_vs30_value exp_vs30_type = jp.vs30_type exp_z1pt0 = jp.depth_to_1pt_0km_per_sec exp_z2pt5 = jp.reference_depth_to_2pt5km_per_sec_param for jsite in jsites: self.assertEqual(exp_vs30, jsite.getParameter('Vs30').getValue().value) self.assertEqual(exp_vs30_type, jsite.getParameter('Vs30 Type').getValue()) self.assertEqual( exp_z1pt0, jsite.getParameter('Depth 1.0 km/sec').getValue().value) self.assertEqual( exp_z2pt5, jsite.getParameter('Depth 2.5 km/sec').getValue().value) self.assertEqual(exp_sadigh, jsite.getParameter('Sadigh Site Type').getValue())
def test_complex_fault_demo_hazard_nrml_written_once(self): """ Run the `complex_fault_demo_hazard` demo and verify that the NRML files are written only once. """ def filter_multi(): """Filter and return files that were written more than once.""" counts = defaultdict(int) files = stats.kvs_op("lrange", key, 0, -1) for file in files: counts[file] += 1 return [(f, c) for f, c in counts.iteritems() if c > 1] job_cfg = helpers.demo_file(os.path.join( "complex_fault_demo_hazard", "config.gem")) helpers.run_job(job_cfg, ['--output-type=xml']) self.job = models.OqJob.objects.latest("id") key = stats.key_name( self.job.id, *stats.STATS_KEYS["hcls_xmlcurvewrites"]) if key: multi_writes = filter_multi() self.assertFalse(multi_writes, str(multi_writes)) key = stats.key_name( self.job.id, *stats.STATS_KEYS["hcls_xmlmapwrites"]) if key: multi_writes = filter_multi() self.assertFalse(multi_writes, str(multi_writes))
def test_complex_fault_demo_hazard_nrml_written_once(self): """ Run the `complex_fault_demo_hazard` demo and verify that the NRML files are written only once. """ def filter_multi(): """Filter and return files that were written more than once.""" counts = defaultdict(int) files = stats.kvs_op("lrange", key, 0, -1) for file in files: counts[file] += 1 return [(f, c) for f, c in counts.iteritems() if c > 1] job_cfg = helpers.demo_file(os.path.join( "complex_fault_demo_hazard", "config.gem")) helpers.run_job(job_cfg, output="xml") self.job = models.OqCalculation.objects.latest("id") key = stats.key_name( self.job.id, *stats.STATS_KEYS["hcls_xmlcurvewrites"]) if key: multi_writes = filter_multi() self.assertFalse(multi_writes, str(multi_writes)) key = stats.key_name( self.job.id, *stats.STATS_KEYS["hcls_xmlmapwrites"]) if key: multi_writes = filter_multi() self.assertFalse(multi_writes, str(multi_writes))
def test_initialize_site_model(self): # we need a slightly different config file for this test cfg = helpers.demo_file( 'simple_fault_demo_hazard/job_with_site_model.ini') self.job = helpers.get_hazard_job(cfg) self.calc = core.ClassicalHazardCalculator(self.job) self.calc.initialize_site_model() # If the site model isn't valid for the calculation geometry, a # `RuntimeError` should be raised here # Okay, it's all good. Now check the count of the site model records. [site_model_inp] = models.inputs4hcalc( self.job.hazard_calculation.id, input_type='site_model') sm_nodes = models.SiteModel.objects.filter(input=site_model_inp) self.assertEqual(2601, len(sm_nodes)) num_pts_to_compute = len( self.job.hazard_calculation.points_to_compute()) [site_data] = models.SiteData.objects.filter( hazard_calculation=self.job.hazard_calculation.id) # The site model is good. Now test that `site_data` was computed. # For now, just test the lengths of the site data collections: self.assertEqual(num_pts_to_compute, len(site_data.lons)) self.assertEqual(num_pts_to_compute, len(site_data.lats)) self.assertEqual(num_pts_to_compute, len(site_data.vs30s)) self.assertEqual(num_pts_to_compute, len(site_data.vs30_measured)) self.assertEqual(num_pts_to_compute, len(site_data.z1pt0s)) self.assertEqual(num_pts_to_compute, len(site_data.z2pt5s))
def hazard_id(self): job = helpers.get_hazard_job( helpers.demo_file("scenario_hazard/job.ini")) hc = job.hazard_calculation job.hazard_calculation = models.HazardCalculation.objects.create( owner=hc.owner, truncation_level=hc.truncation_level, maximum_distance=hc.maximum_distance, intensity_measure_types=["PGA"], calculation_mode="scenario") job.status = "complete" job.save() output = models.Output.objects.create_output( job, "Test Hazard output", "gmf_scenario") fname = os.path.join(os.path.dirname(__file__), 'gmf_scenario.csv') with open(fname, 'rb') as csvfile: gmfreader = csv.reader(csvfile, delimiter=',') locations = gmfreader.next() arr = numpy.array([[float(x) for x in row] for row in gmfreader]) for i, gmvs in enumerate(arr): models.GmfScenario.objects.create( output=output, imt="PGA", gmvs=gmvs, location="POINT(%s)" % locations[i]) return output.id
def test_peer_test_set_1_case_10(self): expected_results = self._load_results("PeerTestSet1Case10") self._run_job(helpers.demo_file( os.path.join("PeerTestSet1Case10", "config.gem"))) self._assert_results_are(expected_results)
def test_peer_test_set_1_case_5(self): expected_results = load_exp_hazcurve_results("PeerTestSet1Case5") helpers.run_job(helpers.demo_file( os.path.join("PeerTestSet1Case5", "config.gem"))) self._assert_hazcurve_results_are(expected_results)
def test_calculator_for_task(self): """Load up a sample calculation (into the db and cache) and make sure we can instantiate the correct calculator for a given calculation id. """ from openquake.calculators.hazard.classical.core import ( ClassicalHazardCalculator) job = engine.prepare_job() job_profile, params, sections = engine.import_job_profile(demo_file( 'simple_fault_demo_hazard/config.gem'), job) job_ctxt = engine.JobContext(params, job.id, oq_job_profile=job_profile, oq_job=job) job_ctxt.to_kvs() with patch( 'openquake.utils.tasks.get_running_job') as grc_mock: # Loading of the JobContext is done by # `get_running_job`, which is covered by other tests. # So, we just want to make sure that it's called here. grc_mock.return_value = job_ctxt calculator = tasks.calculator_for_task(job.id, 'hazard') self.assertTrue(isinstance(calculator, ClassicalHazardCalculator)) self.assertEqual(1, grc_mock.call_count)
def test_compute_uhs_with_site_model(self): the_job = helpers.prepare_job_context( helpers.demo_file('uhs/config_with_site_model.gem')) the_job.to_kvs() site = Site(0, 0) helpers.store_hazard_logic_trees(the_job) get_sm_patch = helpers.patch( 'openquake.calculators.hazard.general.get_site_model') get_closest_patch = helpers.patch( 'openquake.calculators.hazard.general.get_closest_site_model_data') compute_patch = helpers.patch( 'openquake.calculators.hazard.uhs.core._compute_uhs') get_sm_mock = get_sm_patch.start() get_closest_mock = get_closest_patch.start() compute_mock = compute_patch.start() get_closest_mock.return_value = SiteModel( vs30=800, vs30_type='measured', z1pt0=100, z2pt5=200) try: compute_uhs(the_job, site) self.assertEqual(1, get_sm_mock.call_count) self.assertEqual(1, get_closest_mock.call_count) self.assertEqual(1, compute_mock.call_count) finally: get_sm_patch.stop() get_closest_patch.stop() compute_patch.stop()
def hazard_id(self): job = helpers.get_hazard_job( helpers.demo_file("simple_fault_demo_hazard/job.ini")) hazard_curve = [ (0.001, 0.0398612669790014), (0.01, 0.039861266979001400), (0.05, 0.039728757480298900), (0.10, 0.029613426625612500), (0.15, 0.019827328756491600), (0.20, 0.013062270161451900), (0.25, 0.008655387950000430), (0.30, 0.005898520593689670), (0.35, 0.004061698589511780), (0.40, 0.002811727179526820), (0.45, 0.001995117417776690), (0.50, 0.001358705972845710), (0.55, 0.000989667841573727), (0.60, 0.000757544444296432), (0.70, 0.000272824002045979), (0.80, 0.00), (0.9, 0.00), (1.0, 0.00)] hd = models.HazardCurveData.objects.create( hazard_curve=models.HazardCurve.objects.create( output=models.Output.objects.create_output( job, "Test Hazard curve", "hazard_curve"), investigation_time=50, imt="PGA", imls=[hz[0] for hz in hazard_curve], statistics="mean"), poes=[hz[1] for hz in hazard_curve], location="POINT(1 1)") return hd.hazard_curve.output.id
def test_calculator_for_task(self): """Load up a sample calculation (into the db and cache) and make sure we can instantiate the correct calculator for a given calculation id. """ from openquake.calculators.hazard.classical.core import ( ClassicalHazardCalculator) job_profile, params, sections = engine.import_job_profile(demo_file( 'simple_fault_demo_hazard/config.gem')) calculation = OqCalculation(owner=job_profile.owner, oq_job_profile=job_profile) calculation.save() calc_proxy = engine.CalculationProxy(params, calculation.id, oq_job_profile=job_profile, oq_calculation=calculation) calc_proxy.to_kvs() with patch( 'openquake.utils.tasks.get_running_calculation') as grc_mock: # Loading of the CalculationProxy is done by # `get_running_calculation`, which is covered by other tests. # So, we just want to make sure that it's called here. grc_mock.return_value = calc_proxy calculator = tasks.calculator_for_task(calculation.id, 'hazard') self.assertTrue(isinstance(calculator, ClassicalHazardCalculator)) self.assertEqual(1, grc_mock.call_count)
def test_export_for_scenario(self): target_dir = tempfile.mkdtemp() try: cfg = helpers.demo_file('scenario_hazard/job.ini') # run the calculation to create something to export retcode = helpers.run_hazard_job_sp(cfg, silence=True) self.assertEqual(0, retcode) job = models.OqJob.objects.latest('id') outputs = export_core.get_outputs(job.id) self.assertEqual(1, len(outputs)) # 1 GMF gmf_outputs = outputs.filter(output_type='gmf_scenario') self.assertEqual(1, len(gmf_outputs)) exported_files = hazard.export(gmf_outputs[0].id, target_dir) self.assertEqual(1, len(exported_files)) # Check the file paths exist, is absolute, and the file isn't # empty. f = exported_files[0] self._test_exported_file(f) # Check for the correct number of GMFs in the file: tree = etree.parse(f) self.assertEqual(10, number_of('nrml:gmf', tree)) finally: shutil.rmtree(target_dir)
def test_peer_test_set_1_case_5(self): expected_results = load_exp_hazcurve_results("PeerTestSet1Case5") helpers.run_job( helpers.demo_file(os.path.join("PeerTestSet1Case5", "config.gem"))) self._assert_hazcurve_results_are(expected_results)
def test_hazard_input_on_exposure_sites(self): cfg = helpers.demo_file( "scenario_risk/config_hzr_exposure.gem") ret_code = helpers.run_job(cfg, ["--output-type=xml"]) self.assertEquals(0, ret_code) self.assertEqual("succeeded", OqJob.objects.latest("id").status)
def test_scenario_risk(self): # This test exercises the 'mean-based' path through the Scenario Risk # calculator. There is no random sampling done here so the results are # 100% predictable. scen_cfg = helpers.demo_file('scenario_risk/config.gem') exp_mean_loss = 1053.09 exp_stddev_loss = 246.62 expected_loss_map = [ dict(asset='a3', pos='15.48 38.25', mean=180.717534009275, stddev=92.2122644809969), dict(asset='a2', pos='15.56 38.17', mean=432.225448142534, stddev=186.864456949986), dict(asset='a1', pos='15.48 38.09', mean=440.147078317589, stddev=182.615976701858), ] result = helpers.run_job(scen_cfg, ['--output-type=xml'], check_output=True) job = OqJob.objects.latest('id') self.assertEqual('succeeded', job.status) expected_loss_map_file = helpers.demo_file( 'scenario_risk/computed_output/loss-map-%s.xml' % job.id) self.assertTrue(os.path.exists(expected_loss_map_file)) helpers.verify_loss_map(self, expected_loss_map_file, expected_loss_map, self.LOSSMAP_PRECISION) actual_mean, actual_stddev = helpers.mean_stddev_from_result_line( result) self.assertAlmostEqual(exp_mean_loss, actual_mean, places=self.TOTAL_LOSS_PRECISION) self.assertAlmostEqual(exp_stddev_loss, actual_stddev, places=self.TOTAL_LOSS_PRECISION)
def setUp(self): cfg = helpers.demo_file('simple_fault_demo_hazard/job.ini') self.job = helpers.get_hazard_job(cfg, username="******") for i in range(0, random.randint(1, 10)): models.LtRealization( hazard_calculation=self.job.hazard_calculation, ordinal=i, seed=None, weight=1 / (i + 1), sm_lt_path=[i], gsim_lt_path=[i], total_items=0, completed_items=0).save()
def test_log_file_access_denied(self): # Attempt to log to a location for which the user does not have write # access ('/', for example). uhs_cfg = helpers.demo_file('uhs/config.gem') result = helpers.run_job(uhs_cfg, ['--log-file', '/oq.log'], check_output=True) self.assertEqual( 'Error writing to log file /oq.log: Permission denied\n', result)
def test_hazard_computed_on_exposure_sites(self): # slightly different configuration where we # run the engine triggering the hazard computation # on sites defined in the exposure file cfg = helpers.demo_file( "classical_psha_based_risk/config_hzr_exposure.gem") self._run_job(cfg) self._verify_job_succeeded()
def hazard_id(self): job = helpers.get_hazard_job(helpers.demo_file("event_based_hazard/job.ini")) job.hazard_calculation = models.HazardCalculation.objects.create( owner=job.hazard_calculation.owner, truncation_level=job.hazard_calculation.truncation_level, maximum_distance=job.hazard_calculation.maximum_distance, intensity_measure_types_and_levels=(job.hazard_calculation.intensity_measure_types_and_levels), calculation_mode="event_based", investigation_time=50, ses_per_logic_tree_path=1, ) job.save() hc = job.hazard_calculation lt_realization = models.LtRealization.objects.create( hazard_calculation=job.hazard_calculation, ordinal=1, seed=1, weight=None, sm_lt_path="test_sm", gsim_lt_path="test_gsim", is_complete=False, total_items=1, completed_items=1, ) gmf_set = models.GmfSet.objects.create( gmf_collection=models.GmfCollection.objects.create( output=models.Output.objects.create_output(job, "Test Hazard output", "gmf"), lt_realization=lt_realization, complete_logic_tree_gmf=False, ), investigation_time=hc.investigation_time, ses_ordinal=1, complete_logic_tree_gmf=False, ) with open(os.path.join(os.path.dirname(__file__), "gmf.csv"), "rb") as csvfile: gmfreader = csv.reader(csvfile, delimiter=",") locations = gmfreader.next() gmv_matrix = numpy.array([[float(x) for x in row] for row in gmfreader]).transpose() rupture_ids = helpers.get_rupture_ids(job, hc, lt_realization, len(gmv_matrix[0])) for i, gmvs in enumerate(gmv_matrix): models.Gmf.objects.create( gmf_set=gmf_set, imt="PGA", gmvs=gmvs, rupture_ids=map(str, rupture_ids), result_grp_ordinal=1, location="POINT(%s)" % locations[i], ) return gmf_set.gmf_collection.output.id
def test_probabilistic_risk(self): cfg = helpers.demo_file("probabilistic_event_based_risk/config_qa.gem") self._run_job(cfg) self._verify_job_succeeded() self._verify_loss_maps() self._verify_loss_ratio_curves() self._verify_loss_curves() self._verify_aggregate_curve()
def test_classical_psha_based_risk(self): cfg = helpers.demo_file( 'classical_psha_based_risk/config.gem') self._run_job(cfg) self._verify_job_succeeded() self._verify_loss_curve() self._verify_loss_ratio_curve() self._verify_loss_maps()
def test_scenario_damage_con(self): cfg = helpers.demo_file("scenario_damage_risk/config.gem") self._run_job(cfg) self._verify_job_succeeded() self._verify_damage_states() self._verify_dist_per_asset_con() self._verify_dist_per_taxonomy_con() self._verify_total_dist_con()
def test_scenario_risk(self): # The rudimentary beginnings of a QA test for the scenario calc. # For now, just run it end-to-end to make sure it doesn't blow up. scen_cfg = helpers.demo_file('scenario_risk/config.gem') ret_code = helpers.run_job(scen_cfg, ['--output-type=xml']) self.assertEqual(0, ret_code) job = OqJob.objects.latest('id') self.assertEqual('succeeded', job.status)
def test_export_uhs(self): # Tests the UHS calculation run and export end-to-end. # For the export, we only check the quantity, location, and names of # each exported file. We don't check the contents; that's covered in # other tests. uhs_cfg = helpers.demo_file('uhs/config.gem') export_target_dir = tempfile.mkdtemp() expected_export_files = [ os.path.join(export_target_dir, 'uhs_poe:0.1.hdf5'), os.path.join(export_target_dir, 'uhs_poe:0.02.hdf5'), os.path.join(export_target_dir, 'uhs.xml'), ] # Sanity check and precondition: these files should not exist yet for f in expected_export_files: self.assertFalse(os.path.exists(f)) try: ret_code = helpers.run_job(uhs_cfg) self.assertEqual(0, ret_code) job = models.OqJob.objects.latest('id') [output] = models.Output.objects.filter(oq_job=job.id) # Split into a list, 1 result for each row in the output. # The first row of output (the table header) is discarded. listed_calcs = helpers.prepare_cli_output( subprocess.check_output( ['bin/openquake', '--list-calculations'])) check_list_calcs(self, listed_calcs, job.id) listed_outputs = helpers.prepare_cli_output( subprocess.check_output( ['bin/openquake', '--list-outputs', str(job.id)])) check_list_outputs(self, listed_outputs, output.id, 'uh_spectra') listed_exports = helpers.prepare_cli_output( subprocess.check_output([ 'bin/openquake', '--export', str(output.id), export_target_dir ])) self.assertEqual(expected_export_files, listed_exports) # Check that the files actually have been created, # and also verify that the paths are absolute: for f in listed_exports: self.assertTrue(os.path.exists(f)) self.assertTrue(os.path.isabs(f)) finally: shutil.rmtree(export_target_dir)
def test_default_validators_disagg_job(self): """Test to ensure that a Disaggregation job always includes the :class:`openquake.job.config.DisaggregationValidator`. """ da_job_path = helpers.demo_file('disaggregation/config.gem') da_job = helpers.job_from_file(da_job_path) validators = config.default_validators(da_job.sections, da_job.params) # test that the default validators include a DisaggregationValidator self.assertTrue( any(isinstance(v, DisaggregationValidator) for v in validators))
def test_default_validators_classical_bcr_risk(self): # For Classical BCR Hazard+Risk calculations, ensure that a # `ClassicalRiskValidator` is included in the default validators. cfg_path = helpers.demo_file('benefit_cost_ratio/config.gem') job_profile, params, sections = engine.import_job_profile( cfg_path, self.job) validators = config.default_validators(sections, params) self.assertTrue( any(isinstance(v, ClassicalRiskValidator) for v in validators))
def test_default_validators_event_based_bcr_risk(self): # For Event-Based BCR Risk calculations, ensure that a # `EventBasedRiskValidator` is included in the default validators. cfg_path = helpers.demo_file('benefit_cost_ratio/config_ebased.gem') job_profile, params, sections = engine.import_job_profile( cfg_path, self.job) validators = config.default_validators(sections, params) self.assertTrue( any(isinstance(v, EventBasedRiskValidator) for v in validators))
def test_complex_fault_demo_hazard(self): """Run the `complex_fault_demo_hazard` demo and verify all of the resulting hazard curve and hazard map data.""" job_cfg = helpers.demo_file( os.path.join("complex_fault_demo_hazard", "config.gem")) exp_results_dir = os.path.join("complex_fault_demo_hazard", "expected_results") helpers.run_job(job_cfg) self.job = models.OqJob.objects.latest("id") # Check hazard curves for sample 0: # Hazard curve expected results for logic tree sample 0: hazcurve_0 = helpers.demo_file( os.path.join(exp_results_dir, "hazardcurve-0.dat")) verify_hazcurve_results(self, self.job, hazcurve_0, end_branch_label=0) # Check mean hazard curves: hazcurve_mean = helpers.demo_file( os.path.join(exp_results_dir, "hazardcurve-mean.dat")) verify_hazcurve_results(self, self.job, hazcurve_mean, statistic_type="mean") # Check hazard map mean 0.02: hazmap_mean_0_02 = helpers.demo_file( os.path.join(exp_results_dir, "hazardmap-0.02-mean.dat")) verify_hazmap_results(self, self.job, load_expected_map(hazmap_mean_0_02), 0.02, "mean") # Check hazard map mean 0.1: hazmap_mean_0_1 = helpers.demo_file( os.path.join(exp_results_dir, "hazardmap-0.1-mean.dat")) verify_hazmap_results(self, self.job, load_expected_map(hazmap_mean_0_1), 0.1, "mean")
def setUp(self): self.job = engine.prepare_job() self.job_profile, self.params, _sections = (engine.import_job_profile( demo_file('simple_fault_demo_hazard/config.gem'), self.job)) self.params['debug'] = 'warn' # Cache the calc proxy data into the kvs: job_ctxt = engine.JobContext(self.params, self.job.id, oq_job_profile=self.job_profile, oq_job=self.job) job_ctxt.to_kvs()
def test_default_validators_classical_job(self): """Test to ensure that a classical always includes the :class:`openquake.job.config.ClassicalValidator`. """ classical_risk_job_path = helpers.demo_file( 'classical_psha_based_risk/config.gem') classical_risk_job = helpers.job_from_file(classical_risk_job_path) validators = config.default_validators(classical_risk_job.sections, classical_risk_job.params) self.assertTrue( any(isinstance(v, ClassicalValidator) for v in validators))
def test_default_validators_scenario_job(self): """Test to ensure that a Scenario job always includes the :class:`openquake.job.config.ScenarioComputationValidator`.""" scenario_job_path = helpers.demo_file('scenario_risk/config.gem') scenario_job = helpers.job_from_file(scenario_job_path) validators = config.default_validators(scenario_job.sections, scenario_job.params) self.assertTrue( any( isinstance(v, ScenarioComputationValidator) for v in validators))
def test__serialize_xml_filenames(self): # Test that the file names of the loss XML artifacts are correct. # See https://bugs.launchpad.net/openquake/+bug/894706. expected_lrc_file_name = ( 'losscurves-block-#%(job_id)s-block#%(block)s.xml') expected_lr_file_name = ( 'losscurves-loss-block-#%(job_id)s-block#%(block)s.xml') cfg_file = helpers.demo_file('classical_psha_based_risk/config.gem') job = engine.prepare_job() job_profile, params, sections = engine.import_job_profile( cfg_file, job) job_ctxt = engine.JobContext(params, job.id, sections=sections, serialize_results_to=['xml', 'db'], oq_job_profile=job_profile, oq_job=job) calculator = ClassicalRiskCalculator(job_ctxt) with helpers.patch('openquake.writer.FileWriter.serialize'): # The 'curves' key in the kwargs just needs to be present; # because of the serialize mock in place above, it doesn't need # to have a real value. # First, we test loss ratio curve output, # then we'll do the same test for loss curve output. # We expect to get a single file path back. [file_path] = calculator._serialize( 0, **dict(curve_mode='loss_ratio', curves=[])) _dir, file_name = os.path.split(file_path) self.assertEqual( expected_lrc_file_name % dict(job_id=job.id, block=0), file_name) # The same test again, except for loss curves this time. [file_path ] = calculator._serialize(0, **dict(curve_mode='loss', curves=[])) _dir, file_name = os.path.split(file_path) self.assertEqual( expected_lr_file_name % dict(job_id=job.id, block=0), file_name)
def test_disagg(self): self.setup_config() os.environ.update(self.orig_env) cp = ConfigParser.SafeConfigParser() cp.read('openquake.cfg.test_bakk') cp.set('nfs', 'base_dir', '/tmp') cp.write(open('openquake.cfg', 'w')) try: self._do_test( helpers.demo_file('disaggregation/config_with_site_model.gem') ) finally: self.teardown_config()
def test_initialize_calls_validate(self): # Test make sure the calculator `initialize` calls # `validate_site_model`. job_ctxt = helpers.prepare_job_context( helpers.demo_file( 'simple_fault_demo_hazard/config_with_site_model.gem')) calc = general.BaseHazardCalculator(job_ctxt) patch_path = 'openquake.calculators.hazard.general.validate_site_model' with helpers.patch(patch_path) as validate_patch: calc.initialize() # validate_site_model itself is tested in another test # here, we just make sure it gets called self.assertEqual(1, validate_patch.call_count)
def test_log_file(self): # Test logging to a file when running bin/oqscript.py. uhs_cfg = helpers.demo_file('uhs/config.gem') log_file = './%s.log' % helpers.random_string() self.assertFalse(os.path.exists(log_file)) ret_code = helpers.run_job( uhs_cfg, ['--log-level', 'debug', '--log-file', log_file]) self.assertEqual(0, ret_code) self.assertTrue(os.path.exists(log_file)) # Make sure there is something in it. self.assertTrue(os.path.getsize(log_file) > 0) os.unlink(log_file)