def test_export_for_scenario(self): target_dir = tempfile.mkdtemp() try: cfg = helpers.demo_file('scenario_hazard/job.ini') # run the calculation to create something to export retcode = helpers.run_hazard_job_sp(cfg, silence=True) self.assertEqual(0, retcode) job = models.OqJob.objects.latest('id') outputs = export_core.get_outputs(job.id) self.assertEqual(1, len(outputs)) # 1 GMF gmf_outputs = outputs.filter(output_type='gmf_scenario') self.assertEqual(1, len(gmf_outputs)) exported_files = hazard.export(gmf_outputs[0].id, target_dir) self.assertEqual(1, len(exported_files)) # Check the file paths exist, is absolute, and the file isn't # empty. f = exported_files[0] self._test_exported_file(f) # Check for the correct number of GMFs in the file: tree = etree.parse(f) self.assertEqual(10, number_of('nrml:gmf', tree)) finally: shutil.rmtree(target_dir)
def export(self, *args, **kwargs): """ If requested by the user, automatically export all result artifacts to the specified format. (NOTE: The only export format supported at the moment is NRML XML. :returns: A list of the export filenames, including the absolute path to each file. """ exported_files = [] logs.LOG.debug('> starting exports') if 'exports' in kwargs and 'xml' in kwargs['exports']: outputs = export_core.get_outputs(self.job.id) for output in outputs: exported_files.extend(hazard_export.export( output.id, self.job.hazard_calculation.export_dir)) for exp_file in exported_files: logs.LOG.debug('exported %s' % exp_file) logs.LOG.debug('< done with exports') return exported_files
def test_export_for_scenario(self): target_dir = tempfile.mkdtemp() try: cfg = helpers.get_data_path('scenario_hazard/job.ini') # run the calculation in process to create something to export os.environ['OQ_NO_DISTRIBUTE'] = '1' try: helpers.run_hazard_job(cfg) finally: del os.environ['OQ_NO_DISTRIBUTE'] job = models.OqJob.objects.latest('id') self.assertEqual(job.status, 'complete') outputs = export_core.get_outputs(job.id) self.assertEqual(1, len(outputs)) # 1 GMF gmf_outputs = outputs.filter(output_type='gmf_scenario') self.assertEqual(1, len(gmf_outputs)) exported_files = check_export(gmf_outputs[0].id, target_dir) self.assertEqual(1, len(exported_files)) # Check the file paths exist, is absolute, and the file isn't # empty. f = exported_files[0] self._test_exported_file(f) # Check for the correct number of GMFs in the file: tree = etree.parse(f) self.assertEqual(20, number_of('nrml:gmf', tree)) finally: shutil.rmtree(target_dir)
def test_export_for_scenario(self): target_dir = tempfile.mkdtemp() try: cfg = helpers.get_data_path('scenario_hazard/job.ini') # run the calculation in process to create something to export with mock.patch.dict(os.environ, {'OQ_NO_DISTRIBUTE': '1'}): helpers.run_job(cfg) job = models.OqJob.objects.latest('id') self.assertEqual(job.status, 'complete') outputs = core.get_outputs(job.id) self.assertEqual(2, len(outputs)) # 1 GMF, 1 SES gmf_outputs = outputs.filter(output_type='gmf_scenario') self.assertEqual(1, len(gmf_outputs)) exported_file = check_export(gmf_outputs[0].id, target_dir) # Check the file paths exist, is absolute, and the file isn't # empty. self._test_exported_file(exported_file) # Check for the correct number of GMFs in the file: tree = etree.parse(exported_file) self.assertEqual(20, number_of('nrml:gmf', tree)) finally: shutil.rmtree(target_dir)
def test_bcr_risk_export(self): # Tests that outputs of a risk classical calculation are # exported target_dir = tempfile.mkdtemp() try: cfg = helpers.demo_file('classical_bcr/job.ini') # run the calculation to create something to export retcode = helpers.run_risk_job_sp(cfg, self.hazard_id, silence=True) self.assertEqual(0, retcode) job = models.OqJob.objects.latest('id') outputs = export_core.get_outputs(job.id) expected_outputs = 1 # 1 bcr distribution self.assertEqual(expected_outputs, len(outputs)) # Export the loss curves: distribution = outputs.filter(output_type='bcr_distribution')[0] rc_files = risk.export(distribution.id, target_dir) self.assertEqual(1, len(rc_files)) for f in rc_files: self._test_exported_file(f) finally: shutil.rmtree(target_dir)
def test_export_for_scenario(self): target_dir = tempfile.mkdtemp() try: cfg = helpers.get_data_path('scenario_hazard/job.ini') # run the calculation in process to create something to export with mock.patch.dict(os.environ, {'OQ_NO_DISTRIBUTE': '1'}): helpers.run_job(cfg) job = models.OqJob.objects.latest('id') self.assertEqual(job.status, 'complete') outputs = core.get_outputs(job.id) gmf_outputs = outputs.filter(ds_key='gmfs') self.assertEqual(1, len(gmf_outputs)) exported_file = check_export(gmf_outputs[0].id, target_dir) # Check the file paths exist, is absolute, and the file isn't # empty. self._test_exported_file(exported_file) # Check for the correct number of GMFs in the file: tree = etree.parse(exported_file) self.assertEqual(20, number_of('nrml:gmf', tree)) finally: shutil.rmtree(target_dir)
def export(self, *args, **kwargs): """ If requested by the user, automatically export all result artifacts to the specified format. (NOTE: The only export format supported at the moment is NRML XML. :returns: A list of the export filenames, including the absolute path to each file. """ exported_files = [] logs.LOG.debug('> starting exports') if 'exports' in kwargs and 'xml' in kwargs['exports']: outputs = export_core.get_outputs(self.job.id) if not self.hc.export_multi_curves: outputs = outputs.exclude(output_type='hazard_curve_multi') for output in outputs: with self.monitor('exporting %s' % output.output_type): fname = hazard_export.export( output.id, self.job.hazard_calculation.export_dir) exported_files.extend(fname) logs.LOG.debug('exported %s' % fname) logs.LOG.debug('< done with exports') return exported_files
def _get_outputs_for_export(self): """ Util function for getting :class:`openquake.engine.db.models.Output` objects to be exported. Gathers all outputs for the job, but filters out `hazard_curve_multi` outputs if this option was turned off in the calculation profile. """ outputs = export_core.get_outputs(self.job.id) if not self.hc.export_multi_curves: outputs = outputs.exclude(output_type='hazard_curve_multi') return outputs
def test_classical_hazard_export(self): # Run a hazard calculation to compute some curves and maps # Call the exporter and verify that files were created # Since the hazard curve XML writer is concerned with correctly # generating XML, we won't test that here. target_dir = tempfile.mkdtemp() try: cfg = helpers.demo_file('simple_fault_demo_hazard/job.ini') # run the calculation to create something to export retcode = helpers.run_hazard_job_sp(cfg, silence=True) self.assertEqual(0, retcode) job = models.OqJob.objects.latest('id') outputs = export_core.get_outputs(job.id) expected_outputs = 18 # 6 hazard curves + 12 hazard maps self.assertEqual(expected_outputs, len(outputs)) # Export the hazard curves: curves = outputs.filter(output_type='hazard_curve') hc_files = [] for curve in curves: hc_files.extend(hazard.export(curve.id, target_dir)) self.assertEqual(6, len(hc_files)) for f in hc_files: self._test_exported_file(f) # Test hazard map export as well. maps = outputs.filter(output_type='hazard_map') hm_files = [] for haz_map in maps: hm_files.extend(hazard.export(haz_map.id, target_dir)) self.assertEqual(12, len(hm_files)) for f in hm_files: self._test_exported_file(f) finally: shutil.rmtree(target_dir)
def test_classical_risk_export(self): # Tests that outputs of a risk classical calculation are exported target_dir = tempfile.mkdtemp() try: cfg = helpers.demo_file('classical_psha_based_risk/job.ini') # run the calculation to create something to export retcode = helpers.run_risk_job_sp(cfg, self.hazard_id, silence=True) self.assertEqual(0, retcode) job = models.OqJob.objects.latest('id') outputs = export_core.get_outputs(job.id) expected_outputs = 4 # 1 loss curve set + 3 loss curve map set self.assertEqual(expected_outputs, len(outputs)) # Export the loss curves: curves = outputs.filter(output_type='loss_curve') rc_files = [] for curve in curves: rc_files.extend(risk.export(curve.id, target_dir)) self.assertEqual(1, len(rc_files)) for f in rc_files: self._test_exported_file(f) # Test loss map export as well. maps = outputs.filter(output_type='loss_map') lm_files = sum( [risk.export(loss_map.id, target_dir) for loss_map in maps], []) self.assertEqual(3, len(lm_files)) for f in lm_files: self._test_exported_file(f) finally: shutil.rmtree(target_dir)
def test_disagg_hazard_export(self): target_dir = tempfile.mkdtemp() try: cfg = helpers.get_data_path('disaggregation/job.ini') # run the calculation in process to create something to export os.environ['OQ_NO_DISTRIBUTE'] = '1' try: helpers.run_job(cfg) finally: del os.environ['OQ_NO_DISTRIBUTE'] job = models.OqJob.objects.latest('id') self.assertEqual(job.status, 'complete') outputs = export_core.get_outputs(job.id) # Test curve export: curves = outputs.filter(output_type='hazard_curve') self.assertEqual(4, len(curves)) curve_files = [] for curve in curves: curve_files.append(check_export(curve.id, target_dir)) self.assertEqual(4, len(curve_files)) for f in curve_files: self._test_exported_file(f) # Test disagg matrix export: matrices = outputs.filter(output_type='disagg_matrix') self.assertEqual(8, len(matrices)) disagg_files = [] for matrix in matrices: disagg_files.append(check_export(matrix.id, target_dir)) self.assertEqual(8, len(disagg_files)) for f in disagg_files: self._test_exported_file(f) finally: shutil.rmtree(target_dir)
def test_disagg_hazard_export(self): target_dir = tempfile.mkdtemp() try: cfg = helpers.get_data_path('disaggregation/job.ini') # run the calculation in process to create something to export os.environ['OQ_NO_DISTRIBUTE'] = '1' try: helpers.run_job(cfg) finally: del os.environ['OQ_NO_DISTRIBUTE'] job = models.OqJob.objects.latest('id') self.assertEqual(job.status, 'complete') outputs = core.get_outputs(job.id) # Test curve export: curves = outputs.filter(output_type='hazard_curve') self.assertEqual(4, len(curves)) curve_files = [] for curve in curves: curve_files.append(check_export(curve.id, target_dir)) self.assertEqual(4, len(curve_files)) for f in curve_files: self._test_exported_file(f) # Test disagg matrix export: matrices = outputs.filter(output_type='disagg_matrix') self.assertEqual(8, len(matrices)) disagg_files = [] for matrix in matrices: disagg_files.append(check_export(matrix.id, target_dir)) self.assertEqual(8, len(disagg_files)) for f in disagg_files: self._test_exported_file(f) finally: shutil.rmtree(target_dir)
def test_disagg_hazard_export(self): target_dir = tempfile.mkdtemp() try: cfg = helpers.demo_file('disaggregation/job.ini') retcode = helpers.run_hazard_job_sp(cfg, silence=True) self.assertEqual(0, retcode) job = models.OqJob.objects.latest('id') outputs = export_core.get_outputs(job.id) # Test curve export: curves = outputs.filter(output_type='hazard_curve') self.assertEqual(4, len(curves)) curve_files = [] for curve in curves: curve_files.extend(hazard.export(curve.id, target_dir)) self.assertEqual(4, len(curve_files)) for f in curve_files: self._test_exported_file(f) # Test disagg matrix export: matrices = outputs.filter(output_type='disagg_matrix') self.assertEqual(8, len(matrices)) disagg_files = [] for matrix in matrices: disagg_files.extend(hazard.export(matrix.id, target_dir)) self.assertEqual(8, len(disagg_files)) for f in disagg_files: self._test_exported_file(f) finally: shutil.rmtree(target_dir)
def test_export_for_event_based(self): # Run an event-based hazard calculation to compute SESs and GMFs # Call the exporters for both SES and GMF results and verify that # files were created # Since the XML writers (in `openquake.nrmllib.writers`) are concerned # with correctly generating the XML, we don't test that here... # but we should still have an end-to-end QA test. target_dir = tempfile.mkdtemp() try: cfg = helpers.get_data_path('event_based_hazard/job.ini') # run the calculation in process to create something to export os.environ['OQ_NO_DISTRIBUTE'] = '1' try: helpers.run_hazard_job(cfg) finally: del os.environ['OQ_NO_DISTRIBUTE'] job = models.OqJob.objects.latest('id') self.assertEqual(job.status, 'complete') outputs = export_core.get_outputs(job.id) # 2 GMFs, 2 SESs, 1 complete logic tree SES, 1 complete LT GMF, # ((2 imts * 2 realizations) # + ((2 imts + 1 multi) * (1 mean + 3 quantiles)) # hazard curves, # (2 poes * 2 imts * 2 realizations) # + (2 poes * 2 imts * (1 mean + 3 quantiles)) hazard maps # Total: 42 self.assertEqual(46, len(outputs)) ####### # SESs: ses_outputs = outputs.filter(output_type='ses') self.assertEqual(2, len(ses_outputs)) exported_files = [] for ses_output in ses_outputs: out_file = check_export(ses_output.id, target_dir) exported_files.append(out_file) self.assertEqual(2, len(exported_files)) for f in exported_files: self._test_exported_file(f) ################## # Complete LT SES: [complete_lt_ses] = outputs.filter(output_type='complete_lt_ses') exported_file = check_export(complete_lt_ses.id, target_dir) self._test_exported_file(exported_file) ####### # GMFs: gmf_outputs = outputs.filter(output_type='gmf') self.assertEqual(2, len(gmf_outputs)) exported_files = [] for gmf_output in gmf_outputs: out_file = check_export(gmf_output.id, target_dir) exported_files.append(out_file) self.assertEqual(2, len(exported_files)) # Check the file paths exist, are absolute, and the files aren't # empty. for f in exported_files: self._test_exported_file(f) ################## # Complete LT GMF: [complete_lt_gmf] = outputs.filter(output_type='complete_lt_gmf') exported_file = check_export(complete_lt_gmf.id, target_dir) self._test_exported_file(exported_file) # Check for the correct number of GMFs in the file: tree = etree.parse(exported_file) # NB: the number of generated gmfs depends on the number # of ruptures, which is stochastic number; even having fixed # the seed, it will change by changing the order in which the # stochastic functions are called; a test relying on that # precise number would be fragile, this is why here we just # check that there are gmfs (MS) self.assertGreater(number_of('nrml:gmf', tree), 0) ################ # Hazard curves: haz_curves = outputs.filter(output_type='hazard_curve') self.assertEqual(12, haz_curves.count()) for curve in haz_curves: exported_file = check_export(curve.id, target_dir) self._test_exported_file(exported_file) ############## # Hazard maps: haz_maps = outputs.filter(output_type='hazard_map') self.assertEqual(24, haz_maps.count()) for hmap in haz_maps: exported_file = check_export(hmap.id, target_dir) self._test_exported_file(exported_file) finally: shutil.rmtree(target_dir)
def test_classical_hazard_export(self): # Run a hazard calculation to compute some curves and maps # Call the exporter and verify that files were created # Since the hazard curve XML writer is concerned with correctly # generating XML, we won't test that here. target_dir = tempfile.mkdtemp() try: cfg = helpers.get_data_path('simple_fault_demo_hazard/job.ini') # run the calculation to create something to export helpers.run_job(cfg) job = models.OqJob.objects.latest('id') self.assertEqual(job.status, 'complete') outputs = core.get_outputs(job.id) # 10 hazard curves, 20 maps, 10 uhs, 5 multi curves expected_outputs = 45 self.assertEqual(expected_outputs, outputs.count()) # Number of curves: # (2 imts * 2 realizations) # + (2 imts * (1 mean + 2 quantiles) # = 10 curves = outputs.filter(output_type='hazard_curve') self.assertEqual(10, curves.count()) # Number of multi-curves # (2 realizations + 1 mean + 2 quantiles) multi_curves = outputs.filter(output_type="hazard_curve_multi") self.assertEqual(5, multi_curves.count()) # Number of maps: # (2 poes * 2 imts * 2 realizations) # + (2 poes * 2 imts * (1 mean + 2 quantiles)) # = 20 # Number of UHS: maps = outputs.filter(output_type='hazard_map') self.assertEqual(20, maps.count()) # Number of UHS: # (20 maps_PGA_SA / 2 poes) # = 10 uhs = outputs.filter(output_type='uh_spectra') self.assertEqual(10, uhs.count()) # Test hazard curve export: hc_files = [] for curve in curves: hc_files.append(check_export(curve.id, target_dir)) self.assertEqual(10, len(hc_files)) # Test multi hazard curve export: hc_files = [] for curve in multi_curves: hc_files.append(check_export(curve.id, target_dir)) self.assertEqual(5, len(hc_files)) for f in hc_files: self._test_exported_file(f) # Test hazard map export: hm_files = [] for haz_map in maps: hm_files.append(check_export(haz_map.id, target_dir)) self.assertEqual(20, len(hm_files)) for f in hm_files: self._test_exported_file(f) # Test UHS export: uhs_files = [] for u in uhs: uhs_files.append(check_export(u.id, target_dir)) for f in uhs_files: self._test_exported_file(f) finally: shutil.rmtree(target_dir)
def test_export_for_event_based(self): # Run an event-based hazard calculation to compute SESs and GMFs # Call the exporters for both SES and GMF results and verify that # files were created # Since the XML writers (in `openquake.commonlib`) are concerned # with correctly generating the XML, we don't test that here target_dir = tempfile.mkdtemp() try: cfg = helpers.get_data_path('event_based_hazard/job.ini') # run the calculation in process to create something to export with mock.patch.dict(os.environ, {'OQ_NO_DISTRIBUTE': '1'}): job = helpers.run_job(cfg, maximum_distance=1, ses_per_logic_tree_path=1, investigation_time=12, number_of_logic_tree_samples=1).job self.assertEqual(job.status, 'complete') dstore = datastore.DataStore(job.id) # 1 SES + 1 GMF + 1 hazard_curve_multi + 2 hazard_curve + # 4 hazard maps (with poes 0.1, 0.2 and IMT PGA, SA(0.1)) outputs = core.get_outputs(job.id) # SESs ses_outputs = outputs.filter(ds_key='sescollection') self.assertEqual(1, len(ses_outputs)) exported_files = [] for ses_output in ses_outputs: out_file = check_export(ses_output.id, target_dir) exported_files.append(out_file) self.assertEqual(1, len(exported_files)) for f in exported_files: self._test_exported_file(f) # GMFs gmf_outputs = outputs.filter(ds_key='gmfs') self.assertEqual(1, len(gmf_outputs)) exported_files = [] for gmf_output in gmf_outputs: out_file = check_export(gmf_output.id, target_dir) exported_files.append(out_file) self.assertEqual(1, len(exported_files)) # Check the file paths exist, are absolute, and the files aren't # empty. for f in exported_files: self._test_exported_file(f) # check the exact values of the GMFs gmfs = writers.write_csv( io.StringIO(), dstore['gmfs']['col00'].value).encode('utf8') self.check_file_content('expected_gmfset_1.txt', gmfs) # Hazard curves haz_curves = outputs.filter(ds_key='hcurves') self.assertEqual(1, haz_curves.count()) for curve in haz_curves: exported_file = check_export(curve.id, target_dir) self._test_exported_file(exported_file) # Hazard maps haz_maps = outputs.filter(ds_key='hmaps') self.assertEqual(1, haz_maps.count()) for hmap in haz_maps: exported_file = check_export(hmap.id, target_dir) self._test_exported_file(exported_file) finally: shutil.rmtree(target_dir)
def test_export_for_event_based(self): # Run an event-based hazard calculation to compute SESs and GMFs # Call the exporters for both SES and GMF results and verify that # files were created # Since the XML writers (in `openquake.nrmllib.writers`) are concerned # with correctly generating the XML, we don't test that here... # but we should still have an end-to-end QA test. target_dir = tempfile.mkdtemp() try: cfg = helpers.get_data_path('event_based_hazard/job.ini') # run the calculation in process to create something to export with mock.patch.dict(os.environ, {'OQ_NO_DISTRIBUTE': '1'}): job = helpers.run_job(cfg, maximum_distance=1, ses_per_logic_tree_path=1) self.assertEqual(job.status, 'complete') outputs = export_core.get_outputs(job.id) # 2 GMFs, 1 SES, # ((2 imts * 2 realizations) self.assertEqual(45, len(outputs)) ####### # SESs: ses_outputs = outputs.filter(output_type='ses') self.assertEqual(1, len(ses_outputs)) exported_files = [] for ses_output in ses_outputs: out_file = check_export(ses_output.id, target_dir) exported_files.append(out_file) self.assertEqual(1, len(exported_files)) for f in exported_files: self._test_exported_file(f) ####### # GMFs: gmf_outputs = outputs.filter(output_type='gmf') self.assertEqual(2, len(gmf_outputs)) exported_files = [] for gmf_output in gmf_outputs: out_file = check_export(gmf_output.id, target_dir) exported_files.append(out_file) self.assertEqual(2, len(exported_files)) # Check the file paths exist, are absolute, and the files aren't # empty. for f in exported_files: self._test_exported_file(f) # check the exact values of the GMFs [gmfset1] = gmf_outputs[0].gmf [gmfset2] = gmf_outputs[1].gmf self.check_file_content('expected_gmfset_1.txt', str(gmfset1)) self.check_file_content('expected_gmfset_2.txt', str(gmfset2)) ################ # Hazard curves: haz_curves = outputs.filter(output_type='hazard_curve') self.assertEqual(12, haz_curves.count()) for curve in haz_curves: exported_file = check_export(curve.id, target_dir) self._test_exported_file(exported_file) ############## # Hazard maps: haz_maps = outputs.filter(output_type='hazard_map') self.assertEqual(24, haz_maps.count()) for hmap in haz_maps: exported_file = check_export(hmap.id, target_dir) self._test_exported_file(exported_file) finally: shutil.rmtree(target_dir)
def test_export_for_event_based(self): # Run an event-based hazard calculation to compute SESs and GMFs # Call the exporters for both SES and GMF results and verify that # files were created # Since the XML writers (in `openquake.commonlib`) are concerned # with correctly generating the XML, we don't test that here target_dir = tempfile.mkdtemp() try: cfg = helpers.get_data_path('event_based_hazard/job.ini') # run the calculation in process to create something to export with mock.patch.dict(os.environ, {'OQ_NO_DISTRIBUTE': '1'}): job = helpers.run_job(cfg, maximum_distance=1, ses_per_logic_tree_path=1, number_of_logic_tree_samples=1).job self.assertEqual(job.status, 'complete') # 1 SES + 1 GMF + 1 hazard_curve_multi + 2 hazard_curve + # 4 hazard maps (with poes 0.1, 0.2 and IMT PGA, SA(0.1)) outputs = core.get_outputs(job.id) self.assertEqual(9, len(outputs)) # SESs ses_outputs = outputs.filter(output_type='ses') self.assertEqual(1, len(ses_outputs)) exported_files = [] for ses_output in ses_outputs: out_file = check_export(ses_output.id, target_dir) exported_files.append(out_file) self.assertEqual(1, len(exported_files)) for f in exported_files: self._test_exported_file(f) # GMFs gmf_outputs = outputs.filter(output_type='gmf') self.assertEqual(1, len(gmf_outputs)) exported_files = [] for gmf_output in gmf_outputs: out_file = check_export(gmf_output.id, target_dir) exported_files.append(out_file) self.assertEqual(1, len(exported_files)) # Check the file paths exist, are absolute, and the files aren't # empty. for f in exported_files: self._test_exported_file(f) # check the exact values of the GMFs [gmfset1] = gmf_outputs[0].gmf self.check_file_content('expected_gmfset_1.txt', str(gmfset1)) # Hazard curves haz_curves = outputs.filter(output_type='hazard_curve') self.assertEqual(2, haz_curves.count()) for curve in haz_curves: exported_file = check_export(curve.id, target_dir) self._test_exported_file(exported_file) # Hazard maps haz_maps = outputs.filter(output_type='hazard_map') self.assertEqual(4, haz_maps.count()) for hmap in haz_maps: exported_file = check_export(hmap.id, target_dir) self._test_exported_file(exported_file) finally: shutil.rmtree(target_dir)
def test_export_for_event_based(self): # Run an event-based hazard calculation to compute SESs and GMFs # Call the exporters for both SES and GMF results and verify that # files were created # Since the XML writers (in `openquake.nrmllib.writers`) are concerned # with correctly generating the XML, we don't test that here... # but we should still have an end-to-end QA test. target_dir = tempfile.mkdtemp() try: cfg = helpers.get_data_path('event_based_hazard/job.ini') # run the calculation in process to create something to export os.environ['OQ_NO_DISTRIBUTE'] = '1' try: helpers.run_hazard_job(cfg) finally: del os.environ['OQ_NO_DISTRIBUTE'] job = models.OqJob.objects.latest('id') self.assertEqual(job.status, 'complete') outputs = export_core.get_outputs(job.id) # 2 GMFs, 2 SESs, 1 complete logic tree SES, 1 complete LT GMF, # ((2 imts * 2 realizations) # + ((2 imts + 1 multi) * (1 mean + 3 quantiles)) # hazard curves, # (2 poes * 2 imts * 2 realizations) # + (2 poes * 2 imts * (1 mean + 3 quantiles)) hazard maps # Total: 42 self.assertEqual(46, len(outputs)) ####### # SESs: ses_outputs = outputs.filter(output_type='ses') self.assertEqual(2, len(ses_outputs)) exported_files = [] for ses_output in ses_outputs: files = check_export(ses_output.id, target_dir) exported_files.extend(files) self.assertEqual(2, len(exported_files)) for f in exported_files: self._test_exported_file(f) ################## # Complete LT SES: [complete_lt_ses] = outputs.filter(output_type='complete_lt_ses') [exported_file] = check_export(complete_lt_ses.id, target_dir) self._test_exported_file(exported_file) ####### # GMFs: gmf_outputs = outputs.filter(output_type='gmf') self.assertEqual(2, len(gmf_outputs)) exported_files = [] for gmf_output in gmf_outputs: files = check_export(gmf_output.id, target_dir) exported_files.extend(files) self.assertEqual(2, len(exported_files)) # Check the file paths exist, are absolute, and the files aren't # empty. for f in exported_files: self._test_exported_file(f) ################## # Complete LT GMF: [complete_lt_gmf] = outputs.filter(output_type='complete_lt_gmf') [exported_file] = check_export(complete_lt_gmf.id, target_dir) self._test_exported_file(exported_file) # Check for the correct number of GMFs in the file: tree = etree.parse(exported_file) # NB: the number of generated gmfs depends on the number # of ruptures, which is stochastic number; even having fixed # the seed, it will change by changing the order in which the # stochastic functions are called; a test relying on that # precise number would be fragile, this is why here we just # check that there are gmfs (MS) self.assertGreater(number_of('nrml:gmf', tree), 0) ################ # Hazard curves: haz_curves = outputs.filter(output_type='hazard_curve') self.assertEqual(12, haz_curves.count()) for curve in haz_curves: [exported_file] = check_export(curve.id, target_dir) self._test_exported_file(exported_file) ############## # Hazard maps: haz_maps = outputs.filter(output_type='hazard_map') self.assertEqual(24, haz_maps.count()) for hmap in haz_maps: [exported_file] = check_export(hmap.id, target_dir) self._test_exported_file(exported_file) finally: shutil.rmtree(target_dir)
def test_export_for_event_based(self): # Run an event-based hazard calculation to compute SESs and GMFs # Call the exporters for both SES and GMF results and verify that # files were created # Since the XML writers (in `openquake.nrmllib.writers`) are concerned # with correctly generating the XML, we don't test that here... # but we should still have an end-to-end QA test. target_dir = tempfile.mkdtemp() try: cfg = helpers.demo_file('event_based_hazard/job.ini') # run the calculation to create something to export retcode = helpers.run_hazard_job_sp(cfg, silence=True) self.assertEqual(0, retcode) job = models.OqJob.objects.latest('id') outputs = export_core.get_outputs(job.id) # 2 GMFs, 2 SESs, 1 complete logic tree SES, 1 complete LT GMF, # and 4 hazard curve collections self.assertEqual(18, len(outputs)) ####### # SESs: ses_outputs = outputs.filter(output_type='ses') self.assertEqual(2, len(ses_outputs)) exported_files = [] for ses_output in ses_outputs: files = hazard.export(ses_output.id, target_dir) exported_files.extend(files) self.assertEqual(2, len(exported_files)) for f in exported_files: self._test_exported_file(f) ################## # Complete LT SES: [complete_lt_ses] = outputs.filter(output_type='complete_lt_ses') [exported_file] = hazard.export(complete_lt_ses.id, target_dir) self._test_exported_file(exported_file) ####### # GMFs: gmf_outputs = outputs.filter(output_type='gmf') self.assertEqual(2, len(gmf_outputs)) exported_files = [] for gmf_output in gmf_outputs: files = hazard.export(gmf_output.id, target_dir) exported_files.extend(files) self.assertEqual(2, len(exported_files)) # Check the file paths exist, are absolute, and the files aren't # empty. for f in exported_files: self._test_exported_file(f) ################## # Complete LT GMF: [complete_lt_gmf] = outputs.filter(output_type='complete_lt_gmf') [exported_file] = hazard.export(complete_lt_gmf.id, target_dir) self._test_exported_file(exported_file) # Check for the correct number of GMFs in the file: tree = etree.parse(exported_file) self.assertEqual(442, number_of('nrml:gmf', tree)) ################ # Hazard curves: haz_curves = outputs.filter(output_type='hazard_curve') for curve in haz_curves: [exported_file] = hazard.export(curve.id, target_dir) self._test_exported_file(exported_file) finally: shutil.rmtree(target_dir)
def test_event_based_risk_export(self): # Tests that outputs of a risk classical calculation are exported target_dir = tempfile.mkdtemp() try: # use get_risk_job to create a fake GmfCollection job, _ = helpers.get_risk_job('event_based_risk/job.ini', 'event_based_hazard/job.ini', 'gmf') cfg = helpers.demo_file('event_based_risk/job.ini') # run the calculation to create something to export # at the moment, only gmf for a specific realization are # supported as hazard input retcode = helpers.run_risk_job_sp( cfg, silence=True, hazard_id=job.risk_calculation.hazard_output.id) self.assertEqual(0, retcode) job = models.OqJob.objects.latest('id') outputs = export_core.get_outputs(job.id) # 1 loss curve set + 3 loss curve map set + 1 insured + 1 aggregate expected_outputs = 6 self.assertEqual(expected_outputs, len(outputs)) # Export the loss curves... curves = outputs.filter(output_type='loss_curve') rc_files = [] for curve in curves: rc_files.extend(risk.export(curve.id, target_dir)) self.assertEqual(1, len(rc_files)) for f in rc_files: self._test_exported_file(f) # ... loss map ... maps = outputs.filter(output_type='loss_map') lm_files = sum( [risk.export(loss_map.id, target_dir) for loss_map in maps], []) self.assertEqual(3, len(lm_files)) for f in lm_files: self._test_exported_file(f) # ... aggregate losses... maps = outputs.filter(output_type='agg_loss_curve') lm_files = sum( [risk.export(loss_map.id, target_dir) for loss_map in maps], []) self.assertEqual(1, len(lm_files)) for f in lm_files: self._test_exported_file(f) # and insured losses. maps = outputs.filter(output_type='ins_loss_curve') lm_files = sum( [risk.export(loss_map.id, target_dir) for loss_map in maps], []) self.assertEqual(1, len(lm_files)) for f in lm_files: self._test_exported_file(f) finally: shutil.rmtree(target_dir)
def test_export_for_event_based(self): # Run an event-based hazard calculation to compute SESs and GMFs # Call the exporters for both SES and GMF results and verify that # files were created # Since the XML writers (in `openquake.nrmllib.writers`) are concerned # with correctly generating the XML, we don't test that here... # but we should still have an end-to-end QA test. target_dir = tempfile.mkdtemp() try: cfg = helpers.get_data_path('event_based_hazard/job.ini') # run the calculation in process to create something to export os.environ['OQ_NO_DISTRIBUTE'] = '1' try: helpers.run_hazard_job(cfg) finally: del os.environ['OQ_NO_DISTRIBUTE'] job = models.OqJob.objects.latest('id') self.assertEqual(job.status, 'complete') outputs = export_core.get_outputs(job.id) # 2 GMFs, 2 SESs, # ((2 imts * 2 realizations) # + ((2 imts + 1 multi) * (1 mean + 3 quantiles)) # hazard curves, # (2 poes * 2 imts * 2 realizations) # + (2 poes * 2 imts * (1 mean + 3 quantiles)) hazard maps # Total: 42 self.assertEqual(44, len(outputs)) ####### # SESs: ses_outputs = outputs.filter(output_type='ses') self.assertEqual(2, len(ses_outputs)) exported_files = [] for ses_output in ses_outputs: out_file = check_export(ses_output.id, target_dir) exported_files.append(out_file) self.assertEqual(2, len(exported_files)) for f in exported_files: self._test_exported_file(f) ####### # GMFs: gmf_outputs = outputs.filter(output_type='gmf') self.assertEqual(2, len(gmf_outputs)) exported_files = [] for gmf_output in gmf_outputs: out_file = check_export(gmf_output.id, target_dir) exported_files.append(out_file) self.assertEqual(2, len(exported_files)) # Check the file paths exist, are absolute, and the files aren't # empty. for f in exported_files: self._test_exported_file(f) ################ # Hazard curves: haz_curves = outputs.filter(output_type='hazard_curve') self.assertEqual(12, haz_curves.count()) for curve in haz_curves: exported_file = check_export(curve.id, target_dir) self._test_exported_file(exported_file) ############## # Hazard maps: haz_maps = outputs.filter(output_type='hazard_map') self.assertEqual(24, haz_maps.count()) for hmap in haz_maps: exported_file = check_export(hmap.id, target_dir) self._test_exported_file(exported_file) finally: shutil.rmtree(target_dir)