コード例 #1
0
ファイル: hazard_test.py プロジェクト: acerisara/oq-engine
    def test_export_for_scenario(self):
        target_dir = tempfile.mkdtemp()

        try:
            cfg = helpers.get_data_path('scenario_hazard/job.ini')
            # run the calculation in process to create something to export
            with mock.patch.dict(os.environ, {'OQ_NO_DISTRIBUTE': '1'}):
                helpers.run_job(cfg)
            job = models.OqJob.objects.latest('id')
            self.assertEqual(job.status, 'complete')

            outputs = core.get_outputs(job.id)

            gmf_outputs = outputs.filter(ds_key='gmfs')
            self.assertEqual(1, len(gmf_outputs))

            exported_file = check_export(gmf_outputs[0].id, target_dir)

            # Check the file paths exist, is absolute, and the file isn't
            # empty.
            self._test_exported_file(exported_file)

            # Check for the correct number of GMFs in the file:
            tree = etree.parse(exported_file)
            self.assertEqual(20, number_of('nrml:gmf', tree))
        finally:
            shutil.rmtree(target_dir)
コード例 #2
0
ファイル: hazard_test.py プロジェクト: vup1120/oq-engine
    def test_export_for_scenario(self):
        target_dir = tempfile.mkdtemp()

        try:
            cfg = helpers.get_data_path("scenario_hazard/job.ini")

            # run the calculation in process to create something to export
            os.environ["OQ_NO_DISTRIBUTE"] = "1"
            try:
                helpers.run_job(cfg)
            finally:
                del os.environ["OQ_NO_DISTRIBUTE"]
            job = models.OqJob.objects.latest("id")
            self.assertEqual(job.status, "complete")

            outputs = export_core.get_outputs(job.id)

            self.assertEqual(1, len(outputs))  # 1 GMF

            gmf_outputs = outputs.filter(output_type="gmf_scenario")
            self.assertEqual(1, len(gmf_outputs))

            exported_file = check_export(gmf_outputs[0].id, target_dir)

            # Check the file paths exist, is absolute, and the file isn't
            # empty.
            self._test_exported_file(exported_file)

            # Check for the correct number of GMFs in the file:
            tree = etree.parse(exported_file)
            self.assertEqual(20, number_of("nrml:gmf", tree))
        finally:
            shutil.rmtree(target_dir)
コード例 #3
0
    def test_export_for_scenario(self):
        target_dir = tempfile.mkdtemp()

        try:
            cfg = helpers.get_data_path('scenario_hazard/job.ini')
            # run the calculation in process to create something to export
            with mock.patch.dict(os.environ, {'OQ_NO_DISTRIBUTE': '1'}):
                helpers.run_job(cfg)
            job = models.OqJob.objects.latest('id')
            self.assertEqual(job.status, 'complete')

            outputs = core.get_outputs(job.id)

            self.assertEqual(2, len(outputs))  # 1 GMF, 1 SES

            gmf_outputs = outputs.filter(output_type='gmf_scenario')
            self.assertEqual(1, len(gmf_outputs))

            exported_file = check_export(gmf_outputs[0].id, target_dir)

            # Check the file paths exist, is absolute, and the file isn't
            # empty.
            self._test_exported_file(exported_file)

            # Check for the correct number of GMFs in the file:
            tree = etree.parse(exported_file)
            self.assertEqual(20, number_of('nrml:gmf', tree))
        finally:
            shutil.rmtree(target_dir)
コード例 #4
0
ファイル: core_test.py プロジェクト: monellid/oq-engine
 def test(self):
     cfg = helpers.get_data_path('classical_job.ini')
     with mock.patch.dict(os.environ, {'OQ_NO_DISTRIBUTE': '1'}), \
             mock.patch('openquake.engine.logs.LOG.warn') as warn:
         # using a small maximum distance of 1 km, so that no sources
         # are found, and checking that no realizations are generated
         helpers.run_job(cfg, maximum_distance=1)
         self.assertEqual(warn.call_args[0][0],
                          'No realizations for hazard_calculation_id=%d')
コード例 #5
0
ファイル: general_test.py プロジェクト: marmarques/oq-engine
 def test(self):
     cfg = helpers.get_data_path('simple_fault_demo_hazard/job.ini')
     with mock.patch('openquake.engine.logs.LOG.warn') as warn:
         helpers.run_job(cfg, number_of_logic_tree_samples=1,
                         quantile_hazard_curves='0.1 0.2',
                         hazard_maps=None, uniform_hazard_spectra=None)
     msg = warn.call_args[0][0]
     self.assertEqual(msg, 'There is only one realization, the configuration'
                      ' parameter quantile_hazard_curves should not be set')
コード例 #6
0
 def test(self):
     cfg = helpers.get_data_path('simple_fault_demo_hazard/job.ini')
     with mock.patch('openquake.engine.logs.LOG.warn') as warn:
         helpers.run_job(cfg,
                         number_of_logic_tree_samples=1,
                         quantile_hazard_curves='0.1 0.2',
                         hazard_maps='',
                         uniform_hazard_spectra='')
     msg = warn.call_args[0][0]
     self.assertEqual(
         msg, 'There is only one realization, the configuration'
         ' parameter quantile_hazard_curves should not be set')
コード例 #7
0
ファイル: __init__.py プロジェクト: ChristieHale/oq-engine
    def get_hazard_job(self):
        try:
            job = models.JobParam.objects.filter(
                name='description',
                value__contains=self.hazard_calculation_fixture,
                job__status="complete").latest('id').job
        except ObjectDoesNotExist:
            warnings.warn("Computing Hazard input from scratch")
            job = helpers.run_job(
                self._test_path('job_haz.ini')).job
            self.assertEqual('complete', job.status)
        else:
            warnings.warn("Using existing Hazard input")

        if self.save_load:
            # Close the opened transactions
            saved_calculation = save_hazards.main(job.id)

            # FIXME Here on, to avoid deadlocks due to stale
            # transactions, we commit all the opened transactions. We
            # should find who is responsible for the eventual opened
            # transaction
            connection = models.getcursor('job_init').connection
            if connection is not None:
                connection.commit()

            [load_calculation] = load_hazards.hazard_load(
                models.getcursor('admin').connection, saved_calculation)
            return models.OqJob.objects.get(pk=load_calculation)
        else:
            return job
コード例 #8
0
    def get_hazard_job(self):
        try:
            job = models.JobParam.objects.filter(
                name='description',
                value__contains=self.hazard_calculation_fixture,
                job__status="complete").latest('id').job
        except ObjectDoesNotExist:
            warnings.warn("Computing Hazard input from scratch")
            job = helpers.run_job(
                self._test_path('job_haz.ini'))
            self.assertEqual('complete', job.status)
        else:
            warnings.warn("Using existing Hazard input")

        if self.save_load:
            # Close the opened transactions
            saved_calculation = save_hazards.main(job.id)

            # FIXME Here on, to avoid deadlocks due to stale
            # transactions, we commit all the opened transactions. We
            # should find who is responsible for the eventual opened
            # transaction
            connection = models.getcursor('job_init').connection
            if connection is not None:
                connection.commit()

            [load_calculation] = load_hazards.hazard_load(
                models.getcursor('admin').connection, saved_calculation)
            return models.OqJob.objects.get(pk=load_calculation)
        else:
            return job
コード例 #9
0
ファイル: __init__.py プロジェクト: MohsenKohrangi/oq-engine
    def get_hazard_job(self):
        if not self._get_queryset().exists():
            warnings.warn("Computing Hazard input from scratch")
            job = helpers.run_job(
                self._test_path('job_haz.ini'))
            self.assertEqual('complete', job.status)
        else:
            warnings.warn("Using existing Hazard input")
            job = self._get_queryset().latest('oqjob__last_update').oqjob

        if self.save_load:
            # Close the opened transactions
            saved_calculation = save_hazards.main(job.hazard_calculation.id)

            # FIXME Here on, to avoid deadlocks due to stale
            # transactions, we commit all the opened transactions. We
            # should find who is responsible for the eventual opened
            # transaction
            connection = models.getcursor('job_init').connection
            if connection is not None:
                connection.commit()

            [load_calculation] = load_hazards.hazard_load(
                models.getcursor('admin').connection, saved_calculation)
            return models.OqJob.objects.get(
                hazard_calculation__id=load_calculation)
        else:
            return job
コード例 #10
0
 def test(self):
     # The bug can be reproduced with any hazard calculation profile which
     # the following parameters set:
     #
     # * number_of_logic_tree_samples = 1
     # * mean_hazard_curves = false
     # * quantile_hazard_curves =
     # * poes = at least one PoE
     cfg = helpers.get_data_path("calculators/hazard/classical/haz_map_1rlz_no_stats.ini")
     job = helpers.run_job(cfg)
     self.assertEqual(job.status, "complete")
コード例 #11
0
    def test_disagg_hazard_export(self):
        target_dir = tempfile.mkdtemp()

        try:
            cfg = helpers.get_data_path('disaggregation/job.ini')

            # run the calculation in process to create something to export
            os.environ['OQ_NO_DISTRIBUTE'] = '1'
            try:
                helpers.run_job(cfg)
            finally:
                del os.environ['OQ_NO_DISTRIBUTE']

            job = models.OqJob.objects.latest('id')
            self.assertEqual(job.status, 'complete')

            outputs = core.get_outputs(job.id)

            # Test curve export:
            curves = outputs.filter(output_type='hazard_curve')
            self.assertEqual(4, len(curves))
            curve_files = []
            for curve in curves:
                curve_files.append(check_export(curve.id, target_dir))

            self.assertEqual(4, len(curve_files))
            for f in curve_files:
                self._test_exported_file(f)

            # Test disagg matrix export:
            matrices = outputs.filter(output_type='disagg_matrix')
            self.assertEqual(8, len(matrices))
            disagg_files = []
            for matrix in matrices:
                disagg_files.append(check_export(matrix.id, target_dir))

            self.assertEqual(8, len(disagg_files))
            for f in disagg_files:
                self._test_exported_file(f)
        finally:
            shutil.rmtree(target_dir)
コード例 #12
0
ファイル: hazard_test.py プロジェクト: marmarques/oq-engine
    def test_disagg_hazard_export(self):
        target_dir = tempfile.mkdtemp()

        try:
            cfg = helpers.get_data_path('disaggregation/job.ini')

            # run the calculation in process to create something to export
            os.environ['OQ_NO_DISTRIBUTE'] = '1'
            try:
                helpers.run_job(cfg)
            finally:
                del os.environ['OQ_NO_DISTRIBUTE']

            job = models.OqJob.objects.latest('id')
            self.assertEqual(job.status, 'complete')

            outputs = export_core.get_outputs(job.id)

            # Test curve export:
            curves = outputs.filter(output_type='hazard_curve')
            self.assertEqual(4, len(curves))
            curve_files = []
            for curve in curves:
                curve_files.append(check_export(curve.id, target_dir))

            self.assertEqual(4, len(curve_files))
            for f in curve_files:
                self._test_exported_file(f)

            # Test disagg matrix export:
            matrices = outputs.filter(output_type='disagg_matrix')
            self.assertEqual(8, len(matrices))
            disagg_files = []
            for matrix in matrices:
                disagg_files.append(check_export(matrix.id, target_dir))

            self.assertEqual(8, len(disagg_files))
            for f in disagg_files:
                self._test_exported_file(f)
        finally:
            shutil.rmtree(target_dir)
コード例 #13
0
 def test(self):
     # The bug can be reproduced with any hazard calculation profile which
     # the following parameters set:
     #
     # * number_of_logic_tree_samples = 1
     # * mean_hazard_curves = false
     # * quantile_hazard_curves =
     # * poes = at least one PoE
     cfg = helpers.get_data_path(
         'calculators/hazard/classical/haz_map_1rlz_no_stats.ini')
     job = helpers.run_job(cfg).job
     self.assertEqual(job.status, 'complete')
コード例 #14
0
    def run_hazard(self, cfg, exports=''):
        """
        Given the path to job config file, run the job and assert that it was
        successful. If this assertion passes, return the completed job.

        :param str cfg:
            Path to a job config file.
        :returns:
            The completed :class:`~openquake.engine.db.models.OqJob`.
        :raises:
            :exc:`AssertionError` if the job was not successfully run.
        """
        completed_job = helpers.run_job(cfg, exports=exports).job
        self.assertEqual('complete', completed_job.status)

        return completed_job
コード例 #15
0
ファイル: _utils.py プロジェクト: acerisara/oq-engine
    def run_hazard(self, cfg, exports=''):
        """
        Given the path to job config file, run the job and assert that it was
        successful. If this assertion passes, return the completed job.

        :param str cfg:
            Path to a job config file.
        :returns:
            The completed :class:`~openquake.engine.db.models.OqJob`.
        :raises:
            :exc:`AssertionError` if the job was not successfully run.
        """
        completed_job = helpers.run_job(cfg, exports=exports).job
        self.assertEqual('complete', completed_job.status)

        return completed_job
コード例 #16
0
ファイル: _utils.py プロジェクト: vup1120/oq-engine
    def run_hazard(self, cfg, exports=None):
        """
        Given the path to job config file, run the job and assert that it was
        successful. If this assertion passes, return the completed job.

        :param str cfg:
            Path to a job config file.
        :param list exports:
            A list of export format types. Currently only 'xml' is supported.
        :returns:
            The completed :class:`~openquake.engine.db.models.OqJob`.
        :raises:
            :exc:`AssertionError` if the job was not successfully run.
        """
        completed_job = helpers.run_job(cfg, exports=exports)
        self.assertEqual("complete", completed_job.status)

        return completed_job
コード例 #17
0
ファイル: __init__.py プロジェクト: MohsenKohrangi/oq-engine
    def run_risk(self, cfg, hazard_id):
        """
        Given the path to job config file, run the job and assert that it was
        successful. If this assertion passes, return the completed job.

        :param str cfg:
            Path to a job config file.
        :param int hazard_id:
            ID of the hazard output used by the risk calculation
        :returns:
            The completed :class:`~openquake.engine.db.models.OqJob`.
        :raises:
            :exc:`AssertionError` if the job was not successfully run.
        """
        completed_job = helpers.run_job(cfg, hazard_output_id=hazard_id)
        self.assertEqual('complete', completed_job.status)

        return completed_job
コード例 #18
0
ファイル: __init__.py プロジェクト: ChristieHale/oq-engine
    def run_risk(self, cfg, hazard_id):
        """
        Given the path to job config file, run the job and assert that it was
        successful. If this assertion passes, return the completed job.

        :param str cfg:
            Path to a job config file.
        :param int hazard_id:
            ID of the hazard output used by the risk calculation
        :returns:
            The completed :class:`~openquake.engine.db.models.OqJob`.
        :raises:
            :exc:`AssertionError` if the job was not successfully run.
        """
        completed_job = helpers.run_job(cfg, hazard_output_id=hazard_id).job
        self.assertEqual('complete', completed_job.status)

        return completed_job
コード例 #19
0
ファイル: core_test.py プロジェクト: monellid/oq-engine
    def test_complete_event_based_calculation_cycle(self):
        # run the calculation in process (to easy debugging)
        # and check the outputs
        with mock.patch.dict(os.environ, {'OQ_NO_DISTRIBUTE': '1'}):
            job = helpers.run_job(self.cfg)
        hc = job.hazard_calculation
        [rlz1, rlz2] = models.LtRealization.objects.filter(
            lt_model__hazard_calculation=hc.id)

        # check that the parameters are read correctly from the files
        self.assertEqual(hc.ses_per_logic_tree_path, 5)

        # check that we generated the right number of ruptures
        # (this is fixed if the seeds are fixed correctly)
        num_ruptures = models.SESRupture.objects.filter(
            rupture__ses_collection__output__oq_job=job.id).count()
        self.assertEqual(num_ruptures, 94)

        num_gmf1 = models.GmfData.objects.filter(
            gmf__lt_realization=rlz1).count()

        num_gmf2 = models.GmfData.objects.filter(
            gmf__lt_realization=rlz2).count()

        # check that we generated the same number of rows in GmfData
        # for both realizations
        self.assertEqual(num_gmf1, num_gmf2)
        # check that the number of tasks is a multiple of
        # 242 = 121 sites * 2 IMTs
        self.assertEqual(num_gmf1 % 242, 0)

        # Now check for the correct number of hazard curves:
        curves = models.HazardCurve.objects.filter(output__oq_job=job)
        # ((2 IMTs * 2 rlz) + (2 IMTs * (1 mean + 2 quantiles))) = 10
        # + 6 multi-imt curves (3 quantiles + 1 mean + 2 rlz)
        self.assertEqual(15, curves.count())

        # Finally, check for the correct number of hazard maps:
        maps = models.HazardMap.objects.filter(output__oq_job=job)
        # ((2 poes * 2 realizations * 2 IMTs)
        # + (2 poes * 2 IMTs * (1 mean + 2 quantiles))) = 20
        self.assertEqual(20, maps.count())
コード例 #20
0
ファイル: core_test.py プロジェクト: MohsenKohrangi/oq-engine
    def test_complete_event_based_calculation_cycle(self):
        # run the calculation in process (to easy debugging)
        # and check the outputs
        with mock.patch.dict(os.environ, {'OQ_NO_DISTRIBUTE': '1'}):
            job = helpers.run_job(self.cfg)
        hc = job.hazard_calculation
        [(rlz1, rlz2)] = models.LtSourceModel.objects.filter(
            hazard_calculation=hc.id)

        # check that the parameters are read correctly from the files
        self.assertEqual(hc.ses_per_logic_tree_path, 5)

        # check that we generated the right number of ruptures
        # (this is fixed if the seeds are fixed correctly)
        num_ruptures = models.SESRupture.objects.filter(
            rupture__ses_collection__output__oq_job=job.id).count()
        self.assertEqual(num_ruptures, 96)

        # check that we generated the right number of rows in GmfData
        # 242 = 121 sites * 2 IMTs
        num_gmf1 = models.GmfData.objects.filter(
            gmf__lt_realization=rlz1).count()
        num_gmf2 = models.GmfData.objects.filter(
            gmf__lt_realization=rlz2).count()

        # with concurrent_tasks=64, this test generates 17 tasks, but
        # only 15 gives nonzero contribution
        self.assertEqual(num_gmf1, 242 * 15)
        self.assertEqual(num_gmf2, 242 * 15)

        # Now check for the correct number of hazard curves:
        curves = models.HazardCurve.objects.filter(output__oq_job=job)
        # ((2 IMTs * 2 real) + (2 IMTs * (1 mean + 2 quantiles))) = 10
        # + 3 mean and quantiles multi-imt curves
        self.assertEqual(13, curves.count())

        # Finally, check for the correct number of hazard maps:
        maps = models.HazardMap.objects.filter(output__oq_job=job)
        # ((2 poes * 2 realizations * 2 IMTs)
        # + (2 poes * 2 IMTs * (1 mean + 2 quantiles))) = 20
        self.assertEqual(20, maps.count())
コード例 #21
0
ファイル: hazard_test.py プロジェクト: acerisara/oq-engine
    def test_export_for_event_based(self):
        # Run an event-based hazard calculation to compute SESs and GMFs
        # Call the exporters for both SES and GMF results  and verify that
        # files were created
        # Since the XML writers (in `openquake.commonlib`) are concerned
        # with correctly generating the XML, we don't test that here
        target_dir = tempfile.mkdtemp()

        try:
            cfg = helpers.get_data_path('event_based_hazard/job.ini')

            # run the calculation in process to create something to export
            with mock.patch.dict(os.environ, {'OQ_NO_DISTRIBUTE': '1'}):
                job = helpers.run_job(cfg, maximum_distance=1,
                                      ses_per_logic_tree_path=1,
                                      investigation_time=12,
                                      number_of_logic_tree_samples=1).job
            self.assertEqual(job.status, 'complete')

            dstore = datastore.DataStore(job.id)

            # 1 SES + 1 GMF + 1 hazard_curve_multi + 2 hazard_curve +
            # 4 hazard maps (with poes 0.1, 0.2 and IMT PGA, SA(0.1))
            outputs = core.get_outputs(job.id)

            # SESs
            ses_outputs = outputs.filter(ds_key='sescollection')
            self.assertEqual(1, len(ses_outputs))

            exported_files = []
            for ses_output in ses_outputs:
                out_file = check_export(ses_output.id, target_dir)
                exported_files.append(out_file)

            self.assertEqual(1, len(exported_files))

            for f in exported_files:
                self._test_exported_file(f)

            # GMFs
            gmf_outputs = outputs.filter(ds_key='gmfs')
            self.assertEqual(1, len(gmf_outputs))

            exported_files = []
            for gmf_output in gmf_outputs:
                out_file = check_export(gmf_output.id, target_dir)
                exported_files.append(out_file)

            self.assertEqual(1, len(exported_files))
            # Check the file paths exist, are absolute, and the files aren't
            # empty.
            for f in exported_files:
                self._test_exported_file(f)

            # check the exact values of the GMFs
            gmfs = writers.write_csv(
                io.StringIO(), dstore['gmfs']['col00'].value).encode('utf8')
            self.check_file_content('expected_gmfset_1.txt', gmfs)

            # Hazard curves
            haz_curves = outputs.filter(ds_key='hcurves')
            self.assertEqual(1, haz_curves.count())
            for curve in haz_curves:
                exported_file = check_export(curve.id, target_dir)
                self._test_exported_file(exported_file)

            # Hazard maps
            haz_maps = outputs.filter(ds_key='hmaps')
            self.assertEqual(1, haz_maps.count())
            for hmap in haz_maps:
                exported_file = check_export(hmap.id, target_dir)
                self._test_exported_file(exported_file)
        finally:
            shutil.rmtree(target_dir)
コード例 #22
0
ファイル: risk_test.py プロジェクト: ChristieHale/oq-engine
    def test_event_based_risk_export(self):
        target_dir = tempfile.mkdtemp()
        try:
            haz_cfg = helpers.get_data_path(
                'end-to-end-hazard-risk/job_haz_event_based.ini'
            )
            risk_cfg = helpers.get_data_path(
                'end-to-end-hazard-risk/job_risk_event_based.ini'
            )

            haz_job = helpers.run_job(haz_cfg).job
            # Run the risk on all outputs produced by the haz calc:
            risk_job = helpers.run_job(
                risk_cfg, hazard_calculation_id=haz_job.id).job

            risk_outputs = models.Output.objects.filter(oq_job=risk_job)

            agg_loss_curve_outputs = risk_outputs.filter(
                output_type='agg_loss_curve')
            loss_curve_outputs = risk_outputs.filter(output_type='loss_curve')
            loss_map_outputs = risk_outputs.filter(output_type='loss_map')

            # (1 mean + 2 quantiles) * 2 (as there also insured curves)
            self.assertEqual(6, loss_curve_outputs.count())

            # 16 rlzs + 16 (due to insured curves)
            event_loss_curve_outputs = risk_outputs.filter(
                output_type='event_loss_curve')
            self.assertEqual(32, event_loss_curve_outputs.count())
            self.assertEqual(16, agg_loss_curve_outputs.count())

            # make sure the mean and quantile curve sets got created correctly
            loss_curves = models.LossCurve.objects.filter(
                output__oq_job=risk_job
            )
            # sanity check (16 aggregate loss curve + 38 loss curves)
            self.assertEqual(54, loss_curves.count())
            # mean
            self.assertEqual(2, loss_curves.filter(statistics='mean').count())
            # quantiles
            self.assertEqual(
                4, loss_curves.filter(statistics='quantile').count()
            )

            # 16 logic tree realizations = 16 loss map + 1 mean loss
            # map + 2 quantile loss map
            self.assertEqual(19, loss_map_outputs.count())

            # 16 event loss table (1 per rlz)
            event_loss_tables = risk_outputs.filter(output_type="event_loss")
            self.assertEqual(16, event_loss_tables.count())

            # 32 loss fractions
            loss_fraction_outputs = risk_outputs.filter(
                output_type="loss_fraction")
            self.assertEqual(32, loss_fraction_outputs.count())

            # Now try to export everything, just to do a "smoketest" of the
            # exporter code:
            loss_curve_files = []
            for o in loss_curve_outputs:
                loss_curve_files.append(core.export(o.id, target_dir, 'xml'))
            for o in loss_fraction_outputs:
                loss_curve_files.append(core.export(o.id, target_dir, 'xml'))
            for o in event_loss_curve_outputs:
                loss_curve_files.append(core.export(o.id, target_dir, 'xml'))

            agg_loss_curve_files = []
            for o in agg_loss_curve_outputs:
                agg_loss_curve_files.append(
                    core.export(o.id, target_dir, 'xml')
                )

            event_loss_table_files = []
            for o in event_loss_tables:
                event_loss_table_files.append(
                    core.export(o.id, target_dir, 'csv')
                )

            loss_map_files = []
            for o in loss_map_outputs:
                loss_map_files.append(core.export(o.id, target_dir, 'xml'))

            self.assertEqual(70, len(loss_curve_files))
            self.assertEqual(16, len(agg_loss_curve_files))
            self.assertEqual(16, len(event_loss_table_files))
            self.assertEqual(19, len(loss_map_files))

            for f in loss_curve_files:
                self._test_exported_file(f)
            for f in loss_map_files:
                self._test_exported_file(f)
        finally:
            shutil.rmtree(target_dir)
コード例 #23
0
ファイル: risk_test.py プロジェクト: ChristieHale/oq-engine
    def test_classical_risk_export(self):
        target_dir = tempfile.mkdtemp()
        try:
            haz_cfg = helpers.get_data_path(
                'end-to-end-hazard-risk/job_haz_classical.ini'
            )
            risk_cfg = helpers.get_data_path(
                'end-to-end-hazard-risk/job_risk_classical.ini'
            )

            haz_job = helpers.run_job(haz_cfg).job
            # Run the risk on all outputs produced by the haz calc:
            risk_job = helpers.run_job(
                risk_cfg, hazard_calculation_id=haz_job.id).job

            risk_outputs = models.Output.objects.filter(oq_job=risk_job)

            loss_curve_outputs = risk_outputs.filter(output_type='loss_curve')
            loss_map_outputs = risk_outputs.filter(output_type='loss_map')

            # 16 logic tree realizations + 1 mean + 2 quantiles = 19
            # + 19 insured loss curves
            self.assertEqual(38, loss_curve_outputs.count())
            # make sure the mean and quantile curve sets got created correctly
            loss_curves = models.LossCurve.objects.filter(
                output__oq_job=risk_job,
                insured=False
            )
            # sanity check
            self.assertEqual(19, loss_curves.count())

            insured_curves = models.LossCurve.objects.filter(
                output__oq_job=risk_job,
                insured=True
            )
            # sanity check
            self.assertEqual(19, insured_curves.count())

            # mean
            self.assertEqual(1, loss_curves.filter(statistics='mean').count())
            # quantiles
            self.assertEqual(
                2, loss_curves.filter(statistics='quantile').count()
            )

            # mean
            self.assertEqual(
                1, insured_curves.filter(statistics='mean').count())
            # quantiles
            self.assertEqual(
                2, insured_curves.filter(statistics='quantile').count()
            )

            # 16 logic tree realizations = 16 loss map + 1 mean loss
            # map + 2 quantile loss map
            self.assertEqual(19, loss_map_outputs.count())

            # 19 loss fractions
            loss_fraction_outputs = risk_outputs.filter(
                output_type="loss_fraction")
            self.assertEqual(19, loss_fraction_outputs.count())

            # Now try to export everything, just to do a "smoketest" of the
            # exporter code:
            loss_curve_files = []
            for o in loss_curve_outputs:
                loss_curve_files.append(core.export(o.id, target_dir, 'xml'))

            loss_map_files = []
            for o in loss_map_outputs:
                loss_map_files.append(core.export(o.id, target_dir, 'xml'))

            self.assertEqual(38, len(loss_curve_files))
            self.assertEqual(19, len(loss_map_files))

            for f in loss_curve_files:
                self._test_exported_file(f)
            for f in loss_map_files:
                self._test_exported_file(f)
        finally:
            shutil.rmtree(target_dir)
コード例 #24
0
ファイル: core_test.py プロジェクト: ChristieHale/oq-engine
 def test(self):
     cfg = helpers.get_data_path('classical_job.ini')
     with mock.patch.dict(os.environ, {'OQ_NO_DISTRIBUTE': '1'}):
         with self.assertRaises(RuntimeError):
             helpers.run_job(cfg, maximum_distance=1)
コード例 #25
0
 def setUpClass(cls):
     cfg = helpers.get_data_path(
         'calculators/hazard/classical/haz_map_test_job2.ini')
     cls.job = helpers.run_job(cfg).job
     models.JobStats.objects.create(oq_job=cls.job)
     cls.monitor = EnginePerformanceMonitor('', cls.job.id)
コード例 #26
0
    def test_classical_risk_export(self):
        target_dir = tempfile.mkdtemp()
        try:
            haz_cfg = helpers.get_data_path(
                'end-to-end-hazard-risk/job_haz_classical.ini')
            risk_cfg = helpers.get_data_path(
                'end-to-end-hazard-risk/job_risk_classical.ini')

            haz_job = helpers.run_job(haz_cfg).job
            # Run the risk on all outputs produced by the haz calc:
            risk_job = helpers.run_job(risk_cfg,
                                       hazard_calculation_id=haz_job.id).job

            risk_outputs = models.Output.objects.filter(oq_job=risk_job)

            loss_curve_outputs = risk_outputs.filter(output_type='loss_curve')
            loss_map_outputs = risk_outputs.filter(output_type='loss_map')

            # 16 logic tree realizations + 1 mean + 2 quantiles = 19
            # + 19 insured loss curves
            self.assertEqual(38, loss_curve_outputs.count())
            # make sure the mean and quantile curve sets got created correctly
            loss_curves = models.LossCurve.objects.filter(
                output__oq_job=risk_job, insured=False)
            # sanity check
            self.assertEqual(19, loss_curves.count())

            insured_curves = models.LossCurve.objects.filter(
                output__oq_job=risk_job, insured=True)
            # sanity check
            self.assertEqual(19, insured_curves.count())

            # mean
            self.assertEqual(1, loss_curves.filter(statistics='mean').count())
            # quantiles
            self.assertEqual(2,
                             loss_curves.filter(statistics='quantile').count())

            # mean
            self.assertEqual(1,
                             insured_curves.filter(statistics='mean').count())
            # quantiles
            self.assertEqual(
                2,
                insured_curves.filter(statistics='quantile').count())

            # 16 logic tree realizations = 16 loss map + 1 mean loss
            # map + 2 quantile loss map
            self.assertEqual(19, loss_map_outputs.count())

            # 19 loss fractions
            loss_fraction_outputs = risk_outputs.filter(
                output_type="loss_fraction")
            self.assertEqual(19, loss_fraction_outputs.count())

            # Now try to export everything, just to do a "smoketest" of the
            # exporter code:
            loss_curve_files = []
            for o in loss_curve_outputs:
                loss_curve_files.append(core.export(o.id, target_dir, 'xml'))

            loss_map_files = []
            for o in loss_map_outputs:
                loss_map_files.append(core.export(o.id, target_dir, 'xml'))

            self.assertEqual(38, len(loss_curve_files))
            self.assertEqual(19, len(loss_map_files))

            for f in loss_curve_files:
                self._test_exported_file(f)
            for f in loss_map_files:
                self._test_exported_file(f)
        finally:
            shutil.rmtree(target_dir)
コード例 #27
0
 def setUpClass(cls):
     cfg = helpers.get_data_path(
         'calculators/hazard/classical/haz_map_test_job2.ini')
     cls.job = helpers.run_job(cfg).job
     models.JobStats.objects.create(oq_job=cls.job)
     cls.monitor = EnginePerformanceMonitor('', cls.job.id)
コード例 #28
0
ファイル: hazard_test.py プロジェクト: vup1120/oq-engine
    def test_export_for_event_based(self):
        # Run an event-based hazard calculation to compute SESs and GMFs
        # Call the exporters for both SES and GMF results  and verify that
        # files were created
        # Since the XML writers (in `openquake.nrmllib.writers`) are concerned
        # with correctly generating the XML, we don't test that here...
        # but we should still have an end-to-end QA test.
        target_dir = tempfile.mkdtemp()

        try:
            cfg = helpers.get_data_path("event_based_hazard/job.ini")

            # run the calculation in process to create something to export
            os.environ["OQ_NO_DISTRIBUTE"] = "1"
            try:
                job = helpers.run_job(cfg)
            finally:
                del os.environ["OQ_NO_DISTRIBUTE"]
            self.assertEqual(job.status, "complete")

            outputs = export_core.get_outputs(job.id)
            # 2 GMFs, 1 SES,
            # ((2 imts * 2 realizations)
            # + ((2 imts + 1 multi) * (1 mean + 3 quantiles))
            # hazard curves,
            # (2 poes * 2 imts * 2 realizations)
            # + (2 poes * 2 imts * (1 mean + 3 quantiles)) hazard maps
            # Total: 41
            self.assertEqual(43, len(outputs))

            #######
            # SESs:
            ses_outputs = outputs.filter(output_type="ses")
            self.assertEqual(1, len(ses_outputs))

            exported_files = []
            for ses_output in ses_outputs:
                out_file = check_export(ses_output.id, target_dir)
                exported_files.append(out_file)

            self.assertEqual(1, len(exported_files))

            for f in exported_files:
                self._test_exported_file(f)

            #######
            # GMFs:
            gmf_outputs = outputs.filter(output_type="gmf")
            self.assertEqual(2, len(gmf_outputs))

            exported_files = []
            for gmf_output in gmf_outputs:
                out_file = check_export(gmf_output.id, target_dir)
                exported_files.append(out_file)

            self.assertEqual(2, len(exported_files))
            # Check the file paths exist, are absolute, and the files aren't
            # empty.
            for f in exported_files:
                self._test_exported_file(f)

            ################
            # Hazard curves:
            haz_curves = outputs.filter(output_type="hazard_curve")
            self.assertEqual(12, haz_curves.count())
            for curve in haz_curves:
                exported_file = check_export(curve.id, target_dir)
                self._test_exported_file(exported_file)

            ##############
            # Hazard maps:
            haz_maps = outputs.filter(output_type="hazard_map")
            self.assertEqual(24, haz_maps.count())
            for hmap in haz_maps:
                exported_file = check_export(hmap.id, target_dir)
                self._test_exported_file(exported_file)
        finally:
            shutil.rmtree(target_dir)
コード例 #29
0
    def test_export_for_event_based(self):
        # Run an event-based hazard calculation to compute SESs and GMFs
        # Call the exporters for both SES and GMF results  and verify that
        # files were created
        # Since the XML writers (in `openquake.commonlib`) are concerned
        # with correctly generating the XML, we don't test that here
        target_dir = tempfile.mkdtemp()

        try:
            cfg = helpers.get_data_path('event_based_hazard/job.ini')

            # run the calculation in process to create something to export
            with mock.patch.dict(os.environ, {'OQ_NO_DISTRIBUTE': '1'}):
                job = helpers.run_job(cfg,
                                      maximum_distance=1,
                                      ses_per_logic_tree_path=1,
                                      number_of_logic_tree_samples=1).job
            self.assertEqual(job.status, 'complete')

            # 1 SES + 1 GMF + 1 hazard_curve_multi + 2 hazard_curve +
            # 4 hazard maps (with poes 0.1, 0.2 and IMT PGA, SA(0.1))
            outputs = core.get_outputs(job.id)
            self.assertEqual(9, len(outputs))

            # SESs
            ses_outputs = outputs.filter(output_type='ses')
            self.assertEqual(1, len(ses_outputs))

            exported_files = []
            for ses_output in ses_outputs:
                out_file = check_export(ses_output.id, target_dir)
                exported_files.append(out_file)

            self.assertEqual(1, len(exported_files))

            for f in exported_files:
                self._test_exported_file(f)

            # GMFs
            gmf_outputs = outputs.filter(output_type='gmf')
            self.assertEqual(1, len(gmf_outputs))

            exported_files = []
            for gmf_output in gmf_outputs:
                out_file = check_export(gmf_output.id, target_dir)
                exported_files.append(out_file)

            self.assertEqual(1, len(exported_files))
            # Check the file paths exist, are absolute, and the files aren't
            # empty.
            for f in exported_files:
                self._test_exported_file(f)

            # check the exact values of the GMFs
            [gmfset1] = gmf_outputs[0].gmf
            self.check_file_content('expected_gmfset_1.txt', str(gmfset1))

            # Hazard curves
            haz_curves = outputs.filter(output_type='hazard_curve')
            self.assertEqual(2, haz_curves.count())
            for curve in haz_curves:
                exported_file = check_export(curve.id, target_dir)
                self._test_exported_file(exported_file)

            # Hazard maps
            haz_maps = outputs.filter(output_type='hazard_map')
            self.assertEqual(4, haz_maps.count())
            for hmap in haz_maps:
                exported_file = check_export(hmap.id, target_dir)
                self._test_exported_file(exported_file)
        finally:
            shutil.rmtree(target_dir)
コード例 #30
0
    def test_classical_hazard_export(self):
        # Run a hazard calculation to compute some curves and maps
        # Call the exporter and verify that files were created
        # Since the hazard curve XML writer is concerned with correctly
        # generating XML, we won't test that here.
        target_dir = tempfile.mkdtemp()

        try:
            cfg = helpers.get_data_path('simple_fault_demo_hazard/job.ini')

            # run the calculation to create something to export
            helpers.run_job(cfg)

            job = models.OqJob.objects.latest('id')
            self.assertEqual(job.status, 'complete')

            outputs = core.get_outputs(job.id)

            # 10 hazard curves, 20 maps, 10 uhs, 5 multi curves
            expected_outputs = 45
            self.assertEqual(expected_outputs, outputs.count())

            # Number of curves:
            # (2 imts * 2 realizations)
            # + (2 imts * (1 mean + 2 quantiles)
            # = 10
            curves = outputs.filter(output_type='hazard_curve')
            self.assertEqual(10, curves.count())

            # Number of multi-curves
            # (2 realizations + 1 mean + 2 quantiles)
            multi_curves = outputs.filter(output_type="hazard_curve_multi")
            self.assertEqual(5, multi_curves.count())

            # Number of maps:
            # (2 poes * 2 imts * 2 realizations)
            # + (2 poes * 2 imts * (1 mean + 2 quantiles))
            # = 20
            # Number of UHS:
            maps = outputs.filter(output_type='hazard_map')
            self.assertEqual(20, maps.count())

            # Number of UHS:
            # (20 maps_PGA_SA / 2 poes)
            # = 10
            uhs = outputs.filter(output_type='uh_spectra')
            self.assertEqual(10, uhs.count())

            # Test hazard curve export:
            hc_files = []
            for curve in curves:
                hc_files.append(check_export(curve.id, target_dir))

            self.assertEqual(10, len(hc_files))

            # Test multi hazard curve export:
            hc_files = []
            for curve in multi_curves:
                hc_files.append(check_export(curve.id, target_dir))

            self.assertEqual(5, len(hc_files))

            for f in hc_files:
                self._test_exported_file(f)

            # Test hazard map export:
            hm_files = []
            for haz_map in maps:
                hm_files.append(check_export(haz_map.id, target_dir))

            self.assertEqual(20, len(hm_files))

            for f in hm_files:
                self._test_exported_file(f)

            # Test UHS export:
            uhs_files = []
            for u in uhs:
                uhs_files.append(check_export(u.id, target_dir))
            for f in uhs_files:
                self._test_exported_file(f)
        finally:
            shutil.rmtree(target_dir)
コード例 #31
0
ファイル: hazard_test.py プロジェクト: acerisara/oq-engine
    def test_classical_hazard_export(self):
        # Run a hazard calculation to compute some curves and maps
        # Call the exporter and verify that files were created
        # Since the hazard curve XML writer is concerned with correctly
        # generating XML, we won't test that here.
        target_dir = tempfile.mkdtemp()

        try:
            cfg = helpers.get_data_path('simple_fault_demo_hazard/job.ini')

            # run the calculation to create something to export
            helpers.run_job(cfg)

            job = models.OqJob.objects.latest('id')
            self.assertEqual(job.status, 'complete')

            outputs = core.get_outputs(job.id)

            # 10 hazard curves, 20 maps, 10 uhs, 5 multi curves
            expected_outputs = 45
            self.assertEqual(expected_outputs, outputs.count())

            # Number of curves:
            # (2 imts * 2 realizations)
            # + (2 imts * (1 mean + 2 quantiles)
            # = 10
            curves = outputs.filter(output_type='hazard_curve')
            self.assertEqual(10, curves.count())

            # Number of multi-curves
            # (2 realizations + 1 mean + 2 quantiles)
            multi_curves = outputs.filter(output_type="hazard_curve_multi")
            self.assertEqual(5, multi_curves.count())

            # Number of maps:
            # (2 poes * 2 imts * 2 realizations)
            # + (2 poes * 2 imts * (1 mean + 2 quantiles))
            # = 20
            # Number of UHS:
            maps = outputs.filter(output_type='hazard_map')
            self.assertEqual(20, maps.count())

            # Number of UHS:
            # (20 maps_PGA_SA / 2 poes)
            # = 10
            uhs = outputs.filter(output_type='uh_spectra')
            self.assertEqual(10, uhs.count())

            # Test hazard curve export:
            hc_files = []
            for curve in curves:
                hc_files.append(check_export(curve.id, target_dir))

            self.assertEqual(10, len(hc_files))

            # Test multi hazard curve export:
            hc_files = []
            for curve in multi_curves:
                hc_files.append(check_export(curve.id, target_dir))

            self.assertEqual(5, len(hc_files))

            for f in hc_files:
                self._test_exported_file(f)

            # Test hazard map export:
            hm_files = []
            for haz_map in maps:
                hm_files.append(check_export(haz_map.id, target_dir))

            self.assertEqual(20, len(hm_files))

            for f in hm_files:
                self._test_exported_file(f)

            # Test UHS export:
            uhs_files = []
            for u in uhs:
                uhs_files.append(check_export(u.id, target_dir))
            for f in uhs_files:
                self._test_exported_file(f)
        finally:
            shutil.rmtree(target_dir)
コード例 #32
0
ファイル: core_test.py プロジェクト: ChristieHale/oq-engine
 def test(self):
     cfg = helpers.get_data_path('classical_job.ini')
     with mock.patch.dict(os.environ, {'OQ_NO_DISTRIBUTE': '1'}):
         with self.assertRaises(RuntimeError):
             helpers.run_job(cfg, maximum_distance=1)
コード例 #33
0
ファイル: hazard_test.py プロジェクト: marmarques/oq-engine
    def test_export_for_event_based(self):
        # Run an event-based hazard calculation to compute SESs and GMFs
        # Call the exporters for both SES and GMF results  and verify that
        # files were created
        # Since the XML writers (in `openquake.nrmllib.writers`) are concerned
        # with correctly generating the XML, we don't test that here...
        # but we should still have an end-to-end QA test.
        target_dir = tempfile.mkdtemp()

        try:
            cfg = helpers.get_data_path('event_based_hazard/job.ini')

            # run the calculation in process to create something to export
            with mock.patch.dict(os.environ, {'OQ_NO_DISTRIBUTE': '1'}):
                job = helpers.run_job(cfg, maximum_distance=1,
                                      ses_per_logic_tree_path=1)
            self.assertEqual(job.status, 'complete')

            outputs = export_core.get_outputs(job.id)
            # 2 GMFs, 1 SES,
            # ((2 imts * 2 realizations)
            self.assertEqual(45, len(outputs))

            #######
            # SESs:
            ses_outputs = outputs.filter(output_type='ses')
            self.assertEqual(1, len(ses_outputs))

            exported_files = []
            for ses_output in ses_outputs:
                out_file = check_export(ses_output.id, target_dir)
                exported_files.append(out_file)

            self.assertEqual(1, len(exported_files))

            for f in exported_files:
                self._test_exported_file(f)

            #######
            # GMFs:
            gmf_outputs = outputs.filter(output_type='gmf')
            self.assertEqual(2, len(gmf_outputs))

            exported_files = []
            for gmf_output in gmf_outputs:
                out_file = check_export(gmf_output.id, target_dir)
                exported_files.append(out_file)

            self.assertEqual(2, len(exported_files))
            # Check the file paths exist, are absolute, and the files aren't
            # empty.
            for f in exported_files:
                self._test_exported_file(f)

            # check the exact values of the GMFs
            [gmfset1] = gmf_outputs[0].gmf
            [gmfset2] = gmf_outputs[1].gmf
            self.check_file_content('expected_gmfset_1.txt', str(gmfset1))
            self.check_file_content('expected_gmfset_2.txt', str(gmfset2))

            ################
            # Hazard curves:
            haz_curves = outputs.filter(output_type='hazard_curve')
            self.assertEqual(12, haz_curves.count())
            for curve in haz_curves:
                exported_file = check_export(curve.id, target_dir)
                self._test_exported_file(exported_file)

            ##############
            # Hazard maps:
            haz_maps = outputs.filter(output_type='hazard_map')
            self.assertEqual(24, haz_maps.count())
            for hmap in haz_maps:
                exported_file = check_export(hmap.id, target_dir)
                self._test_exported_file(exported_file)
        finally:
            shutil.rmtree(target_dir)
コード例 #34
0
    def test_event_based_risk_export(self):
        target_dir = tempfile.mkdtemp()
        try:
            haz_cfg = helpers.get_data_path(
                'end-to-end-hazard-risk/job_haz_event_based.ini')
            risk_cfg = helpers.get_data_path(
                'end-to-end-hazard-risk/job_risk_event_based.ini')

            haz_job = helpers.run_job(haz_cfg).job
            # Run the risk on all outputs produced by the haz calc:
            risk_job = helpers.run_job(risk_cfg,
                                       hazard_calculation_id=haz_job.id).job

            risk_outputs = models.Output.objects.filter(oq_job=risk_job)

            agg_loss_curve_outputs = risk_outputs.filter(
                output_type='agg_loss_curve')
            loss_curve_outputs = risk_outputs.filter(output_type='loss_curve')
            loss_map_outputs = risk_outputs.filter(output_type='loss_map')

            # (1 mean + 2 quantiles) * 2 (as there also insured curves)
            self.assertEqual(6, loss_curve_outputs.count())

            # 16 rlzs + 16 (due to insured curves)
            event_loss_curve_outputs = risk_outputs.filter(
                output_type='event_loss_curve')
            self.assertEqual(32, event_loss_curve_outputs.count())
            self.assertEqual(16, agg_loss_curve_outputs.count())

            # make sure the mean and quantile curve sets got created correctly
            loss_curves = models.LossCurve.objects.filter(
                output__oq_job=risk_job)
            # sanity check (16 aggregate loss curve + 38 loss curves)
            self.assertEqual(54, loss_curves.count())
            # mean
            self.assertEqual(2, loss_curves.filter(statistics='mean').count())
            # quantiles
            self.assertEqual(4,
                             loss_curves.filter(statistics='quantile').count())

            # 16 logic tree realizations = 16 loss map + 1 mean loss
            # map + 2 quantile loss map
            self.assertEqual(19, loss_map_outputs.count())

            # 16 event loss table (1 per rlz)
            event_loss_tables = risk_outputs.filter(output_type="event_loss")
            self.assertEqual(16, event_loss_tables.count())

            # 32 loss fractions
            loss_fraction_outputs = risk_outputs.filter(
                output_type="loss_fraction")
            self.assertEqual(32, loss_fraction_outputs.count())

            # Now try to export everything, just to do a "smoketest" of the
            # exporter code:
            loss_curve_files = []
            for o in loss_curve_outputs:
                loss_curve_files.append(core.export(o.id, target_dir, 'xml'))
            for o in loss_fraction_outputs:
                loss_curve_files.append(core.export(o.id, target_dir, 'xml'))
            for o in event_loss_curve_outputs:
                loss_curve_files.append(core.export(o.id, target_dir, 'xml'))

            agg_loss_curve_files = []
            for o in agg_loss_curve_outputs:
                agg_loss_curve_files.append(
                    core.export(o.id, target_dir, 'xml'))

            event_loss_table_files = []
            for o in event_loss_tables:
                event_loss_table_files.append(
                    core.export(o.id, target_dir, 'csv'))

            loss_map_files = []
            for o in loss_map_outputs:
                loss_map_files.append(core.export(o.id, target_dir, 'xml'))

            self.assertEqual(70, len(loss_curve_files))
            self.assertEqual(16, len(agg_loss_curve_files))
            self.assertEqual(16, len(event_loss_table_files))
            self.assertEqual(19, len(loss_map_files))

            for f in loss_curve_files:
                self._test_exported_file(f)
            for f in loss_map_files:
                self._test_exported_file(f)
        finally:
            shutil.rmtree(target_dir)
コード例 #35
0
 def setUpClass(cls):
     cfg = helpers.get_data_path(
         'calculators/hazard/classical/haz_map_test_job2.ini')
     cls.job = helpers.run_job(cfg).job
     models.JobStats.objects.create(oq_job=cls.job)