def test_load_from_file_with_local_overriding_global(self): """ The config data in the local and global files is loaded correctly. The local data will override the global one. """ content = ''' [A] a=1 b=c [B] b=2''' site_path = touch(content=textwrap.dedent(content)) os.environ["OQ_SITE_CFG_PATH"] = site_path content = ''' [A] a=2 d=e [D] c=d-1 d=4''' local_path = touch(content=textwrap.dedent(content)) os.environ["OQ_LOCAL_CFG_PATH"] = local_path config.Config().cfg.clear() config.Config()._load_from_file() self.assertEqual(["A", "B", "D"], sorted(config.Config().cfg.keys())) self.assertEqual({ "a": "2", "b": "c", "d": "e" }, config.Config().cfg.get("A")) self.assertEqual({"b": "2"}, config.Config().cfg.get("B")) self.assertEqual({"c": "d-1", "d": "4"}, config.Config().cfg.get("D"))
def test_load_from_file_with_local_and_global(self): """ The config data in the local and global files is loaded correctly. """ content = """ [A] a=1 b=c [B] b=2""" site_path = touch(content=textwrap.dedent(content)) os.environ["OQ_SITE_CFG_PATH"] = site_path content = """ [C] c=3 d=e [D] d=4""" local_path = touch(content=textwrap.dedent(content)) os.environ["OQ_LOCAL_CFG_PATH"] = local_path config.Config().cfg.clear() config.Config()._load_from_file() self.assertEqual(["A", "B", "C", "D"], sorted(config.Config().cfg.keys())) self.assertEqual({"a": "1", "b": "c"}, config.Config().cfg.get("A")) self.assertEqual({"b": "2"}, config.Config().cfg.get("B")) self.assertEqual({"c": "3", "d": "e"}, config.Config().cfg.get("C")) self.assertEqual({"d": "4"}, config.Config().cfg.get("D"))
def test_load_from_file_with_local_overriding_global(self): """ The config data in the local and global files is loaded correctly. The local data will override the global one. """ content = ''' [A] a=1 b=c [B] b=2''' site_path = touch(content=textwrap.dedent(content)) os.environ["OQ_SITE_CFG_PATH"] = site_path content = ''' [A] a=2 d=e [D] c=d-1 d=4''' local_path = touch(content=textwrap.dedent(content)) os.environ["OQ_LOCAL_CFG_PATH"] = local_path config.Config().cfg.clear() config.Config()._load_from_file() self.assertEqual(["A", "B", "D"], sorted(config.Config().cfg.keys())) self.assertEqual({"a": "2", "b": "c", "d": "e"}, config.Config().cfg.get("A")) self.assertEqual({"b": "2"}, config.Config().cfg.get("B")) self.assertEqual({"c": "d-1", "d": "4"}, config.Config().cfg.get("D"))
def test_parse_config_with_files(self): temp_dir = tempfile.mkdtemp() site_model_input = helpers.touch(dir=temp_dir, content="foo") job_config = helpers.touch(dir=temp_dir, content=""" [general] calculation_mode = classical [site] site_model_file = %s maximum_distance=0 truncation_level=0 random_seed=0 """ % site_model_input) try: exp_base_path = os.path.dirname(job_config) expected_params = { 'base_path': exp_base_path, 'calculation_mode': 'classical', 'truncation_level': '0', 'random_seed': '0', 'maximum_distance': '0', 'inputs': {'site_model': site_model_input}, } params = engine.parse_config(open(job_config, 'r')) self.assertEqual(expected_params, params) self.assertEqual(['site_model'], params['inputs'].keys()) self.assertEqual([site_model_input], params['inputs'].values()) finally: shutil.rmtree(temp_dir)
def test_prepare_path_parameters(self): content = ''' [GENERAL] CALCULATION_MODE = Event Based OUTPUT_DIR = output [HAZARD] SOURCE_MODEL_LOGIC_TREE_FILE = source-model.xml GMPE_LOGIC_TREE_FILE = gmpe.xml [RISK] EXPOSURE = /absolute/exposure.xml VULNERABILITY = vulnerability.xml ''' config_path = helpers.touch(content=textwrap.dedent(content)) params, sections = _parse_config_file(config_path) params, sections = _prepare_config_parameters(params, sections) self.assertEqual( {'BASE_PATH': gettempdir(), 'OUTPUT_DIR': 'output', 'SOURCE_MODEL_LOGIC_TREE_FILE': os.path.join(gettempdir(), 'source-model.xml'), 'GMPE_LOGIC_TREE_FILE': os.path.join(gettempdir(), 'gmpe.xml'), 'EXPOSURE': '/absolute/exposure.xml', 'VULNERABILITY': os.path.join(gettempdir(), 'vulnerability.xml'), 'CALCULATION_MODE': 'Event Based'}, params) self.assertEqual(['GENERAL', 'HAZARD', 'RISK'], sorted(sections))
def test_optimize_source_model(self): in_file = helpers.touch(content=self.input_source_model) out_file = helpers.touch(content=self.output_source_model) area_src_disc = 50.0 try: source_input.optimize_source_model(in_file, area_src_disc, out_file) expected = ElementTree.tostring( ElementTree.XML(self.output_source_model)) actual = ElementTree.tostring( ElementTree.XML(open(out_file).read())) self.assertEqual(expected, actual) finally: os.unlink(in_file) os.unlink(out_file)
def test_risk_mandatory_parameters(self): sections = [ config.RISK_SECTION, config.HAZARD_SECTION, config.GENERAL_SECTION] dummy_exposure = helpers.touch() params = {} validator = config.default_validators(sections, params) self.assertFalse(validator.is_valid()[0]) params = {config.EXPOSURE: dummy_exposure, config.DEPTHTO1PT0KMPERSEC: "33.33", config.VS30_TYPE: "measured"} validator = config.default_validators(sections, params) self.assertFalse(validator.is_valid()[0]) params = {config.EXPOSURE: dummy_exposure, config.REGION_GRID_SPACING: '0.5', config.DEPTHTO1PT0KMPERSEC: "33.33", config.VS30_TYPE: "measured"} validator = config.default_validators(sections, params) self.assertFalse(validator.is_valid()[0]) params = {config.EXPOSURE: dummy_exposure, config.INPUT_REGION: "1.0, 2.0, 3.0, 4.0, 5.0, 6.0", config.REGION_GRID_SPACING: '0.5', config.DEPTHTO1PT0KMPERSEC: "33.33", config.VS30_TYPE: "measured"} validator = config.default_validators(sections, params) self.assertTrue(validator.is_valid()[0])
def test_prepare_path_parameters(self): content = """ [GENERAL] CALCULATION_MODE = Event Based OUTPUT_DIR = output [HAZARD] SOURCE_MODEL_LOGIC_TREE_FILE = source-model.xml GMPE_LOGIC_TREE_FILE = gmpe.xml [RISK] EXPOSURE = /absolute/exposure.xml VULNERABILITY = vulnerability.xml """ config_path = helpers.touch(content=textwrap.dedent(content)) params, sections = _parse_config_file(config_path) params, sections = _prepare_config_parameters(params, sections) self.assertEqual( { "BASE_PATH": gettempdir(), "OUTPUT_DIR": "output", "SOURCE_MODEL_LOGIC_TREE_FILE": "source-model.xml", "GMPE_LOGIC_TREE_FILE": "gmpe.xml", "EXPOSURE": "/absolute/exposure.xml", "VULNERABILITY": "vulnerability.xml", "CALCULATION_MODE": "Event Based", }, params, ) self.assertEqual(["GENERAL", "HAZARD", "RISK"], sorted(sections))
def test_parse_config_with_sites_csv(self): sites_csv = helpers.touch(content='1.0,2.1\n3.0,4.1\n5.0,6.1') try: source = StringIO.StringIO(""" [general] calculation_mode = classical [geometry] sites_csv = %s [misc] maximum_distance=0 truncation_level=3 random_seed=5 """ % sites_csv) source.name = 'path/to/some/job.ini' exp_base_path = os.path.dirname( os.path.join(os.path.abspath('.'), source.name)) expected_params = { 'base_path': exp_base_path, 'sites': 'MULTIPOINT(1.0 2.1, 3.0 4.1, 5.0 6.1)', 'calculation_mode': 'classical', 'truncation_level': '3', 'random_seed': '5', 'maximum_distance': '0', 'inputs': {}, } params = engine.parse_config(source) self.assertEqual(expected_params, params) finally: os.unlink(sites_csv)
def test_optimize_source_model(self): in_file = helpers.touch(content=self.input_source_model) out_file = helpers.touch(content=self.output_source_model) area_src_disc = 50.0 try: source_input.optimize_source_model(in_file, area_src_disc, out_file) expected = etree.tostring( etree.parse(StringIO.StringIO(self.output_source_model)), pretty_print=True ) actual = etree.tostring(etree.parse(out_file), pretty_print=True) self.assertEqual(expected, actual) finally: os.unlink(in_file) os.unlink(out_file)
def test_parse_config_with_files(self): site_model_input = helpers.touch(content="foo") try: source = StringIO.StringIO(""" [general] calculation_mode = classical [site] site_model_file = %s maximum_distance=0 truncation_level=0 random_seed=0 """ % site_model_input) # Add a 'name' to make this look like a real file: source.name = 'path/to/some/job.ini' exp_base_path = os.path.dirname( os.path.join(os.path.abspath('.'), source.name)) expected_params = { 'base_path': exp_base_path, 'calculation_mode': 'classical', 'truncation_level': '0', 'random_seed': '0', 'maximum_distance': '0' } params, files = engine.parse_config(source) self.assertEqual(expected_params, params) self.assertEqual(['site_model_file'], files.keys()) self.assertEqual('acbd18db4cc2f85cedef654fccc4a4d8', files['site_model_file'].digest) finally: os.unlink(site_model_input)
def test_prepare_path_parameters(self): content = ''' [GENERAL] CALCULATION_MODE = Event Based OUTPUT_DIR = output [HAZARD] SOURCE_MODEL_LOGIC_TREE_FILE = source-model.xml GMPE_LOGIC_TREE_FILE = gmpe.xml [RISK] EXPOSURE = /absolute/exposure.xml VULNERABILITY = vulnerability.xml ''' config_path = helpers.touch(content=textwrap.dedent(content)) params, sections = _parse_config_file(config_path) params, sections = _prepare_config_parameters(params, sections) self.assertEqual( {'BASE_PATH': gettempdir(), 'OUTPUT_DIR': 'output', 'SOURCE_MODEL_LOGIC_TREE_FILE': 'source-model.xml', 'GMPE_LOGIC_TREE_FILE': 'gmpe.xml', 'EXPOSURE': '/absolute/exposure.xml', 'VULNERABILITY': 'vulnerability.xml', 'CALCULATION_MODE': 'Event Based'}, params) self.assertEqual(['GENERAL', 'HAZARD', 'RISK'], sorted(sections))
def test_parse_config_with_files_no_force_inputs(self): site_model_input = helpers.touch(content="foo") source = StringIO.StringIO(""" [general] calculation_mode = classical [site] site_model_file = %s maximum_distance=0 truncation_level=0 random_seed=0 """ % site_model_input) # Add a 'name' to make this look like a real file: source.name = 'path/to/some/job.ini' exp_base_path = os.path.dirname( os.path.join(os.path.abspath('.'), source.name)) expected_params = { 'base_path': exp_base_path, 'force_inputs': False, 'calculation_mode': 'classical', 'truncation_level': '0', 'random_seed': '0', 'maximum_distance': '0' } # Run first with force_inputs=True to create the new Input. params, expected_files = engine2.parse_config( source, force_inputs=True) # In order for us to reuse the existing input, we need to associate # each input with a successful job. job = engine2.prepare_job(getpass.getuser()) job.hazard_calculation = engine2.create_hazard_calculation( job.owner, params, expected_files.values()) job.status = 'complete' job.save() # Now run twice with force_inputs=False (the default). source.seek(0) params1, files1 = engine2.parse_config(source) source.seek(0) params2, files2 = engine2.parse_config(source) # Check the params just for sanity. self.assertEqual(expected_params, params1) self.assertEqual(expected_params, params2) # Finally, check that the Input returned by the latest 2 calls matches # the input we created above. self.assertEqual(len(expected_files), len(files1)) self.assertEqual(len(expected_files), len(files2)) self.assertEqual( expected_files['site_model_file'].id, files1['site_model_file'].id) self.assertEqual( expected_files['site_model_file'].id, files2['site_model_file'].id)
def test_model_content_unknown_content_type(self): test_file = helpers.touch() params = dict(GMPE_LOGIC_TREE_FILE=test_file, BASE_PATH='/') engine._insert_input_files(params, self.job, True) [glt] = models.inputs4job(self.job.id, input_type="gsim_logic_tree") self.assertEqual('unknown', glt.model_content.content_type)
def test_model_content_unknown_content_type(self): test_file = helpers.touch() params = dict(GMPE_LOGIC_TREE_FILE=test_file) engine._insert_input_files(params, self.job, True) [glt] = models.inputs4job(self.job.id, input_type="lt_gmpe") self.assertEqual('unknown', glt.model_content.content_type)
def test_parse_config_with_files_force_inputs(self): site_model_input = helpers.touch(content="site model") sm_lt_input = helpers.touch(content="source model logic tree") gsim_lt_input = helpers.touch(content="gsim logic tree") source = StringIO.StringIO(""" [hazard_or_whatever] calculation_mode = classical gsim_logic_tree_file = %s source_model_logic_tree_file = %s site_model_file = %s not_a_valid_file = foo.xml """ % (gsim_lt_input, sm_lt_input, site_model_input)) # Add a 'name' to make this look like a real file: source.name = 'path/to/some/job.ini' exp_base_path = os.path.dirname( os.path.join(os.path.abspath('.'), source.name)) expected_params = { 'base_path': exp_base_path, 'force_inputs': True, 'calculation_mode': 'classical', 'not_a_valid_file': 'foo.xml', } params, files = engine2.parse_config(source, force_inputs=True) expected_files = { 'site_model_file': models.Input.objects.filter( input_type='site_model').latest('id'), 'source_model_logic_tree_file': models.Input.objects.filter( input_type='lt_source').latest('id'), 'gsim_logic_tree_file': models.Input.objects.filter( input_type='lt_gsim').latest('id'), } self.assertEqual(expected_params, params) self.assertEqual(len(expected_files), len(files)) for key in expected_files: self.assertEqual(expected_files[key].id, files[key].id)
def test_parse_config_with_files_no_force_inputs(self): site_model_input = helpers.touch(content="foo") source = StringIO.StringIO(""" [general] calculation_mode = classical [site] site_model_file = %s not_a_valid_file = foo.xml """ % site_model_input) # Add a 'name' to make this look like a real file: source.name = 'path/to/some/job.ini' exp_base_path = os.path.dirname( os.path.join(os.path.abspath('.'), source.name)) expected_params = { 'base_path': exp_base_path, 'force_inputs': False, 'calculation_mode': 'classical', 'not_a_valid_file': 'foo.xml', } # Run first with force_inputs=True to create the new Input. _, expected_files = engine2.parse_config(source, force_inputs=True) # In order for us to reuse the existing input, we need to associate # each input with a successful job. job = engine2.prepare_job(getpass.getuser()) job.status = 'complete' job.save() for inp in expected_files.values(): i2j = models.Input2job(input=inp, oq_job=job) i2j.save() # Now run twice with force_inputs=False (the default). source.seek(0) params1, files1 = engine2.parse_config(source) source.seek(0) params2, files2 = engine2.parse_config(source) # Check the params just for sanity. self.assertEqual(expected_params, params1) self.assertEqual(expected_params, params2) # Finally, check that the Input returned by the latest 2 calls matches # the input we created above. self.assertEqual(len(expected_files), len(files1)) self.assertEqual(len(expected_files), len(files2)) for key in expected_files: self.assertEqual(expected_files[key].id, files1[key].id) self.assertEqual(expected_files[key].id, files2[key].id)
def test_model_content_detect_content_type(self): # Test detection of the content type (using the file extension). test_file = helpers.touch(suffix=".html") # We use the gmpe logic tree as our test target because there is no # parsing required in the function under test. Thus, we can put # whatever test garbage we want in the file, or just use an empty file # (which is the case here). params = dict(GMPE_LOGIC_TREE_FILE=test_file) engine._insert_input_files(params, self.job, True) [glt] = models.inputs4job(self.job.id, input_type="lt_gmpe") self.assertEqual('html', glt.model_content.content_type)
def test_get_with_known_section(self): """ get() will correctly return configuration data for known sections. """ content = ''' [E] f=6 g=h''' site_path = touch(content=textwrap.dedent(content)) os.environ["OQ_SITE_CFG_PATH"] = site_path config.Config().cfg.clear() config.Config()._load_from_file() self.assertEqual({"f": "6", "g": "h"}, config.Config().get("E"))
def test_model_content_detect_content_type(self): # Test detection of the content type (using the file extension). test_file = helpers.touch(suffix=".html") # We use the gmpe logic tree as our test target because there is no # parsing required in the function under test. Thus, we can put # whatever test garbage we want in the file, or just use an empty file # (which is the case here). params = dict(GMPE_LOGIC_TREE_FILE=test_file, BASE_PATH='/') engine._insert_input_files(params, self.job, True) [glt] = models.inputs4job(self.job.id, input_type="gsim_logic_tree") self.assertEqual('html', glt.model_content.content_type)
def test_parse_file(self): content = """ [GENERAL] CALCULATION_MODE = Event Based [HAZARD] MINIMUM_MAGNITUDE = 5.0 """ config_path = helpers.touch(dir=gettempdir(), content=textwrap.dedent(content)) params, sections = _parse_config_file(config_path) self.assertEqual( {"BASE_PATH": gettempdir(), "CALCULATION_MODE": "Event Based", "MINIMUM_MAGNITUDE": "5.0"}, params ) self.assertEqual(["GENERAL", "HAZARD"], sorted(sections))
def test_load_from_file_with_local(self): """The config data in the local file is loaded correctly.""" content = """ [C] c=3 d=e [D] d=4""" local_path = touch(content=textwrap.dedent(content)) os.environ["OQ_LOCAL_CFG_PATH"] = local_path config.Config().cfg.clear() config.Config()._load_from_file() self.assertEqual(["C", "D"], sorted(config.Config().cfg.keys())) self.assertEqual({"c": "3", "d": "e"}, config.Config().cfg.get("C")) self.assertEqual({"d": "4"}, config.Config().cfg.get("D"))
def test_load_from_file_with_global(self): """The config data in the global file is loaded correctly.""" content = ''' [A] a=1 b=c [B] b=2''' site_path = touch(content=textwrap.dedent(content)) os.environ["OQ_SITE_CFG_PATH"] = site_path config.Config().cfg.clear() config.Config()._load_from_file() self.assertEqual(["A", "B"], sorted(config.Config().cfg.keys())) self.assertEqual({"a": "1", "b": "c"}, config.Config().cfg.get("A")) self.assertEqual({"b": "2"}, config.Config().cfg.get("B"))
def test_file_path_validation(self): # existing file params = {'EXPOSURE': helpers.touch(), 'BASE_PATH': '/'} validator = config.FilePathValidator(params) self.assertTrue(validator.is_valid()[0]) # non-existing file params = {'VULNERABILITY': '/a/b/c', 'SOURCE_MODEL_LOGIC_TREE_FILE': '/a/b/c', 'BASE_PATH': '/'} validator = config.FilePathValidator(params) valid, messages = validator.is_valid() self.assertFalse(valid) self.assertEquals(2, len(messages))
def test_file_path_validation(self): # existing file params = dict() params['EXPOSURE'] = helpers.touch() validator = config.FilePathValidator(params) self.assertTrue(validator.is_valid()[0]) # non-existing file params = dict() params['VULNERABILITY'] = '/a/b/c' params['SOURCE_MODEL_LOGIC_TREE_FILE'] = '/a/b/c' validator = config.FilePathValidator(params) valid, messages = validator.is_valid() self.assertFalse(valid) self.assertEquals(2, len(messages))
def test_parse_file(self): content = ''' [GENERAL] CALCULATION_MODE = Event Based [HAZARD] MINIMUM_MAGNITUDE = 5.0 ''' config_path = helpers.touch( dir=gettempdir(), content=textwrap.dedent(content)) params, sections = _parse_config_file(config_path) self.assertEqual( {'BASE_PATH': gettempdir(), 'CALCULATION_MODE': 'Event Based', 'MINIMUM_MAGNITUDE': '5.0'}, params) self.assertEqual(['GENERAL', 'HAZARD'], sorted(sections))
def test_risk_mandatory_parameters(self): sections = [ config.RISK_SECTION, config.HAZARD_SECTION, config.GENERAL_SECTION ] dummy_exposure = helpers.touch() params = {} validator = config.default_validators(sections, params) self.assertFalse(validator.is_valid()[0]) params = { config.EXPOSURE: dummy_exposure, config.DEPTHTO1PT0KMPERSEC: "33.33", config.VS30_TYPE: "measured" } validator = config.default_validators(sections, params) self.assertFalse(validator.is_valid()[0]) params = { config.EXPOSURE: dummy_exposure, config.REGION_GRID_SPACING: '0.5', config.DEPTHTO1PT0KMPERSEC: "33.33", config.VS30_TYPE: "measured" } validator = config.default_validators(sections, params) self.assertFalse(validator.is_valid()[0]) params = { config.EXPOSURE: dummy_exposure, config.INPUT_REGION: "1.0, 2.0, 3.0, 4.0, 5.0, 6.0", config.REGION_GRID_SPACING: '0.5', config.DEPTHTO1PT0KMPERSEC: "33.33", config.VS30_TYPE: "measured" } validator = config.default_validators(sections, params) self.assertTrue(validator.is_valid()[0])
def test_prepare_parameters(self): content = """ [GENERAL] CALCULATION_MODE = Event Based # unknown parameter FOO = 5 [HAZARD] MINIMUM_MAGNITUDE = 5.0 # not used for this calc mode COMPUTE_MEAN_HAZARD_CURVE = true """ config_path = helpers.touch(dir=gettempdir(), content=textwrap.dedent(content)) params, sections = _parse_config_file(config_path) params, sections = _prepare_config_parameters(params, sections) self.assertEqual( {"BASE_PATH": gettempdir(), "MINIMUM_MAGNITUDE": "5.0", "CALCULATION_MODE": "Event Based"}, params ) self.assertEqual(["GENERAL", "HAZARD"], sorted(sections))
def test_prepare_parameters_for_uhs_set_imt_to_sa(self): # The imt is always set to "sa" for uhs jobs. content = """ [general] CALCULATION_MODE = UHS SITES = 0.0, 0.0 DESCRIPTION = Uniform Hazard Spectra Demo [HAZARD] # parameters for UHS calculations UHS_PERIODS = 0.025, 0.45, 2.5 POES = 0.1, 0.02 INTENSITY_MEASURE_TYPE = PGA """ config_path = helpers.touch(dir=gettempdir(), content=textwrap.dedent(content)) params, sections = _parse_config_file(config_path) params, sections = _prepare_config_parameters(params, sections) self.assertEqual("SA", params["INTENSITY_MEASURE_TYPE"])
def test_prepare_parameters(self): content = ''' [GENERAL] CALCULATION_MODE = Event Based # unknown parameter FOO = 5 [HAZARD] MINIMUM_MAGNITUDE = 5.0 # not used for this calc mode COMPUTE_MEAN_HAZARD_CURVE = true ''' config_path = helpers.touch( dir=gettempdir(), content=textwrap.dedent(content)) params, sections = _parse_config_file(config_path) params, sections = _prepare_config_parameters(params, sections) self.assertEqual( {'BASE_PATH': gettempdir(), 'MINIMUM_MAGNITUDE': '5.0', 'CALCULATION_MODE': 'Event Based'}, params) self.assertEqual(['GENERAL', 'HAZARD'], sorted(sections))
def test_prepare_parameters_for_uhs_set_imt_to_sa(self): # The imt is always set to "sa" for uhs jobs. content = ''' [general] CALCULATION_MODE = UHS SITES = 0.0, 0.0 DESCRIPTION = Uniform Hazard Spectra Demo [HAZARD] # parameters for UHS calculations UHS_PERIODS = 0.025, 0.45, 2.5 POES = 0.1, 0.02 INTENSITY_MEASURE_TYPE = PGA ''' config_path = helpers.touch( dir=gettempdir(), content=textwrap.dedent(content)) params, sections = _parse_config_file(config_path) params, sections = _prepare_config_parameters(params, sections) self.assertEqual("SA", params["INTENSITY_MEASURE_TYPE"])
def setup_parser(content): path = helpers.touch(content=content.strip()) return fragility.FragilityModelParser(path)