def test_case_master(self):
if sys.platform == 'darwin':
raise unittest.SkipTest('macOS')
self.run_calc(case_master.__file__, 'job.ini', insured_losses='false')
calc0 = self.calc.datastore # event_based_risk
self.run_calc(case_master.__file__, 'job.ini', insured_losses='false',
calculation_mode='event_based')
calc1 = self.calc.datastore # event_based
self.run_calc(case_master.__file__, 'job.ini', insured_losses='false',
calculation_mode='gmf_ebrisk',
hazard_calculation_id=str(calc1.calc_id),
concurrent_tasks='0') # to avoid numeric issues
calc2 = self.calc.datastore # gmf_ebrisk
check_csm_info(calc0, calc2) # the csm_info arrays must be equal
# compare the average losses for an event_based_risk
# case_master calculation with an equivalent gmf_ebrisk calculation
f0 = writetmp(view('mean_avg_losses', calc0))
self.assertEqualFiles('expected/avg_losses.txt', f0, delta=1E-5)
f2 = writetmp(view('mean_avg_losses', calc2))
self.assertEqualFiles('expected/avg_losses.txt', f2, delta=1E-4)
# compare the event loss table generated by an event_based_risk
# case_master calculation with an equivalent gmf_ebrisk calculation
f0 = writetmp(view('elt', calc0))
self.assertEqualFiles('expected/elt.txt', f0, delta=1E-5)
f2 = writetmp(view('elt', calc2))
self.assertEqualFiles('expected/elt.txt', f2, delta=1E-5)
def test_case_master(self):
self.assert_stats_ok(case_master, 'job.ini', individual_curves='false')
fnames = export(('loss_maps-rlzs', 'csv'), self.calc.datastore)
for fname in fnames:
self.assertEqualFiles('expected/' + strip_calc_id(fname), fname)
fname = writetmp(view('portfolio_loss', self.calc.datastore))
self.assertEqualFiles('expected/portfolio_loss.txt', fname, delta=1E-5)
# check rup_data is stored correctly
fname = writetmp(view('ruptures_events', self.calc.datastore))
self.assertEqualFiles('expected/ruptures_events.txt', fname)
# export a specific eid
fnames = export(('all_loss_ratios:0', 'csv'), self.calc.datastore)
for fname in fnames:
self.assertEqualFiles('expected/' + strip_calc_id(fname), fname)
self.assertEqualFiles('expected/losses-eid=0.csv', fname)
# export a specific pair (sm_id, eid)
fnames = export(('all_loss_ratios:1:0', 'csv'),
self.calc.datastore)
for fname in fnames:
self.assertEqualFiles('expected/%s' % strip_calc_id(fname), fname)
def test_wrong_sites_csv(self):
sites_csv = general.writetmp(
'site_id,lon,lat\n1,1.0,2.1\n2,3.0,4.1\n3,5.0,6.1')
source = general.writetmp("""
[general]
calculation_mode = scenario
[geometry]
sites_csv = %s
[misc]
maximum_distance=1
truncation_level=3
random_seed=5
[site_params]
reference_vs30_type = measured
reference_vs30_value = 600.0
reference_depth_to_2pt5km_per_sec = 5.0
reference_depth_to_1pt0km_per_sec = 100.0
intensity_measure_types_and_levels = {'PGA': [0.1, 0.2]}
investigation_time = 50.
export_dir = %s
""" % (os.path.basename(sites_csv), TMP))
oq = readinput.get_oqparam(source)
with self.assertRaises(InvalidFile) as ctx:
readinput.get_mesh(oq)
self.assertIn('expected site_id=0, got 1', str(ctx.exception))
os.unlink(sites_csv)
def test_get_oqparam_with_files(self):
temp_dir = tempfile.mkdtemp()
source_model_input = general.writetmp(dir=temp_dir)
site_model_input = general.writetmp(dir=temp_dir, content="foo")
job_config = general.writetmp(dir=temp_dir,
content="""
[general]
calculation_mode = event_based
[site]
sites = 0 0
source_model_file = %s
site_model_file = %s
maximum_distance=1
truncation_level=0
random_seed=0
intensity_measure_types = PGA
investigation_time = 50
export_dir = %s
""" % (os.path.basename(source_model_input),
os.path.basename(site_model_input), TMP))
try:
exp_base_path = os.path.dirname(job_config)
expected_params = {
'export_dir': TMP,
'base_path': exp_base_path,
'calculation_mode': 'event_based',
'truncation_level': 0.0,
'random_seed': 0,
'maximum_distance': {
'default': 1
},
'inputs': {
'job_ini': job_config,
'site_model': site_model_input,
'source': [source_model_input],
'source_model': source_model_input
},
'sites': [(0.0, 0.0, 0.0)],
'hazard_imtls': {
'PGA': None
},
'investigation_time': 50.0,
'risk_investigation_time': 50.0,
}
params = getparams(readinput.get_oqparam(job_config))
for key in expected_params:
self.assertEqual(expected_params[key], params[key])
items = sorted(params['inputs'].items())
keys, values = zip(*items)
self.assertEqual(
('job_ini', 'site_model', 'source', 'source_model'), keys)
self.assertEqual(
(job_config, site_model_input, [source_model_input
], source_model_input), values)
finally:
shutil.rmtree(temp_dir)
def test_get_oqparam_with_files(self):
temp_dir = tempfile.mkdtemp()
site_model_input = general.writetmp(dir=temp_dir, content="foo")
job_config = general.writetmp(dir=temp_dir,
content="""
[general]
calculation_mode = event_based
[foo]
bar = baz
[site]
sites = 0 0
site_model_file = %s
maximum_distance=1
truncation_level=0
random_seed=0
intensity_measure_types = PGA
investigation_time = 50
export_dir = %s
""" % (site_model_input, TMP))
try:
exp_base_path = os.path.dirname(job_config)
expected_params = {
'export_dir': TMP,
'base_path': exp_base_path,
'calculation_mode': 'event_based',
'truncation_level': 0.0,
'random_seed': 0,
'maximum_distance': {
'default': 1
},
'inputs': {
'job_ini': job_config,
'site_model': site_model_input
},
'sites': [(0.0, 0.0, 0.0)],
'hazard_imtls': {
'PGA': None
},
'investigation_time': 50.0,
'risk_investigation_time': 50.0,
}
with mock.patch('logging.warn') as warn:
params = getparams(readinput.get_oqparam(job_config))
for key in expected_params:
self.assertEqual(expected_params[key], params[key])
items = sorted(params['inputs'].items())
keys, values = zip(*items)
self.assertEqual(('job_ini', 'site_model'), keys)
self.assertEqual((job_config, site_model_input), values)
# checking that warnings work
self.assertEqual(warn.call_args[0][0],
"The parameter 'bar' is unknown, ignoring")
finally:
shutil.rmtree(temp_dir)
def test_event_based_sampling(self):
self.run_calc(ucerf.__file__, 'job_ebh.ini')
# check the GMFs
gmdata = self.calc.datastore['gmdata'].value
got = writetmp(rst_table(gmdata, fmt='%.6f'))
self.assertEqualFiles('expected/gmdata.csv', got)
# check the mean hazard map
got = writetmp(view('hmap', self.calc.datastore))
self.assertEqualFiles('expected/hmap.rst', got)
def test_get_oqparam_with_sites_csv(self):
sites_csv = general.writetmp('1.0,2.1\n3.0,4.1\n5.0,6.1')
try:
source = general.writetmp("""
[general]
calculation_mode = classical
[geometry]
sites_csv = %s
[misc]
maximum_distance=1
truncation_level=3
random_seed=5
[site_params]
reference_vs30_type = measured
reference_vs30_value = 600.0
reference_depth_to_2pt5km_per_sec = 5.0
reference_depth_to_1pt0km_per_sec = 100.0
intensity_measure_types_and_levels = {'PGA': [0.1, 0.2]}
investigation_time = 50.
export_dir = %s
""" % (os.path.basename(sites_csv), TMP))
exp_base_path = os.path.dirname(
os.path.join(os.path.abspath('.'), source))
expected_params = {
'export_dir': TMP,
'hazard_calculation_id': 1,
'base_path': exp_base_path,
'calculation_mode': 'classical',
'truncation_level': 3.0,
'random_seed': 5,
'maximum_distance': {
'default': 1.0
},
'inputs': {
'job_ini': source,
'sites': sites_csv
},
'reference_depth_to_1pt0km_per_sec': 100.0,
'reference_depth_to_2pt5km_per_sec': 5.0,
'reference_vs30_type': 'measured',
'reference_vs30_value': 600.0,
'hazard_imtls': {
'PGA': [0.1, 0.2]
},
'investigation_time': 50.0,
'risk_investigation_time': 50.0,
}
params = getparams(readinput.get_oqparam(source, hc_id=1))
self.assertEqual(expected_params, params)
finally:
os.unlink(sites_csv)
def test_get_oqparam_with_files(self):
temp_dir = tempfile.mkdtemp()
site_model_input = general.writetmp(dir=temp_dir, content="foo")
job_config = general.writetmp(dir=temp_dir, content="""
[general]
calculation_mode = event_based
[foo]
bar = baz
[site]
sites = 0 0
site_model_file = %s
maximum_distance=1
truncation_level=0
random_seed=0
intensity_measure_types = PGA
investigation_time = 50
export_dir = %s
""" % (site_model_input, TMP))
try:
exp_base_path = os.path.dirname(job_config)
expected_params = {
'export_dir': TMP,
'base_path': exp_base_path,
'calculation_mode': 'event_based',
'truncation_level': 0.0,
'random_seed': 0,
'maximum_distance': {'default': 1},
'inputs': {'job_ini': job_config,
'site_model': site_model_input},
'sites': [(0.0, 0.0)],
'hazard_imtls': {'PGA': None},
'investigation_time': 50.0,
'risk_investigation_time': 50.0,
}
with mock.patch('logging.warn') as warn:
params = getparams(readinput.get_oqparam(job_config))
for key in expected_params:
self.assertEqual(expected_params[key], params[key])
items = sorted(params['inputs'].items())
keys, values = zip(*items)
self.assertEqual(('job_ini', 'site_model'), keys)
self.assertEqual((job_config, site_model_input), values)
# checking that warnings work
self.assertEqual(warn.call_args[0][0],
"The parameter 'bar' is unknown, ignoring")
finally:
shutil.rmtree(temp_dir)
def possibly_gunzip(fname):
"""
A file can be .gzipped to save space (this happens
in the debian package); in that case, let's gunzip it.
:param fname: a file name (not zipped)
"""
is_gz = os.path.exists(fname) and fname.endswith('.gz')
there_is_gz = not os.path.exists(fname) and os.path.exists(fname + '.gz')
if is_gz:
return writetmp(gzip.open(fname).read())
elif there_is_gz:
return writetmp(gzip.open(fname + '.gz').read())
return fname
def possibly_gunzip(fname):
"""
A file can be .gzipped to save space (this happens
in the debian package); in that case, let's gunzip it.
:param fname: a file name (not zipped)
"""
is_gz = os.path.exists(fname) and fname.endswith('.gz')
there_is_gz = not os.path.exists(fname) and os.path.exists(fname + '.gz')
if is_gz:
return writetmp(gzip.open(fname).read())
elif there_is_gz:
return writetmp(gzip.open(fname + '.gz').read())
return fname
def _assert_equal_sources(nodes):
if hasattr(nodes[0], 'source_id'):
n0 = nodes[0]
for n in nodes[1:]:
n.assert_equal(n0, ignore=('id', 'src_group_id'))
else: # assume source nodes
n0 = nodes[0].to_str()
for n in nodes[1:]:
eq = n.to_str() == n0
if not eq:
f0 = writetmp(n0)
f1 = writetmp(n.to_str())
assert eq, 'different parameters for source %s, run meld %s %s' % (
n['id'], f0, f1)
return nodes
def test_lr_eq_0_cov_gt_0(self):
# If a vulnerability function loss ratio is 0 and its corresponding CoV
# is > 0, a ValueError should be raised
vuln_content = writetmp(u"""\
<?xml version='1.0' encoding='utf-8'?>
<nrml xmlns="http://openquake.org/xmlns/nrml/0.4"
xmlns:gml="http://www.opengis.net/gml">
<vulnerabilityModel>
<discreteVulnerabilitySet vulnerabilitySetID="PAGER"
assetCategory="population"
lossCategory="occupants">
<IML IMT="PGV">0.005 0.007 0.0098 0.0137</IML>
<discreteVulnerability vulnerabilityFunctionID="A"
probabilisticDistribution="LN">
<lossRatio>0.00 0.06 0.18 0.36</lossRatio>
<coefficientsVariation>0.30 0.30 0.30 0.30
</coefficientsVariation>
</discreteVulnerability>
</discreteVulnerabilitySet>
</vulnerabilityModel>
</nrml>
""")
with self.assertRaises(ValueError) as ar:
nrml.to_python(vuln_content)
self.assertIn(
'It is not valid to define a loss ratio = 0.0 with a '
'corresponding coeff. of variation > 0.0', str(ar.exception))
def test_case_master(self):
# this tests exercise the case of a complex logic tree; it also
# prints the warning on poe_agg very different from the expected poe
self.run_calc(case_master.__file__, 'job.ini')
fname = writetmp(view('mean_disagg', self.calc.datastore))
self.assertEqualFiles('expected/mean_disagg.rst', fname)
os.remove(fname)
def test_missing_line(self):
fname = general.writetmp('''\
col=00|ses=0001|src=test|rup=001-00,0 1,3.05128000E-01 6.04032000E-01
col=00|ses=0001|src=test|rup=001-01,0 1,2.67031000E-01 3.34878000E-01
''')
with self.assertRaises(readinput.InvalidFile):
readinput.get_gmfs_from_csv(self.oqparam, self.sitecol, fname)
def test(self):
fname = general.writetmp('''\
<?xml version="1.0" encoding="utf-8"?>
<nrml xmlns:gml="http://www.opengis.net/gml"
xmlns="http://openquake.org/xmlns/nrml/0.4">
<!-- Spectral Acceleration (SA) example -->
<hazardCurves sourceModelTreePath="b1_b2_b4" gsimTreePath="b1_b2" investigationTime="50.0" IMT="SA" saPeriod="0.025" saDamping="5.0">
<IMLs>5.0000e-03 7.0000e-03 1.3700e-02</IMLs>
<hazardCurve>
<gml:Point>
<gml:pos>-122.5000 37.5000</gml:pos>
</gml:Point>
<poEs>9.8728e-01 9.8266e-01 9.4957e-01</poEs>
</hazardCurve>
<hazardCurve>
<gml:Point>
<gml:pos>-123.5000 37.5000</gml:pos>
</gml:Point>
<poEs>9.8727e-02 9.8265e-02 9.4956e-02</poEs>
</hazardCurve>
</hazardCurves>
<!-- Basic example, using PGA as IMT -->
<hazardCurves sourceModelTreePath="b1_b2_b3" gsimTreePath="b1_b7" investigationTime="50.0" IMT="PGA">
<IMLs>5.0000e-03 7.0000e-03 1.3700e-02 3.3700e-02</IMLs>
<hazardCurve>
<gml:Point>
<gml:pos>-122.5000 37.5000</gml:pos>
</gml:Point>
<poEs>9.8728e-01 9.8226e-01 9.4947e-01 9.2947e-01</poEs>
</hazardCurve>
<hazardCurve>
<gml:Point>
<gml:pos>-123.5000 37.5000</gml:pos>
</gml:Point>
<poEs>9.8728e-02 9.8216e-02 9.4945e-02 9.2947e-02</poEs>
</hazardCurve>
</hazardCurves>
</nrml>
''',
suffix='.xml')
oqparam = object.__new__(oqvalidation.OqParam)
oqparam.inputs = dict(hazard_curves=fname)
sitecol, pmap = readinput.get_pmap(oqparam)
self.assertEqual(len(sitecol), 2)
self.assertEqual(sorted(oqparam.hazard_imtls.items()),
[('PGA', [0.005, 0.007, 0.0137, 0.0337]),
('SA(0.025)', [0.005, 0.007, 0.0137])])
hcurves = pmap.convert(oqparam.imtls, 2)
assert_allclose(
hcurves['PGA'],
numpy.array([[0.098728, 0.098216, 0.094945, 0.092947],
[0.98728, 0.98226, 0.94947, 0.92947]]))
assert_allclose(
hcurves['SA(0.025)'],
numpy.array([[0.098727, 0.098265, 0.094956],
[0.98728, 0.98266, 0.94957]]))
def test_lr_eq_0_cov_gt_0(self):
# If a vulnerability function loss ratio is 0 and its corresponding CoV
# is > 0, a ValueError should be raised
vuln_content = writetmp(u"""\
<?xml version='1.0' encoding='utf-8'?>
<nrml xmlns="http://openquake.org/xmlns/nrml/0.4"
xmlns:gml="http://www.opengis.net/gml">
<vulnerabilityModel>
<discreteVulnerabilitySet vulnerabilitySetID="PAGER"
assetCategory="population"
lossCategory="occupants">
<IML IMT="PGV">0.005 0.007 0.0098 0.0137</IML>
<discreteVulnerability vulnerabilityFunctionID="A"
probabilisticDistribution="LN">
<lossRatio>0.00 0.06 0.18 0.36</lossRatio>
<coefficientsVariation>0.30 0.30 0.30 0.30
</coefficientsVariation>
</discreteVulnerability>
</discreteVulnerabilitySet>
</vulnerabilityModel>
</nrml>
""")
with self.assertRaises(ValueError) as ar:
nrml.parse(vuln_content)
self.assertIn('It is not valid to define a loss ratio = 0.0 with a '
'corresponding coeff. of variation > 0.0',
str(ar.exception))
def test_get_oqparam_no_files(self):
# sections are there just for documentation
# when we parse the file, we ignore these
source = general.writetmp("""
[general]
calculation_mode = classical_risk
region = 1 1, 2 2, 3 3
[foo]
bar = baz
intensity_measure_types = PGA
export_dir = %s
""" % TMP)
exp_base_path = os.path.dirname(source)
expected_params = {
'export_dir': TMP,
'base_path': exp_base_path,
'calculation_mode': 'classical_risk',
'region': [(1.0, 1.0), (2.0, 2.0), (3.0, 3.0)],
'inputs': {},
'intensity_measure_types_and_levels': {'PGA': None},
}
# checking that warnings work
with mock.patch('logging.warn') as warn:
oqparam = readinput.get_oqparam(source)
self.assertEqual(warn.call_args[0][0],
"The parameter 'bar' is unknown, ignoring")
self.assertEqual(expected_params, vars(oqparam))
def test_missing_line(self):
self.oqparam.inputs['gmfs'] = general.writetmp('''\
col=00|ses=0001|src=test|rup=001-00,0 1,3.05128000E-01 6.04032000E-01
col=00|ses=0001|src=test|rup=001-01,0 1,2.67031000E-01 3.34878000E-01
''')
with self.assertRaises(readinput.InvalidFile):
readinput.get_gmfs(self.oqparam, self.sitecol)
def __fromh5__(self, dic, attrs):
# TODO: this is called more times than needed, maybe we should cache it
sg_data = group_array(dic['sg_data'], 'sm_id')
sm_data = dic['sm_data']
vars(self).update(attrs)
self.gsim_fname = decode(self.gsim_fname)
if self.gsim_fname.endswith('.xml'):
# otherwise it would look in the current directory
GMPETable.GMPE_DIR = os.path.dirname(self.gsim_fname)
trts = sorted(self.trts)
tmp = writetmp(self.gsim_lt_xml, suffix='.xml')
self.gsim_lt = logictree.GsimLogicTree(tmp, trts)
else: # fake file with the name of the GSIM
self.gsim_lt = logictree.GsimLogicTree.from_(self.gsim_fname)
self.source_models = []
for sm_id, rec in enumerate(sm_data):
tdata = sg_data[sm_id]
srcgroups = [
sourceconverter.SourceGroup(
self.trts[data['trti']], id=data['grp_id'],
eff_ruptures=data['effrup'], tot_ruptures=get_totrup(data))
for data in tdata if data['effrup']]
path = tuple(str(decode(rec['path'])).split('_'))
trts = set(sg.trt for sg in srcgroups)
sm = logictree.SourceModel(
rec['name'], rec['weight'], path, srcgroups,
rec['num_rlzs'], sm_id, rec['samples'])
self.source_models.append(sm)
self.init()
try:
os.remove(tmp) # gsim_lt file
except NameError: # tmp is defined only in the regular case, see above
pass
def test_event_based(self):
self.run_calc(ucerf.__file__, 'job.ini')
[fname] = export(('ruptures', 'csv'), self.calc.datastore)
# check that we get the expected number of events
with open(fname) as f:
self.assertEqual(len(f.readlines()), 37)
self.assertEqualFiles('expected/ruptures.csv', fname, lastline=20)
# run a regular event based on top of the UCERF ruptures and
# check the generated hazard maps
self.run_calc(ucerf.__file__,
'job.ini',
calculation_mode='event_based',
hazard_calculation_id=str(self.calc.datastore.calc_id))
# check the GMFs
gmdata = self.calc.datastore['gmdata'].value
got = writetmp(rst_table(gmdata, fmt='%.6f'))
self.assertEqualFiles('expected/gmdata_eb.csv', got)
# check the mean hazard map
[fname] = [
f for f in export(('hmaps', 'csv'), self.calc.datastore)
if 'mean' in f
]
self.assertEqualFiles('expected/hazard_map-mean.csv', fname)
def test_get_oqparam_with_sites_csv(self):
sites_csv = general.writetmp('1.0,2.1\n3.0,4.1\n5.0,6.1')
try:
source = general.writetmp("""
[general]
calculation_mode = classical
[geometry]
sites_csv = %s
[misc]
maximum_distance=1
truncation_level=3
random_seed=5
[site_params]
reference_vs30_type = measured
reference_vs30_value = 600.0
reference_depth_to_2pt5km_per_sec = 5.0
reference_depth_to_1pt0km_per_sec = 100.0
intensity_measure_types_and_levels = {'PGA': [0.1, 0.2]}
investigation_time = 50.
export_dir = %s
""" % (sites_csv, TMP))
exp_base_path = os.path.dirname(
os.path.join(os.path.abspath('.'), source))
expected_params = {
'export_dir': TMP,
'base_path': exp_base_path,
'calculation_mode': 'classical',
'truncation_level': 3.0,
'random_seed': 5,
'maximum_distance': 1.0,
'inputs': {'job_ini': source,
'sites': sites_csv},
'reference_depth_to_1pt0km_per_sec': 100.0,
'reference_depth_to_2pt5km_per_sec': 5.0,
'reference_vs30_type': 'measured',
'reference_vs30_value': 600.0,
'hazard_imtls': {'PGA': [0.1, 0.2]},
'risk_imtls': {},
'investigation_time': 50.0,
'risk_investigation_time': 50.0,
}
params = vars(readinput.get_oqparam(source))
self.assertEqual(expected_params, params)
finally:
os.unlink(sites_csv)
def test_negative_indices(self):
self.oqparam.inputs['gmfs'] = general.writetmp('''\
col=00|ses=0001|src=test|rup=001-00,0 -1,1.59434000E-01 3.92602000E-01
col=00|ses=0001|src=test|rup=001-01,0 1,3.05128000E-01 6.04032000E-01
col=00|ses=0001|src=test|rup=001-02,0 1,2.67031000E-01 3.34878000E-01
''')
with self.assertRaises(readinput.InvalidFile):
readinput.get_gmfs(self.oqparam, self.sitecol)
def test_negative_indices(self):
fname = general.writetmp('''\
col=00|ses=0001|src=test|rup=001-00,0 -1,1.59434000E-01 3.92602000E-01
col=00|ses=0001|src=test|rup=001-01,0 1,3.05128000E-01 6.04032000E-01
col=00|ses=0001|src=test|rup=001-02,0 1,2.67031000E-01 3.34878000E-01
''')
with self.assertRaises(readinput.InvalidFile):
readinput.get_gmfs_from_csv(self.oqparam, self.sitecol, fname)
def test_not_ordered_tags(self):
self.oqparam.inputs['gmfs'] = general.writetmp('''\
col=00|ses=0001|src=test|rup=001-02,0 1,1.59434000E-01 3.92602000E-01
col=00|ses=0001|src=test|rup=001-00,0 1,3.05128000E-01 6.04032000E-01
col=00|ses=0001|src=test|rup=001-01,0 1,2.67031000E-01 3.34878000E-01
''')
with self.assertRaises(readinput.InvalidFile):
readinput.get_gmfs(self.oqparam, self.sitecol)
def test_missing_bad_indices(self):
self.oqparam.inputs['gmfs'] = general.writetmp('''\
col=00|ses=0001|src=test|rup=001-00,,1.59434000E-01 3.92602000E-01
col=00|ses=0001|src=test|rup=001-01,0 1,3.05128000E-01 6.04032000E-01
col=00|ses=0001|src=test|rup=001-02,X,2.67031000E-01
''')
with self.assertRaises(readinput.InvalidFile):
readinput.get_gmfs(self.oqparam, self.sitecol)
def test_missing_line(self):
fname = general.writetmp('''\
0 0,0 1
col=00|ses=0001|src=test|rup=001-00,0 1,3.05128000E-01 6.04032000E-01
col=00|ses=0001|src=test|rup=001-01,0 1,2.67031000E-01 3.34878000E-01
''')
with self.assertRaises(readinput.InvalidFile):
readinput.get_gmfs_from_txt(self.oqparam, fname)
class FakeParams(hdf5.LiteralAttrs):
export_dir = '/tmp'
hazard_calculation_id = None
inputs = {'job_ini': writetmp('fake_job.ini')}
concurrent_tasks = 0
def to_params(self):
return {}
def test_not_ordered_etags(self):
fname = general.writetmp('''\
0 0,0 1
col=00|ses=0001|src=test|rup=001-02,0 1,1.59434000E-01 3.92602000E-01
col=00|ses=0001|src=test|rup=001-00,0 1,3.05128000E-01 6.04032000E-01
col=00|ses=0001|src=test|rup=001-01,0 1,2.67031000E-01 3.34878000E-01
''')
with self.assertRaises(readinput.InvalidFile):
readinput.get_gmfs_from_txt(self.oqparam, fname)
def get_path(fname_or_fileobject):
if isinstance(fname_or_fileobject, str):
return fname_or_fileobject
elif hasattr(fname_or_fileobject, 'getvalue'):
return writetmp(fname_or_fileobject.getvalue())
elif hasattr(fname_or_fileobject, 'name'):
return fname_or_fileobject.name
else:
return TypeError(fname_or_fileobject)
def test_ebr_2(self):
self.run_calc(ebr_2.__file__, 'job_ebrisk.ini', exports='csv')
fname = writetmp(view('mean_avg_losses', self.calc.datastore))
self.assertEqualFiles('expected/avg_losses.txt', fname)
alt = self.calc.datastore['agg_loss_table']
self.assertEqual(len(alt), 20)
self.assertEqual(set(alt['rlzi']), set([0])) # single rlzi
totloss = alt['loss'].sum()
aae(totloss, 20210.27, decimal=2)
def test_case_4(self):
# this test is sensitive to the ordering of the epsilons
# in openquake.riskinput.make_eps
out = self.run_calc(case_4.__file__, 'job.ini', exports='csv')
fname = writetmp(view('totlosses', self.calc.datastore))
self.assertEqualFiles('expected/totlosses.txt', fname)
[fname] = out['agglosses-rlzs', 'csv']
self.assertEqualFiles('expected/agglosses.csv', fname)
def test_event_based_risk(self):
self.run_calc(ucerf.__file__, 'job_ebr.ini',
number_of_logic_tree_samples='2')
fname = writetmp(view('portfolio_loss', self.calc.datastore))
self.assertEqualFiles('expected/portfolio_loss.txt', fname)
# make sure this runs
view('fullreport', self.calc.datastore)
def get(cls, path, **data):
resp = cls.c.get('/v1/calc/%s' % path, data, HTTP_HOST='127.0.0.1')
assert resp.content
try:
return json.loads(resp.content.decode('utf8'))
except:
print('Invalid JSON, see %s' % writetmp(resp.content),
file=sys.stderr)
return {}
def test_case_4(self):
# this test is sensitive to the ordering of the epsilons
# in openquake.riskinput.make_eps
out = self.run_calc(case_4.__file__, 'job.ini', exports='csv')
fname = writetmp(view('totlosses', self.calc.datastore))
self.assertEqualFiles('expected/totlosses.txt', fname)
[fname] = out['agglosses-rlzs', 'csv']
self.assertEqualFiles('expected/agglosses.csv', fname, delta=1E-6)
def test_case_miriam(self):
# this is a case with a grid and asset-hazard association
out = self.run_calc(case_miriam.__file__, 'job.ini', exports='csv')
[fname] = out['agg_loss_table', 'csv']
self.assertEqualFiles('expected/agg_losses-rlz000-structural.csv',
fname)
fname = writetmp(view('portfolio_loss', self.calc.datastore))
self.assertEqualFiles(
'expected/portfolio_loss.txt', fname, delta=1E-5)
def test_not_ordered_eids(self):
fname = general.writetmp('''\
0 0,0 1
col=00|ses=0001|src=test|rup=001-02,0 1,1.59434000E-01 3.92602000E-01
col=00|ses=0001|src=test|rup=001-00,0 1,3.05128000E-01 6.04032000E-01
col=00|ses=0001|src=test|rup=001-01,0 1,2.67031000E-01 3.34878000E-01
''')
with self.assertRaises(readinput.InvalidFile):
readinput.get_gmfs_from_txt(self.oqparam, fname)
def test_case_master(self):
if sys.platform == 'darwin':
raise unittest.SkipTest('MacOSX')
self.run_calc(case_master.__file__, 'job.ini', exports='csv')
fnames = export(('avg_losses-stats', 'csv'), self.calc.datastore)
assert fnames, 'avg_losses-stats not exported?'
for fname in fnames:
self.assertEqualFiles('expected/' + strip_calc_id(fname),
fname,
delta=1E-5)
# extract loss_curves/rlz-1 (with the first asset having zero losses)
[fname] = export(('loss_curves/rlz-1', 'csv'), self.calc.datastore)
self.assertEqualFiles('expected/' + strip_calc_id(fname),
fname,
delta=1E-5)
fnames = export(('loss_maps-rlzs', 'csv'), self.calc.datastore)
assert fnames, 'loss_maps-rlzs not exported?'
if REFERENCE_OS:
for fname in fnames:
self.assertEqualFiles('expected/' + strip_calc_id(fname),
fname,
delta=1E-5)
# extract curves by tag
tags = ['taxonomy=tax1', 'state=01', 'cresta=0.11']
a = extract(self.calc.datastore, 'aggcurves/structural', *tags)
self.assertEqual(a.array.shape, (4, 2)) # 4 stats, 2 return periods
fname = writetmp(view('portfolio_loss', self.calc.datastore))
self.assertEqualFiles('expected/portfolio_loss.txt', fname, delta=1E-5)
os.remove(fname)
# check ruptures are stored correctly
fname = writetmp(view('ruptures_events', self.calc.datastore))
self.assertEqualFiles('expected/ruptures_events.txt', fname)
os.remove(fname)
# check job_info is stored
job_info = {decode(k) for k in dict(self.calc.datastore['job_info'])}
self.assertIn('build_curves_maps.sent', job_info)
self.assertIn('build_curves_maps.received', job_info)
check_total_losses(self.calc)
def test_case_miriam(self):
# this is a case with a grid and asset-hazard association
out = self.run_calc(case_miriam.__file__, 'job.ini', exports='csv')
[fname] = out['agg_loss_table', 'csv']
self.assertEqualFiles('expected/agg_losses-rlz000-structural.csv',
fname, delta=1E-5)
fname = writetmp(view('portfolio_loss', self.calc.datastore))
self.assertEqualFiles(
'expected/portfolio_loss.txt', fname, delta=1E-5)
os.remove(fname)
def test_invalid_magnitude_distance_filter(self):
source = general.writetmp("""
[general]
maximum_distance=[(200, 8)]
""")
with self.assertRaises(ValueError) as ctx:
readinput.get_oqparam(source)
self.assertIn('magnitude 200.0 is bigger than the maximum (11): '
'could not convert to maximum_distance:',
str(ctx.exception))
def test_no_nrml(self):
fname = writetmp('''\
<?xml version="1.0" encoding="UTF-8"?>
<fragilityModel id="Ethiopia" assetCategory="buildings"
lossCategory="structural" />
''')
with self.assertRaises(ValueError) as ctx:
read(fname)
self.assertIn('expected a node of kind nrml, got fragilityModel',
str(ctx.exception))
def test_missing_indices_are_ok(self):
self.oqparam.inputs['gmfs'] = general.writetmp('''\
col=00|ses=0001|src=test|rup=001-00,,1.59434000E-01 3.92602000E-01
col=00|ses=0001|src=test|rup=001-01,0 1,3.05128000E-01 6.04032000E-01
col=00|ses=0001|src=test|rup=001-02,0,2.67031000E-01
''')
gmfs = readinput.get_gmfs(self.oqparam, self.sitecol)
gmvs1, gmvs2 = gmfs['PGA']
assert_allclose(gmvs1, [0.159434, 0.305128, 0.267031])
assert_allclose(gmvs2, [0.392602, 0.604032, 0.])
def get(cls, path, **params):
resp = requests.get("http://%s/v1/calc/%s" % (cls.hostport, path), params=params)
if not resp.text:
sys.stderr.write(open(cls.errfname).read())
return {}
try:
return json.loads(resp.text)
except:
print("Invalid JSON, see %s" % writetmp(resp.text), file=sys.stderr)
return {}
def test_gmf_ok(self):
fname = general.writetmp('''\
0 0,0 1
col=00|ses=0001|src=test|rup=001-00,0 1,3.05128000E-01 6.04032000E-01
col=00|ses=0001|src=test|rup=001-01,0 1,2.67031000E-01 3.34878000E-01
col=00|ses=0001|src=test|rup=001-02,0 1,1.59434000E-01 3.92602000E-01
''')
_, _, gmfs = readinput.get_gmfs_from_txt(self.oqparam, fname)
gmvs1, gmvs2 = gmfs['PGA']
assert_allclose(gmvs1, [0.305128, 0.267031, 0.159434])
assert_allclose(gmvs2, [0.604032, 0.334878, 0.392602])
def test_case_5(self):
# test with different curve resolution for different taxonomies
self.run_calc(case_5.__file__, 'job_h.ini,job_r.ini')
# check mean loss curves
[fname] = export(('loss_curves-stats', 'xml'), self.calc.datastore)
self.assertEqualFiles('expected/loss_curves-mean.xml', fname)
# check individual avg losses
fname = writetmp(view('loss_curves_avg', self.calc.datastore))
self.assertEqualFiles('expected/loss_curves_avg.txt', fname)
def test_gmf_ok(self):
fname = general.writetmp('''\
0 0,0 1
col=00|ses=0001|src=test|rup=001-00,0 1,3.05128000E-01 6.04032000E-01
col=00|ses=0001|src=test|rup=001-01,0 1,2.67031000E-01 3.34878000E-01
col=00|ses=0001|src=test|rup=001-02,0 1,1.59434000E-01 3.92602000E-01
''')
_, _, gmfs = readinput.get_gmfs_from_txt(self.oqparam, fname)
gmvs1, gmvs2 = gmfs['PGA']
assert_allclose(gmvs1, [0.305128, 0.267031, 0.159434])
assert_allclose(gmvs2, [0.604032, 0.334878, 0.392602])
def test_case_2(self):
self.assert_stats_ok(case_2, 'job.ini', individual_curves='true')
fname = writetmp(view('mean_avg_losses', self.calc.datastore))
self.assertEqualFiles('expected/mean_avg_losses.txt', fname)
# test the case when all GMFs are filtered out
with self.assertRaises(RuntimeError) as ctx:
self.run_calc(case_2.__file__, 'job.ini', minimum_intensity='10.0')
self.assertEqual(
str(ctx.exception),
'No GMFs were generated, perhaps they were all below the '
'minimum_intensity threshold')
def test_assetcol(self):
expected = writetmp('''\
asset_ref:|S20:,site_id:uint32:,structural:float64:,deductible~structural:float64:,insurance_limit~structural:float64:
a0,0,3000,25,100
a1,1,2000,0.1,0.2
a2,2,1000,0.02,0.08
a3,2,5000,1000,3000
a4,3,500000,1000,3000
''')
assetcol = riskinput.build_asset_collection(self.assets_by_site)
numpy.testing.assert_equal(
assetcol, readers.read_composite_array(expected))
def test_case_6a(self):
# case with two gsims
out = self.run_calc(case_6a.__file__, 'job_haz.ini,job_risk.ini',
exports='csv')
f1, f2 = out['agglosses-rlzs', 'csv']
self.assertEqualFiles('expected/agg-gsimltp_b1_structural.csv', f1)
self.assertEqualFiles('expected/agg-gsimltp_b2_structural.csv', f2)
# testing the totlosses view
dstore = self.calc.datastore
fname = writetmp(view('totlosses', dstore))
self.assertEqualFiles('expected/totlosses.txt', fname)
def test_assetcol(self):
expected = writetmp('''\
asset_ref:|S100:,site_id:uint32:,taxonomy:uint32:,fatalities:float64:,structural:float64:,deductible~structural:float64:,insurance_limit~structural:float64:
a0,0,1,10,3000,.25,1.0
a1,1,0,20,2000,0.25,0.5
a2,2,2,30,1000,0.2,0.8
a3,2,1,0,5000,2.0,6.0
a4,3,1,50,500000,2.0,6.0
''')
assetcol = riskinput.build_asset_collection(self.assets_by_site)
numpy.testing.assert_equal(
assetcol, readers.read_composite_array(expected))
def test_assetcol(self):
expected = writetmp('''\
idx:uint32,lon,lat,site_id:uint32,taxonomy:uint32:,number,area,occupants:float64:,structural:float64:,deductible-structural:float64:,insurance_limit-structural:float64:
0,8.12985001E+01,2.91098003E+01,0,1,3.00000000E+00,1.00000000E+01,1.00000000E+01,1.00000000E+02,2.50000000E+01,1.00000000E+02
1,8.30822983E+01,2.79006004E+01,1,0,5.00000000E+02,1.00000000E+01,2.00000000E+01,4.00000000E-01,1.00000000E-01,2.00000000E-01
''')
assetcol = riskinput.AssetCollection(self.assets_by_site, None, None)
numpy.testing.assert_equal(
assetcol.array, writers.read_composite_array(expected))
# pickleability
pickle.loads(pickle.dumps(assetcol))
def test_event_based_risk(self):
if h5py.__version__ < '2.6.0':
raise unittest.SkipTest # UCERF requires vlen arrays
self.run_calc(ucerf.__file__, 'job_ebr.ini',
number_of_logic_tree_samples='2')
fnames = export(('agg_loss_table', 'csv'), self.calc.datastore)
for fname in fnames:
self.assertEqualFiles('expected/%s' % strip_calc_id(fname), fname)
fname = writetmp(view('portfolio_loss', self.calc.datastore))
self.assertEqualFiles('expected/portfolio_loss.txt', fname)
def test(self):
fname = general.writetmp('''\
<?xml version="1.0" encoding="utf-8"?>
<nrml xmlns:gml="http://www.opengis.net/gml"
xmlns="http://openquake.org/xmlns/nrml/0.4">
<!-- Spectral Acceleration (SA) example -->
<hazardCurves sourceModelTreePath="b1_b2_b4" gsimTreePath="b1_b2" investigationTime="50.0" IMT="SA" saPeriod="0.025" saDamping="5.0">
<IMLs>5.0000e-03 7.0000e-03 1.3700e-02</IMLs>
<hazardCurve>
<gml:Point>
<gml:pos>-122.5000 37.5000</gml:pos>
</gml:Point>
<poEs>9.8728e-01 9.8266e-01 9.4957e-01</poEs>
</hazardCurve>
<hazardCurve>
<gml:Point>
<gml:pos>-123.5000 37.5000</gml:pos>
</gml:Point>
<poEs>9.8727e-02 9.8265e-02 9.4956e-02</poEs>
</hazardCurve>
</hazardCurves>
<!-- Basic example, using PGA as IMT -->
<hazardCurves sourceModelTreePath="b1_b2_b3" gsimTreePath="b1_b7" investigationTime="50.0" IMT="PGA">
<IMLs>5.0000e-03 7.0000e-03 1.3700e-02 3.3700e-02</IMLs>
<hazardCurve>
<gml:Point>
<gml:pos>-122.5000 37.5000</gml:pos>
</gml:Point>
<poEs>9.8728e-01 9.8226e-01 9.4947e-01 9.2947e-01</poEs>
</hazardCurve>
<hazardCurve>
<gml:Point>
<gml:pos>-123.5000 37.5000</gml:pos>
</gml:Point>
<poEs>9.8728e-02 9.8216e-02 9.4945e-02 9.2947e-02</poEs>
</hazardCurve>
</hazardCurves>
</nrml>
''', suffix='.xml')
oqparam = mock.Mock()
oqparam.inputs = dict(hazard_curves=fname)
sitecol, hcurves = readinput.get_hcurves(oqparam)
self.assertEqual(len(sitecol), 2)
self.assertEqual(sorted(oqparam.hazard_imtls.items()),
[('PGA', [0.005, 0.007, 0.0137, 0.0337]),
('SA(0.025)', [0.005, 0.007, 0.0137])])
self.assertEqual(str(hcurves), '''\
[([0.098727, 0.098265, 0.094956], [0.098728, 0.098216, 0.094945, 0.092947])
([0.98728, 0.98266, 0.94957], [0.98728, 0.98226, 0.94947, 0.92947])]''')
def to_nrml(self, key, data, fname=None, fmt=FIVEDIGITS):
"""
:param key:
`dmg_dist_per_asset|dmg_dist_per_taxonomy|dmg_dist_total|collapse_map`
:param data: sequence of rows to serialize
:fname: the path name of the output file; if None, build a name
:returns: path name of the saved file
"""
fname = fname or writetmp()
node = getattr(self, key + '_node')(data)
with open(fname, 'wb') as out:
nrml.write([node], out, fmt)
return fname
def validate_nrml(request):
"""
Leverage oq-risklib to check if a given XML text is a valid NRML
:param request:
a `django.http.HttpRequest` object containing the mandatory
parameter 'xml_text': the text of the XML to be validated as NRML
:returns: a JSON object, containing:
* 'valid': a boolean indicating if the provided text is a valid NRML
* 'error_msg': the error message, if any error was found
(None otherwise)
* 'error_line': line of the given XML where the error was found
(None if no error was found or if it was not a
validation error)
"""
xml_text = request.POST.get('xml_text')
if not xml_text:
return HttpResponseBadRequest(
'Please provide the "xml_text" parameter')
xml_file = writetmp(xml_text, suffix='.xml')
try:
nrml.read(xml_file)
except etree.ParseError as exc:
return _make_response(error_msg=exc.message.message,
error_line=exc.message.lineno,
valid=False)
except Exception as exc:
# get the exception message
exc_msg = exc.args[0]
if isinstance(exc_msg, bytes):
exc_msg = exc_msg.decode('utf-8') # make it a unicode object
elif isinstance(exc_msg, unicode):
pass
else:
# if it is another kind of object, it is not obvious a priori how
# to extract the error line from it
return _make_response(
error_msg=unicode(exc_msg), error_line=None, valid=False)
# if the line is not mentioned, the whole message is taken
error_msg = exc_msg.split(', line')[0]
# check if the exc_msg contains a line number indication
search_match = re.search(r'line \d+', exc_msg)
if search_match:
error_line = int(search_match.group(0).split()[1])
else:
error_line = None
return _make_response(
error_msg=error_msg, error_line=error_line, valid=False)
else:
return _make_response(error_msg=None, error_line=None, valid=True)
def test_get_oqparam_with_files(self):
temp_dir = tempfile.mkdtemp()
site_model_input = general.writetmp(dir=temp_dir, content="foo")
job_config = general.writetmp(dir=temp_dir, content="""
[general]
calculation_mode = classical
[site]
sites = 0 0
site_model_file = %s
maximum_distance=1
truncation_level=0
random_seed=0
intensity_measure_types = PGA
export_dir = %s
""" % (site_model_input, TMP))
try:
exp_base_path = os.path.dirname(job_config)
expected_params = {
'export_dir': TMP,
'base_path': exp_base_path,
'calculation_mode': 'classical',
'truncation_level': 0.0,
'random_seed': 0,
'maximum_distance': 1.0,
'inputs': {'site_model': site_model_input},
'sites': [(0.0, 0.0)],
'intensity_measure_types_and_levels': {'PGA': None},
}
params = vars(readinput.get_oqparam(job_config))
self.assertEqual(expected_params, params)
self.assertEqual(['site_model'], params['inputs'].keys())
self.assertEqual([site_model_input], params['inputs'].values())
finally:
shutil.rmtree(temp_dir)
def test_case_master_ebr(self):
out = self.run_calc(case_master.__file__, 'job.ini',
calculation_mode='ebrisk',
investigation_time='1',
insured_losses='false',
exports='csv')
for fname in out['losses_by_taxon', 'csv']:
self.assertEqualFiles('expected/%s' % strip_calc_id(fname), fname)
for fname in out['agg_loss_table', 'csv']:
self.assertEqualFiles('expected/%s' % strip_calc_id(fname), fname)
fname = writetmp(view('portfolio_loss', self.calc.datastore))
self.assertEqualFiles(
'expected/portfolio_loss_ebr.txt', fname, delta=1E-5)
def test_get_rlzs_assoc(self):
two_trts = general.writetmp("""\
<?xml version='1.0' encoding='utf-8'?>
<nrml xmlns:gml="http://www.opengis.net/gml"
xmlns="http://openquake.org/xmlns/nrml/0.4">
<logicTree logicTreeID='lt1'>
<logicTreeBranchingLevel branchingLevelID="bl1">
<logicTreeBranchSet uncertaintyType="gmpeModel" branchSetID="bs1"
applyToTectonicRegionType="Active Shallow Crust">
<logicTreeBranch branchID="b1">
<uncertaintyModel>BooreAtkinson2008</uncertaintyModel>
<uncertaintyWeight>0.75</uncertaintyWeight>
</logicTreeBranch>
<logicTreeBranch branchID="b2">
<uncertaintyModel>ChiouYoungs2008</uncertaintyModel>
<uncertaintyWeight>0.25</uncertaintyWeight>
</logicTreeBranch>
</logicTreeBranchSet>
</logicTreeBranchingLevel>
<logicTreeBranchingLevel branchingLevelID="bl2">
<logicTreeBranchSet uncertaintyType="gmpeModel" branchSetID="bs2"
applyToTectonicRegionType="Active Deep Crust">
<logicTreeBranch branchID="b1">
<uncertaintyModel>BooreAtkinson2008</uncertaintyModel>
<uncertaintyWeight>0.75</uncertaintyWeight>
</logicTreeBranch>
<logicTreeBranch branchID="b2">
<uncertaintyModel>ChiouYoungs2008</uncertaintyModel>
<uncertaintyWeight>0.25</uncertaintyWeight>
</logicTreeBranch>
</logicTreeBranchSet>
</logicTreeBranchingLevel>
</logicTree>
</nrml>""")
oqparam = mock.Mock()
oqparam.base_path = '/'
oqparam.inputs = dict(gsim_logic_tree=two_trts)
with self.assertRaises(readinput.InvalidFile) as ctx:
readinput.get_rlzs_assoc(oqparam)
self.assertIn('must contain a single tectonic region type',
str(ctx.exception))
def test_risk_default_export_type(self):
with mock.patch('openquake.engine.export.core.export') as export:
with mock.patch('openquake.engine.db.models'
'.Output.objects.get') as output_get:
output_get.return_value.output_type = 'test_output'
output_get.return_value.oq_job.status = 'complete'
export.return_value = writetmp('Fake result file content')
request = self.factory.get('/v1/calc/result/37')
response = views.get_result(request, 37)
self.assertEqual(200, response.status_code)
self.assertEqual('Fake result file content', response.content)
self.assertEqual(1, export.call_count)
self.assertEqual(37, export.call_args[0][0])
self.assertEqual('xml', export.call_args[1]['export_type'])
