def test_esm_flatfile(self):
input_file = os.path.join(os.path.dirname(self.input_file),
'esm_sa_flatfile_2018.csv')
log = EsmParser.parse(input_file,
output_path=self.output_file)
self.assertEqual(log['total'], 98)
self.assertEqual(log['written'], 98)
missingvals = log['missing_values']
self.assertTrue(missingvals['rjb'] == missingvals['rrup'] ==
missingvals['rupture_length'] ==
missingvals['ry0'] == missingvals['rx'] ==
missingvals['rupture_width'] == 97)
self.assertTrue(missingvals['strike_1'] == missingvals['dip_1'] ==
missingvals['rake_1'] == missingvals['duration_5_75']
== 98)
self.assertTrue(all(_ not in missingvals
for _ in ('pga', 'pgv', 'sa', 'duration_5_95')))
self.assertTrue(all(_ + '_components' not in missingvals
for _ in ('pga', 'pgv', 'sa', 'duration_5_95')))
self.assertEqual(missingvals['duration_5_75'], 98)
self.assertTrue(missingvals['magnitude'] ==
missingvals['magnitude_type'] == 13)
gmdb = GroundMotionTable(self.output_file, 'esm_sa_flatfile_2018')
with self.assertRaises(ValueError):
# trying to filter inside a with statement
with gmdb:
gmdb.filter('magnitude <= 4')
gmdb2 = gmdb.filter('magnitude <= 4')
# underlying HDF5 file not open (ValueError):
with self.assertRaises(ValueError):
for rec in gmdb2.records:
pass
# check that we correctly wrote default attrs:
with gmdb2:
tbl = gmdb2.table.attrs
self.assertTrue(isinstance(tbl.parser_stats, dict))
self.assertEqual(tbl.filename, 'template_basic_flatfile.hd5')
self.assertEqual(len(gmdb2.attrnames()), 6)
# now it works:
with gmdb2:
mag_le_4 = 0
for rec in gmdb2.records:
self.assertTrue(rec['magnitude'] <= 4)
mag_le_4 += 1
gmdb2 = gmdb.filter('magnitude > 4')
with gmdb2:
mag_gt_4 = 0
for rec in gmdb2.records:
self.assertTrue(rec['magnitude'] > 4)
mag_gt_4 += 1
self.assertTrue(mag_le_4 + mag_gt_4 == 98 - 13)
def testing(params):
'''Core method to compute testing data
:param params: dict with the request parameters
:return: json serializable dict to be passed into a Response object
'''
GMDB = 'gmdb' # pylint: disable=invalid-name
GSIM = 'gsim' # pylint: disable=invalid-name
IMT = 'imt' # pylint: disable=invalid-name
FIT_M = 'fit_measure' # pylint: disable=invalid-name
CONFIG = 'config' # pylint: disable=invalid-name
SEL = 'selexpr' # pylint: disable=invalid-name
# params[GMDB] is the tuple (hdf file name, table name):
gmdb_base = GroundMotionTable(*params[GMDB], mode='r')
ret = {}
obs_count = defaultdict(int)
gsim_skipped = {}
config = params.get(CONFIG, {})
# columns: "Measure of fit" "imt" "gsim" "value(s)"
for gsim in params[GSIM]:
try:
residuals = Residuals([gsim], params[IMT])
selexpr = _get_selexpr(gsim, params.get(SEL, ''))
# we have some record to be used, compute residuals:
gmdb = gmdb_base.filter(selexpr)
numrecords = _gmdb_records(residuals, gmdb)
obs_count[gsim] = numrecords
if not numrecords:
gsim_skipped[gsim] = 'No matching db record found'
continue
gsim_values = []
for key, name, func in params[FIT_M]:
result = func(residuals, config)
gsim_values.extend(_itervalues(gsim, key, name, result))
for moffit, imt, value in gsim_values:
# note: value isa Numpy scalar, but not ALL numpy scalar
# are json serializable: only those that are equal to Python's
ret.setdefault(moffit, {}).\
setdefault(imt, {})[gsim] = value.item()
except Exception as exc: # pylint: disable=broad-except
gsim_skipped[gsim] = str(exc)
return {
'Measure of fit': ret,
'Db records': obs_count,
'Gsim skipped': gsim_skipped
}
def check_gsim_defined_for_current_db(testdata):
'''no test function, it is used to inspect in debug mode in order to get
gsims with records in the current gmdb used for tests'''
for gsim in OQ.gsims():
try:
residuals = Residuals([gsim], ['PGA', 'PGV', 'SA(0.1)'])
gmdbpath = testdata.path('esm_sa_flatfile_2018.csv.hd5')
gm_table = GroundMotionTable(gmdbpath,
'esm_sa_flatfile_2018',
mode='r')
selexpr = get_selexpr(gsim)
num = gmdb_records(residuals, gm_table.filter(selexpr))
except:
pass
def get_residuals(params):
'''Core method to compute residuals plots data
:param params: dict with the request parameters
:return: json serializable dict to be passed into a Response object
'''
# params:
GMDB = 'gmdb' # pylint: disable=invalid-name
GSIM = 'gsim' # pylint: disable=invalid-name
IMT = 'imt' # pylint: disable=invalid-name
PLOTTYPE = 'plot_type' # pylint: disable=invalid-name
SEL = 'selexpr' # pylint: disable=invalid-name
func, kwargs = params[PLOTTYPE]
residuals = Residuals(params[GSIM], params[IMT])
# Compute residuals.
# params[GMDB] is the tuple (hdf file name, table name):
gmdb = GroundMotionTable(*params[GMDB], mode='r')
if params.get(SEL):
gmdb = gmdb.filter(params[SEL])
residuals.get_residuals(gmdb)
# statistics = residuals.get_residual_statistics()
ret = defaultdict(lambda: defaultdict(lambda: {}))
# extend keyword arguments:
kwargs = dict(kwargs, residuals=residuals, as_json=True)
# linestep = binwidth/10
for gsim in residuals.residuals:
for imt in residuals.residuals[gsim]:
kwargs['gmpe'] = gsim
kwargs['imt'] = imt
imt2 = _relabel_sa(imt)
res_plots = func(**kwargs)
for res_type, res_plot in res_plots.items():
for stat in RESIDUALS_STATS:
res_plot.setdefault(stat, None)
if imt2 != imt:
res_plot['xlabel'] = _relabel_sa(res_plot['xlabel'])
res_plot['ylabel'] = _relabel_sa(res_plot['ylabel'])
# make also x and y keys consistent with trellis response:
res_plot['xvalues'] = res_plot.pop('x')
res_plot['yvalues'] = res_plot.pop('y')
ret[imt2][res_type][gsim] = res_plot
return ret
