def test_esm_flatfile(self): input_file = os.path.join(os.path.dirname(self.input_file), 'esm_sa_flatfile_2018.csv') log = EsmParser.parse(input_file, output_path=self.output_file) self.assertEqual(log['total'], 98) self.assertEqual(log['written'], 98) missingvals = log['missing_values'] self.assertTrue(missingvals['rjb'] == missingvals['rrup'] == missingvals['rupture_length'] == missingvals['ry0'] == missingvals['rx'] == missingvals['rupture_width'] == 97) self.assertTrue(missingvals['strike_1'] == missingvals['dip_1'] == missingvals['rake_1'] == missingvals['duration_5_75'] == 98) self.assertTrue(all(_ not in missingvals for _ in ('pga', 'pgv', 'sa', 'duration_5_95'))) self.assertTrue(all(_ + '_components' not in missingvals for _ in ('pga', 'pgv', 'sa', 'duration_5_95'))) self.assertEqual(missingvals['duration_5_75'], 98) self.assertTrue(missingvals['magnitude'] == missingvals['magnitude_type'] == 13) gmdb = GroundMotionTable(self.output_file, 'esm_sa_flatfile_2018') with self.assertRaises(ValueError): # trying to filter inside a with statement with gmdb: gmdb.filter('magnitude <= 4') gmdb2 = gmdb.filter('magnitude <= 4') # underlying HDF5 file not open (ValueError): with self.assertRaises(ValueError): for rec in gmdb2.records: pass # check that we correctly wrote default attrs: with gmdb2: tbl = gmdb2.table.attrs self.assertTrue(isinstance(tbl.parser_stats, dict)) self.assertEqual(tbl.filename, 'template_basic_flatfile.hd5') self.assertEqual(len(gmdb2.attrnames()), 6) # now it works: with gmdb2: mag_le_4 = 0 for rec in gmdb2.records: self.assertTrue(rec['magnitude'] <= 4) mag_le_4 += 1 gmdb2 = gmdb.filter('magnitude > 4') with gmdb2: mag_gt_4 = 0 for rec in gmdb2.records: self.assertTrue(rec['magnitude'] > 4) mag_gt_4 += 1 self.assertTrue(mag_le_4 + mag_gt_4 == 98 - 13)
def testing(params): '''Core method to compute testing data :param params: dict with the request parameters :return: json serializable dict to be passed into a Response object ''' GMDB = 'gmdb' # pylint: disable=invalid-name GSIM = 'gsim' # pylint: disable=invalid-name IMT = 'imt' # pylint: disable=invalid-name FIT_M = 'fit_measure' # pylint: disable=invalid-name CONFIG = 'config' # pylint: disable=invalid-name SEL = 'selexpr' # pylint: disable=invalid-name # params[GMDB] is the tuple (hdf file name, table name): gmdb_base = GroundMotionTable(*params[GMDB], mode='r') ret = {} obs_count = defaultdict(int) gsim_skipped = {} config = params.get(CONFIG, {}) # columns: "Measure of fit" "imt" "gsim" "value(s)" for gsim in params[GSIM]: try: residuals = Residuals([gsim], params[IMT]) selexpr = _get_selexpr(gsim, params.get(SEL, '')) # we have some record to be used, compute residuals: gmdb = gmdb_base.filter(selexpr) numrecords = _gmdb_records(residuals, gmdb) obs_count[gsim] = numrecords if not numrecords: gsim_skipped[gsim] = 'No matching db record found' continue gsim_values = [] for key, name, func in params[FIT_M]: result = func(residuals, config) gsim_values.extend(_itervalues(gsim, key, name, result)) for moffit, imt, value in gsim_values: # note: value isa Numpy scalar, but not ALL numpy scalar # are json serializable: only those that are equal to Python's ret.setdefault(moffit, {}).\ setdefault(imt, {})[gsim] = value.item() except Exception as exc: # pylint: disable=broad-except gsim_skipped[gsim] = str(exc) return { 'Measure of fit': ret, 'Db records': obs_count, 'Gsim skipped': gsim_skipped }
def check_gsim_defined_for_current_db(testdata): '''no test function, it is used to inspect in debug mode in order to get gsims with records in the current gmdb used for tests''' for gsim in OQ.gsims(): try: residuals = Residuals([gsim], ['PGA', 'PGV', 'SA(0.1)']) gmdbpath = testdata.path('esm_sa_flatfile_2018.csv.hd5') gm_table = GroundMotionTable(gmdbpath, 'esm_sa_flatfile_2018', mode='r') selexpr = get_selexpr(gsim) num = gmdb_records(residuals, gm_table.filter(selexpr)) except: pass
def get_residuals(params): '''Core method to compute residuals plots data :param params: dict with the request parameters :return: json serializable dict to be passed into a Response object ''' # params: GMDB = 'gmdb' # pylint: disable=invalid-name GSIM = 'gsim' # pylint: disable=invalid-name IMT = 'imt' # pylint: disable=invalid-name PLOTTYPE = 'plot_type' # pylint: disable=invalid-name SEL = 'selexpr' # pylint: disable=invalid-name func, kwargs = params[PLOTTYPE] residuals = Residuals(params[GSIM], params[IMT]) # Compute residuals. # params[GMDB] is the tuple (hdf file name, table name): gmdb = GroundMotionTable(*params[GMDB], mode='r') if params.get(SEL): gmdb = gmdb.filter(params[SEL]) residuals.get_residuals(gmdb) # statistics = residuals.get_residual_statistics() ret = defaultdict(lambda: defaultdict(lambda: {})) # extend keyword arguments: kwargs = dict(kwargs, residuals=residuals, as_json=True) # linestep = binwidth/10 for gsim in residuals.residuals: for imt in residuals.residuals[gsim]: kwargs['gmpe'] = gsim kwargs['imt'] = imt imt2 = _relabel_sa(imt) res_plots = func(**kwargs) for res_type, res_plot in res_plots.items(): for stat in RESIDUALS_STATS: res_plot.setdefault(stat, None) if imt2 != imt: res_plot['xlabel'] = _relabel_sa(res_plot['xlabel']) res_plot['ylabel'] = _relabel_sa(res_plot['ylabel']) # make also x and y keys consistent with trellis response: res_plot['xvalues'] = res_plot.pop('x') res_plot['yvalues'] = res_plot.pop('y') ret[imt2][res_type][gsim] = res_plot return ret