def get_residuals(params):
'''Core method to compute residuals plots data
:param params: dict with the request parameters
:return: json serializable dict to be passed into a Response object
'''
# params:
GMDB = 'gmdb' # pylint: disable=invalid-name
GSIM = 'gsim' # pylint: disable=invalid-name
IMT = 'imt' # pylint: disable=invalid-name
PLOTTYPE = 'plot_type' # pylint: disable=invalid-name
SEL = 'selexpr' # pylint: disable=invalid-name
func, kwargs = params[PLOTTYPE]
residuals = Residuals(params[GSIM], params[IMT])
# Compute residuals.
# params[GMDB] is the tuple (hdf file name, table name):
gmdb = GroundMotionTable(*params[GMDB], mode='r')
if params.get(SEL):
gmdb = gmdb.filter(params[SEL])
residuals.get_residuals(gmdb)
# statistics = residuals.get_residual_statistics()
ret = defaultdict(lambda: defaultdict(lambda: {}))
# extend keyword arguments:
kwargs = dict(kwargs, residuals=residuals, as_json=True)
# linestep = binwidth/10
for gsim in residuals.residuals:
for imt in residuals.residuals[gsim]:
kwargs['gmpe'] = gsim
kwargs['imt'] = imt
imt2 = _relabel_sa(imt)
res_plots = func(**kwargs)
for res_type, res_plot in res_plots.items():
for stat in RESIDUALS_STATS:
res_plot.setdefault(stat, None)
if imt2 != imt:
res_plot['xlabel'] = _relabel_sa(res_plot['xlabel'])
res_plot['ylabel'] = _relabel_sa(res_plot['ylabel'])
# make also x and y keys consistent with trellis response:
res_plot['xvalues'] = res_plot.pop('x')
res_plot['yvalues'] = res_plot.pop('y')
ret[imt2][res_type][gsim] = res_plot
return ret
def setUpClass(cls):
"""
Import the database and the rupture before tests
"""
input_db = os.path.join(BASE_DATA_PATH, "LAquila_Database")
input_dir = os.path.join(BASE_DATA_PATH, "LAquila_Good_Records")
# Build the record database using the L'Aquila subset
builder = sdb.SMDatabaseBuilder(SigmaDatabaseMetadataReader,
input_db)
builder.build_database("001", "LAquila Mainshock", input_dir)
builder.parse_records(SigmaRecordParser, SigmaSpectraParser)
sdb.add_horizontal_im(builder.database, ["PGA", "PGV", "Geometric"])
# Load in the data from the database
cls.db = load_pickle(os.path.join(input_db, "metadatafile.pkl"))
input_rupture_file = os.path.join(BASE_DATA_PATH,
"laquila_rupture.xml")
cls.rupture = csim.build_rupture_from_file(input_rupture_file)
cls.gsims = ["AkkarEtAlRjb2014"]
cls.imts = ["PGA", "SA(1.0)"]
for rec in cls.db:
rec.datafile = os.path.join(os.path.dirname(__file__),
rec.datafile)
print(rec.datafile)
# Generate the residuals
cls.residuals = Residuals(cls.gsims, cls.imts)
cls.residuals.get_residuals(cls.db)
def testing(params):
'''Core method to compute testing data
:param params: dict with the request parameters
:return: json serializable dict to be passed into a Response object
'''
GMDB = 'gmdb' # pylint: disable=invalid-name
GSIM = 'gsim' # pylint: disable=invalid-name
IMT = 'imt' # pylint: disable=invalid-name
FIT_M = 'fit_measure' # pylint: disable=invalid-name
CONFIG = 'config' # pylint: disable=invalid-name
SEL = 'selexpr' # pylint: disable=invalid-name
# params[GMDB] is the tuple (hdf file name, table name):
gmdb_base = GroundMotionTable(*params[GMDB], mode='r')
ret = {}
obs_count = defaultdict(int)
gsim_skipped = {}
config = params.get(CONFIG, {})
# columns: "Measure of fit" "imt" "gsim" "value(s)"
for gsim in params[GSIM]:
try:
residuals = Residuals([gsim], params[IMT])
selexpr = _get_selexpr(gsim, params.get(SEL, ''))
# we have some record to be used, compute residuals:
gmdb = gmdb_base.filter(selexpr)
numrecords = _gmdb_records(residuals, gmdb)
obs_count[gsim] = numrecords
if not numrecords:
gsim_skipped[gsim] = 'No matching db record found'
continue
gsim_values = []
for key, name, func in params[FIT_M]:
result = func(residuals, config)
gsim_values.extend(_itervalues(gsim, key, name, result))
for moffit, imt, value in gsim_values:
# note: value isa Numpy scalar, but not ALL numpy scalar
# are json serializable: only those that are equal to Python's
ret.setdefault(moffit, {}).\
setdefault(imt, {})[gsim] = value.item()
except Exception as exc: # pylint: disable=broad-except
gsim_skipped[gsim] = str(exc)
return {
'Measure of fit': ret,
'Db records': obs_count,
'Gsim skipped': gsim_skipped
}
def check_gsim_defined_for_current_db(testdata):
'''no test function, it is used to inspect in debug mode in order to get
gsims with records in the current gmdb used for tests'''
for gsim in OQ.gsims():
try:
residuals = Residuals([gsim], ['PGA', 'PGV', 'SA(0.1)'])
gmdbpath = testdata.path('esm_sa_flatfile_2018.csv.hd5')
gm_table = GroundMotionTable(gmdbpath,
'esm_sa_flatfile_2018',
mode='r')
selexpr = get_selexpr(gsim)
num = gmdb_records(residuals, gm_table.filter(selexpr))
except:
pass
def setUpClass(cls):
"""
Import the database and the rupture before tests
"""
input_db = os.path.join(BASE_DATA_PATH, "LAquila_Database")
with open(os.path.join(input_db, "metadatafile.pkl"), "r") as fid:
cls.db = cPickle.load(fid)
input_rupture_file = os.path.join(BASE_DATA_PATH,
"laquila_rupture.xml")
cls.rupture = csim.build_rupture_from_file(input_rupture_file)
cls.gsims = ["AkkarEtAlRjb2014"]
cls.imts = ["PGA", "SA(1.0)"]
for rec in cls.db:
rec.datafile = os.path.join(os.path.dirname(__file__),
rec.datafile)
print(rec.datafile)
# Generate the residuals
cls.residuals = Residuals(cls.gsims, cls.imts)
cls.residuals.get_residuals(cls.db)
def get_conditional_gmfs(database, rupture, sites, gsims, imts,
number_simulations, truncation_level,
correlation_model=DEFAULT_CORRELATION):
"""
Get a set of random fields conditioned on a set of observations
:param database:
Ground motion records for the event as instance of :class:
smtk.sm_database.GroundMotionDatabase
:param rupture:
Event rupture as instance of :class:
openquake.hazardlib.source.rupture.Rupture
:param sites:
Target sites as instance of :class:
openquake.hazardlib.site.SiteCollection
:param list gsims:
List of GMPEs required
:param list imts:
List of intensity measures required
:param int number_simulations:
Number of simulated fields required
:param float truncation_level:
Ground motion truncation level
"""
# Get known sites mesh
known_sites = database.get_site_collection()
# Get Observed Residuals
residuals = Residuals(gsims, imts)
residuals.get_residuals(database)
imt_dict = OrderedDict([
(imtx, np.zeros([len(sites.lons), number_simulations]))
for imtx in imts])
gmfs = OrderedDict([(gmpe, imt_dict) for gmpe in gsims])
gmpe_list = [GSIM_LIST[gmpe]() for gmpe in gsims]
cmaker = ContextMaker(gmpe_list)
sctx, rctx, dctx = cmaker.make_contexts(sites, rupture)
for gsim in gmpe_list:
gmpe = gsim.__class__.__name__
#gsim = GSIM_LIST[gmpe]()
#sctx, rctx, dctx = gsim.make_contexts(sites, rupture)
for imtx in imts:
if truncation_level == 0:
gmfs[gmpe][imtx], _ = gsim.get_mean_and_stddevs(sctx, rctx,
dctx, from_string(imtx), stddev_types=[])
continue
if "Intra event" in gsim.DEFINED_FOR_STANDARD_DEVIATION_TYPES:
epsilon = conditional_simulation(
known_sites,
residuals.residuals[gmpe][imtx]["Intra event"],
sites,
imtx,
number_simulations,
correlation_model)
tau = np.unique(residuals.residuals[gmpe][imtx]["Inter event"])
mean, [stddev_inter, stddev_intra] = gsim.get_mean_and_stddevs(
sctx,
rctx,
dctx,
from_string(imtx),
["Inter event", "Intra event"])
for iloc in range(0, number_simulations):
gmfs[gmpe][imtx][:, iloc] = np.exp(
mean +
(tau * stddev_inter) +
(epsilon[:, iloc].A1 * stddev_intra))
else:
epsilon = conditional_simulation(
known_sites,
residuals.residuals[gmpe][imtx]["Total"],
sites,
imtx,
number_simulations,
correlation_model)
tau = None
mean, [stddev_total] = gsim.get_mean_and_stddevs(
sctx,
rctx,
dctx,
from_string(imtx),
["Total"])
for iloc in range(0, number_simulations):
gmfs[gmpe][imtx][:, iloc] = np.exp(
mean +
(epsilon[:, iloc].A1 * stddev_total.flatten()))
return gmfs