def get_residuals(self, database, nodal_plane_index=1, component="Geometric", normalise=True): """ Calculate the residuals for a set of ground motion records """ # Contexts is a list of dictionaries contexts = database.get_contexts(nodal_plane_index) self.database = SMRecordSelector(database) self.contexts = [] for context in contexts: # Get the observed strong ground motions context = self.get_observations(context, component) # Get the expected ground motions context = self.get_expected_motions(context) context = self.calculate_residuals(context, normalise) for gmpe in self.residuals.keys(): for imtx in self.residuals[gmpe].keys(): if not context["Residual"][gmpe][imtx]: continue for res_type in self.residuals[gmpe][imtx].keys(): if res_type == "Inter event": inter_ev = \ context["Residual"][gmpe][imtx][res_type] if np.all( np.fabs(inter_ev - inter_ev[0]) < 1.0E-12): # Single inter-event residual self.residuals[gmpe][imtx][res_type].append( inter_ev[0]) # Append indices self.unique_indices[gmpe][imtx].append( np.array([0])) else: # Inter event residuals per-site e.g. Chiou # & Youngs (2008; 2014) case self.residuals[gmpe][imtx][res_type].extend( inter_ev.tolist()) self.unique_indices[gmpe][imtx].append( np.arange(len(inter_ev))) else: self.residuals[gmpe][imtx][res_type].extend( context["Residual"][gmpe][imtx][res_type].tolist()) self.modelled[gmpe][imtx][res_type].extend( context["Expected"][gmpe][imtx][res_type].tolist()) self.modelled[gmpe][imtx]["Mean"].extend( context["Expected"][gmpe][imtx]["Mean"].tolist()) self.contexts.append(context) for gmpe in self.residuals.keys(): for imtx in self.residuals[gmpe].keys(): if not self.residuals[gmpe][imtx]: continue for res_type in self.residuals[gmpe][imtx].keys(): self.residuals[gmpe][imtx][res_type] = np.array( self.residuals[gmpe][imtx][res_type]) self.modelled[gmpe][imtx][res_type] = np.array( self.modelled[gmpe][imtx][res_type]) self.modelled[gmpe][imtx]["Mean"] = np.array( self.modelled[gmpe][imtx]["Mean"])
def db_magnitude_distance_by_trt(db1, dist_type, figure_size=(7, 5), filename=None, filetype="png", dpi=300): """ Plot magnitude-distance comparison by tectonic region """ trts = [] for i in db1.records: trts.append(i.event.tectonic_region) trt_types = list(set(trts)) selector = SMRecordSelector(db1) plt.figure(figsize=figure_size) for trt in trt_types: subdb = selector.select_trt_type(trt, as_db=True) mag, dists = get_magnitude_distances(subdb, dist_type) plt.semilogx(dists, mag, "o", mec='k', mew=0.5, label=trt) plt.xlabel(DISTANCE_LABEL[dist_type], fontsize=14) plt.ylabel("Magnitude", fontsize=14) plt.title("Magnitude vs Distance by Tectonic Region", fontsize=18) plt.legend(loc='lower right', numpoints=1) plt.grid() _save_image(filename, plt.gcf(), filetype, dpi) plt.show()
def db_magnitude_distance_by_site(db1, dist_type, classification="NEHRP", figure_size=(7, 5), filename=None, filetype="png", dpi=300): """ Plot magnitude-distance comparison by site NEHRP or Eurocode 8 Site class """ if classification == "NEHRP": site_bounds = NEHRP_BOUNDS elif classification == "EC8": site_bounds = EC8_BOUNDS else: raise ValueError("Unrecognised Site Classifier!") selector = SMRecordSelector(db1) plt.figure(figsize=figure_size) total_idx = [] for site_class in site_bounds.keys(): site_idx = _site_selection(db1, site_class, classification) if site_idx: site_db = selector.select_records(site_idx, as_db=True) mags, dists = get_magnitude_distances(site_db, dist_type) plt.plot(np.array(dists), np.array(mags), "o", mec='k', mew=0.5, label="Site Class %s" % site_class) total_idx.extend(site_idx) unc_idx = set(range(db1.number_records())).difference(set(total_idx)) unc_db = selector.select_records(unc_idx, as_db=True) mag, dists = get_magnitude_distances(site_db, dist_type) plt.semilogx(np.array(dists), np.array(mags), "o", mfc="None", mec='k', mew=0.5, label="Unclassified", zorder=0) plt.xlabel(DISTANCE_LABEL[dist_type], fontsize=14) plt.ylabel("Magnitude", fontsize=14) plt.grid() plt.legend(ncol=2,loc="lower right", numpoints=1) plt.title("Magnitude vs Distance (by %s Site Class)" % classification, fontsize=18) _save_image(filename, filetype, dpi) plt.show()
def get_residuals(self, database, nodal_plane_index=1, component="Geometric", normalise=True): """ Calculate the residuals for a set of ground motion records """ # Contexts is a list of dictionaries contexts = database.get_contexts(nodal_plane_index) self.database = SMRecordSelector(database) self.contexts = [] for context in contexts: # Get the observed strong ground motions context = self.get_observations(context, component) # Get the expected ground motions context = self.get_expected_motions(context) context = self.calculate_residuals(context, normalise) for gmpe in self.residuals.keys(): for imtx in self.residuals[gmpe].keys(): if not context["Residual"][gmpe][imtx]: continue for res_type in self.residuals[gmpe][imtx].keys(): if res_type == "Inter event": inter_ev = \ context["Residual"][gmpe][imtx][res_type] inter_ev, inter_idx = np.unique(inter_ev, return_index=True) self.residuals[gmpe][imtx][res_type].extend( inter_ev.tolist()) #inter_mags = (context["Rupture"].mag * # np.ones(len(inter_ev))).tolist() self.unique_indices[gmpe][imtx].append(inter_idx) else: self.residuals[gmpe][imtx][res_type].extend( context["Residual"][gmpe][imtx] [res_type].tolist()) self.modelled[gmpe][imtx][res_type].extend( context["Expected"][gmpe][imtx][res_type].tolist()) self.modelled[gmpe][imtx]["Mean"].extend( context["Expected"][gmpe][imtx]["Mean"].tolist()) self.contexts.append(context) for gmpe in self.residuals.keys(): for imtx in self.residuals[gmpe].keys(): if not self.residuals[gmpe][imtx]: continue for res_type in self.residuals[gmpe][imtx].keys(): self.residuals[gmpe][imtx][res_type] = np.array( self.residuals[gmpe][imtx][res_type]) self.modelled[gmpe][imtx][res_type] = np.array( self.modelled[gmpe][imtx][res_type]) self.modelled[gmpe][imtx]["Mean"] = np.array( self.modelled[gmpe][imtx]["Mean"])
def get_site_residuals(self, database, component="Geometric"): """ Calculates the total, inter-event and within-event residuals for each site """ imt_dict = dict([(imtx, {}) for imtx in self.imts]) for site_id in self.site_ids: print site_id selector = SMRecordSelector(database) site_db = selector.select_from_site_id(site_id, as_db=True) resid = Residuals(self.input_gmpe_list, self.imts) resid.get_residuals(site_db, normalise=False, component=component) setattr(resid, "site_analysis", self._set_empty_dict()) setattr(resid, "site_expected", self._set_empty_dict()) self.site_residuals.append(resid)
def get_site_residuals(self, database): """ Calculates the total, inter-event and within-event residuals for each site """ imt_dict = dict([(imtx, {}) for imtx in self.imts]) for site_id in self.site_ids: print site_id selector = SMRecordSelector(database) site_db = selector.select_from_site_id(site_id, as_db=True) resid = Residuals(self.input_gmpe_list, self.imts) resid.get_residuals(site_db, normalise=False) setattr( resid, "site_analysis", self._set_empty_dict()) setattr( resid, "site_expected", self._set_empty_dict()) self.site_residuals.append(resid)
def get_site_residuals(self, database): """ Calculates the total, inter-event and within-event residuals for each site """ imt_dict = dict([(imtx, {}) for imtx in self.imts]) for site_id in self.site_ids: print site_id selector = SMRecordSelector(database) site_db = selector.select_from_site_id(site_id, as_db=True) resid = Residuals(self.gmpe_list, self.imts) resid.get_residuals(site_db, normalise=False) setattr( resid, "site_analysis", OrderedDict([(gmpe, imt_dict) for gmpe in self.gmpe_list])) setattr( resid, "site_expected", OrderedDict([(gmpe, imt_dict) for gmpe in self.gmpe_list])) self.site_residuals.append(resid)
def db_magnitude_distance_by_trt(db1, dist_type, figure_size=(7, 5), filename=None, filetype="png", dpi=300): """ Plot magnitude-distance comparison by tectonic region """ trts=[] for i in db1.records: trts.append(i.event.tectonic_region) trt_types=list(set(trts)) selector = SMRecordSelector(db1) plt.figure(figsize=figure_size) for trt in trt_types: subdb = selector.select_trt_type(trt, as_db=True) mag, dists = get_magnitude_distances(subdb, dist_type) plt.semilogx(dists, mag, "o", mec='k', mew=0.5, label=trt) plt.xlabel(DISTANCE_LABEL[dist_type], fontsize=14) plt.ylabel("Magnitude", fontsize=14) plt.title("Magnitude vs Distance by Tectonic Region", fontsize=18) plt.legend(loc='lower right', numpoints=1) plt.grid() _save_image(filename, filetype, dpi) plt.show()
def get_residuals(self, database, nodal_plane_index=1, component="Geometric", normalise=True): """ Calculate the residuals for a set of ground motion records """ # Contexts is a list of dictionaries contexts = database.get_contexts(nodal_plane_index) self.database = SMRecordSelector(database) self.contexts = [] for context in contexts: # Get the observed strong ground motions context = self.get_observations(context, component) # Get the expected ground motions context = self.get_expected_motions(context) context = self.calculate_residuals(context, normalise) for gmpe in self.residuals.keys(): for imtx in self.residuals[gmpe].keys(): for res_type in self.residuals[gmpe][imtx].keys(): if res_type == "Inter event": inter_ev = \ context["Residual"][gmpe][imtx][res_type] inter_ev, inter_idx = np.unique( inter_ev, return_index=True) self.residuals[gmpe][imtx][res_type].extend( inter_ev.tolist()) #inter_mags = (context["Rupture"].mag * # np.ones(len(inter_ev))).tolist() self.unique_indices[gmpe][imtx].append( inter_idx) else: self.residuals[gmpe][imtx][res_type].extend( context["Residual"][gmpe][imtx][res_type].tolist()) self.modelled[gmpe][imtx][res_type].extend( context["Expected"][gmpe][imtx][res_type].tolist()) self.modelled[gmpe][imtx]["Mean"].extend( context["Expected"][gmpe][imtx]["Mean"].tolist()) self.contexts.append(context) for gmpe in self.residuals.keys(): for imtx in self.residuals[gmpe].keys(): for res_type in self.residuals[gmpe][imtx].keys(): self.residuals[gmpe][imtx][res_type] = np.array( self.residuals[gmpe][imtx][res_type]) self.modelled[gmpe][imtx][res_type] = np.array( self.modelled[gmpe][imtx][res_type]) self.modelled[gmpe][imtx]["Mean"] = np.array( self.modelled[gmpe][imtx]["Mean"])
def get_residuals(self, database, nodal_plane_index=1, component="Geometric", normalise=True): """ Calculate the residuals for a set of ground motion records """ # Contexts is a list of dictionaries contexts = database.get_contexts(nodal_plane_index) self.database = SMRecordSelector(database) self.contexts = [] for context in contexts: #print context # Get the observed strong ground motions context = self.get_observations(context, component) # Get the expected ground motions context = self.get_expected_motions(context) context = self.calculate_residuals(context, normalise) for gmpe in self.residuals.keys(): for imtx in self.residuals[gmpe].keys(): for res_type in self.residuals[gmpe][imtx].keys(): self.residuals[gmpe][imtx][res_type].extend( context["Residual"][gmpe][imtx][res_type].tolist()) self.modelled[gmpe][imtx][res_type].extend( context["Expected"][gmpe][imtx][res_type].tolist()) self.modelled[gmpe][imtx]["Mean"].extend( context["Expected"][gmpe][imtx]["Mean"].tolist()) self.contexts.append(context) for gmpe in self.residuals.keys(): for imtx in self.residuals[gmpe].keys(): for res_type in self.residuals[gmpe][imtx].keys(): self.residuals[gmpe][imtx][res_type] = np.array( self.residuals[gmpe][imtx][res_type]) self.modelled[gmpe][imtx][res_type] = np.array( self.modelled[gmpe][imtx][res_type]) self.modelled[gmpe][imtx]["Mean"] = np.array( self.modelled[gmpe][imtx]["Mean"])
def get_residuals(self, database, nodal_plane_index=1, component="Geometric", normalise=True): """ Calculate the residuals for a set of ground motion records """ # Contexts is a list of dictionaries contexts = database.get_contexts(nodal_plane_index) self.database = SMRecordSelector(database) self.contexts = [] for context in contexts: # Get the observed strong ground motions context = self.get_observations(context, component) # Get the expected ground motions context = self.get_expected_motions(context) context = self.calculate_residuals(context, normalise) for gmpe in self.residuals.keys(): for imtx in self.residuals[gmpe].keys(): for res_type in self.residuals[gmpe][imtx].keys(): self.residuals[gmpe][imtx][res_type].extend( context["Residual"][gmpe][imtx][res_type].tolist()) self.modelled[gmpe][imtx][res_type].extend( context["Expected"][gmpe][imtx][res_type].tolist()) self.modelled[gmpe][imtx]["Mean"].extend( context["Expected"][gmpe][imtx]["Mean"].tolist()) self.contexts.append(context) for gmpe in self.residuals.keys(): for imtx in self.residuals[gmpe].keys(): for res_type in self.residuals[gmpe][imtx].keys(): self.residuals[gmpe][imtx][res_type] = np.array( self.residuals[gmpe][imtx][res_type]) self.modelled[gmpe][imtx][res_type] = np.array( self.modelled[gmpe][imtx][res_type]) self.modelled[gmpe][imtx]["Mean"] = np.array( self.modelled[gmpe][imtx]["Mean"])
class Residuals(object): """ Class to derive sets of residuals for a list of ground motion residuals according to the GMPEs """ def __init__(self, gmpe_list, imts): """ :param list gmpe_list: List of GMPE names (using the standard openquake strings) :param list imts: List of Intensity Measures """ self.gmpe_list = _check_gsim_list(gmpe_list) self.number_gmpes = len(self.gmpe_list) self.types = OrderedDict([(gmpe, {}) for gmpe in self.gmpe_list]) self.residuals = [] self.modelled = [] self.imts = imts self.unique_indices = {} for gmpe in self.gmpe_list: gmpe_dict_1 = {} gmpe_dict_2 = {} self.unique_indices[gmpe] = {} for imtx in self.imts: gmpe_dict_1[imtx] = {} gmpe_dict_2[imtx] = {} self.unique_indices[gmpe][imtx] = [] self.types[gmpe][imtx] = [] for res_type in \ self.gmpe_list[gmpe].DEFINED_FOR_STANDARD_DEVIATION_TYPES: gmpe_dict_1[imtx][res_type] = [] gmpe_dict_2[imtx][res_type] = [] self.types[gmpe][imtx].append(res_type) gmpe_dict_2[imtx]["Mean"] = [] self.residuals.append([gmpe, gmpe_dict_1]) self.modelled.append([gmpe, gmpe_dict_2]) self.residuals = OrderedDict(self.residuals) self.modelled = OrderedDict(self.modelled) self.database = None self.number_records = None self.contexts = None def get_residuals(self, database, nodal_plane_index=1, component="Geometric", normalise=True): """ Calculate the residuals for a set of ground motion records """ # Contexts is a list of dictionaries contexts = database.get_contexts(nodal_plane_index) self.database = SMRecordSelector(database) self.contexts = [] for context in contexts: # Get the observed strong ground motions context = self.get_observations(context, component) # Get the expected ground motions context = self.get_expected_motions(context) context = self.calculate_residuals(context, normalise) for gmpe in self.residuals.keys(): for imtx in self.residuals[gmpe].keys(): for res_type in self.residuals[gmpe][imtx].keys(): if res_type == "Inter event": inter_ev = \ context["Residual"][gmpe][imtx][res_type] inter_ev, inter_idx = np.unique( inter_ev, return_index=True) self.residuals[gmpe][imtx][res_type].extend( inter_ev.tolist()) #inter_mags = (context["Rupture"].mag * # np.ones(len(inter_ev))).tolist() self.unique_indices[gmpe][imtx].append( inter_idx) else: self.residuals[gmpe][imtx][res_type].extend( context["Residual"][gmpe][imtx][res_type].tolist()) self.modelled[gmpe][imtx][res_type].extend( context["Expected"][gmpe][imtx][res_type].tolist()) self.modelled[gmpe][imtx]["Mean"].extend( context["Expected"][gmpe][imtx]["Mean"].tolist()) self.contexts.append(context) for gmpe in self.residuals.keys(): for imtx in self.residuals[gmpe].keys(): for res_type in self.residuals[gmpe][imtx].keys(): self.residuals[gmpe][imtx][res_type] = np.array( self.residuals[gmpe][imtx][res_type]) self.modelled[gmpe][imtx][res_type] = np.array( self.modelled[gmpe][imtx][res_type]) self.modelled[gmpe][imtx]["Mean"] = np.array( self.modelled[gmpe][imtx]["Mean"]) #self.unique_magnitudes[gmpe][imtx] = np.array( # self.unique_magnitudes[gmpe][imtx]) def get_observations(self, context, component="Geometric"): """ Get the obsered ground motions from the database """ select_records = self.database.select_from_event_id(context["EventID"]) observations = OrderedDict([(imtx, []) for imtx in self.imts]) selection_string = "IMS/H/Spectra/Response/Acceleration/" for record in select_records: fle = h5py.File(record.datafile, "r") for imtx in self.imts: if imtx in SCALAR_IMTS: if imtx == "PGA": observations[imtx].append( get_scalar(fle, imtx, component) / 981.0) else: observations[imtx].append( get_scalar(fle, imtx, component)) elif "SA(" in imtx: target_period = imt.from_string(imtx).period spectrum = fle[selection_string + component + "/damping_05"].value periods = fle["IMS/H/Spectra/Response/Periods"].value observations[imtx].append(get_interpolated_period( target_period, periods, spectrum) / 981.0) else: raise "IMT %s is unsupported!" % imtx fle.close() for imtx in self.imts: observations[imtx] = np.array(observations[imtx]) context["Observations"] = observations context["Num. Sites"] = len(select_records) return context def get_expected_motions(self, context): """ Calculate the expected ground motions from the context """ # TODO Rake hack will be removed!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! if not context["Rupture"].rake: context["Rupture"].rake = 0.0 expected = OrderedDict([(gmpe, {}) for gmpe in self.gmpe_list]) for gmpe in self.gmpe_list: expected[gmpe] = OrderedDict([(imtx, {}) for imtx in self.imts]) for imtx in self.imts: gsim = self.gmpe_list[gmpe] mean, stddev = gsim.get_mean_and_stddevs( context["Sites"], context["Rupture"], context["Distances"], imt.from_string(imtx), self.types[gmpe][imtx]) expected[gmpe][imtx]["Mean"] = mean for i, res_type in enumerate(self.types[gmpe][imtx]): expected[gmpe][imtx][res_type] = stddev[i] context["Expected"] = expected return context def calculate_residuals(self, context, normalise=True): """ Calculate the residual terms """ # Calculate residual residual = {} for gmpe in self.gmpe_list: residual[gmpe] = {} for imtx in self.imts: residual[gmpe][imtx] = {} obs = np.log(context["Observations"][imtx]) mean = context["Expected"][gmpe][imtx]["Mean"] total_stddev = context["Expected"][gmpe][imtx]["Total"] residual[gmpe][imtx]["Total"] = (obs - mean) / total_stddev if "Inter event" in self.residuals[gmpe][imtx].keys(): inter, intra = self._get_random_effects_residuals( obs, mean, context["Expected"][gmpe][imtx]["Inter event"], context["Expected"][gmpe][imtx]["Intra event"], normalise) residual[gmpe][imtx]["Inter event"] = inter residual[gmpe][imtx]["Intra event"] = intra context["Residual"] = residual return context def _get_random_effects_residuals(self, obs, mean, inter, intra, normalise=True): """ Calculates the random effects residuals using the inter-event residual formula described in Abrahamson & Youngs (1992) Eq. 10 """ nvals = float(len(mean)) inter_res = ((inter ** 2.) * sum(obs - mean)) /\ (nvals * (inter ** 2.) + (intra ** 2.)) intra_res = obs - (mean + inter_res) if normalise: return inter_res / inter, intra_res / intra else: return inter_res, intra_res def get_residual_statistics(self): """ Retreives the mean and standard deviation values of the residuals """ statistics = OrderedDict([(gmpe, {}) for gmpe in self.gmpe_list]) for gmpe in self.gmpe_list: for imtx in self.imts: statistics[gmpe][imtx] = {} for res_type in self.types[gmpe][imtx]: # if res_type == "Inter event": # # As the inter-event term may be vectorial with # # repeated columns, get take only one value of inter- # # event residual if unique # delta_e = np.array([], dtype="float") # for ctxt in self.contexts: # iev_res = np.unique( # ctxt["Residual"][gmpe][imtx]["Inter event"]) # delta_e = np.hstack([delta_e, iev_res]) # #print delta_e # data = { # "Mean": np.mean(delta_e), # #self.residuals[gmpe][imtx][res_type]), # "Std Dev": np.std(delta_e)} # #self.residuals[gmpe][imtx][res_type])} # else: data = { "Mean": np.mean( self.residuals[gmpe][imtx][res_type]), "Std Dev": np.std( self.residuals[gmpe][imtx][res_type])} statistics[gmpe][imtx][res_type] = data return statistics def pretty_print(self, filename=None, sep=","): """ Print the information to screen or to file """ if filename: fid = open(filename, "w") else: fid = sys.stdout fid.write("Ground Motion Residuals\n") # Prin headers event = self.contexts[0] header_set = [] header_set.extend([key for key in event["Distances"].__dict__]) header_set.extend([key for key in event["Sites"].__dict__]) header_set.extend(["{:s}-Obs.".format(imt) for imt in self.imts]) for imt in self.imts: for gmpe in self.gmpe_list: for key in event["Expected"][gmpe][imt].keys(): header_set.append( "{:s}-{:s}-{:s}-Exp.".format(imt, gmpe, key)) for imt in self.imts: for gmpe in self.gmpe_list: for key in event["Residual"][gmpe][imt].keys(): header_set.append( "{:s}-{:s}-{:s}-Res.".format(imt, gmpe, key)) header_set = self._extend_header_set(header_set) fid.write("%s\n" % sep.join(header_set)) for event in self.contexts: self._pprint_event(fid, event, sep) if filename: fid.close() def _pprint_event(self, fid, event, sep): """ Pretty print the information for each event """ # Print rupture info rupture_str = sep.join([ "{:s}{:s}{:s}".format(key, sep, str(val)) for key, val in event["Rupture"].__dict__.items()]) fid.write("Rupture: %s %s %s\n" % (str(event["EventID"]), sep, rupture_str)) # For each record for i in range(event["Num. Sites"]): data = [] # Distances for key in event["Distances"].__dict__: data.append("{:.4f}".format( getattr(event["Distances"], key)[i])) # Sites for key in event["Sites"].__dict__: data.append("{:.4f}".format(getattr(event["Sites"], key)[i])) # Observations for imt in self.imts: data.append("{:.8e}".format(event["Observations"][imt][i])) # Expected for imt in self.imts: for gmpe in self.gmpe_list: for key in event["Expected"][gmpe][imt].keys(): data.append("{:.8e}".format( event["Expected"][gmpe][imt][key][i])) # Residuals for imt in self.imts: for gmpe in self.gmpe_list: for key in event["Residual"][gmpe][imt].keys(): data.append("{:.8e}".format( event["Residual"][gmpe][imt][key][i])) self._extend_data_print(data, event, i) fid.write("%s\n" % sep.join(data)) def _extend_header_set(self, header_set): """ Additional headers to add to the pretty print - does nothing here but overwritten in subclasses """ return header_set def _extend_data_print(self, data, event, i): """ Additional data to add to the pretty print - also does nothing here but overwritten in subclasses """ return data def _get_magnitudes(self): """ Returns an array of magnitudes equal in length to the number of residuals """ magnitudes = np.array([]) for ctxt in self.contexts: magnitudes = np.hstack([ magnitudes, ctxt["Rupture"].mag * np.ones(len(ctxt["Distances"].repi))]) return magnitudes
class Residuals(object): """ Class to derive sets of residuals for a list of ground motion residuals according to the GMPEs """ def __init__(self, gmpe_list, imts): """ :param list gmpe_list: List of GMPE names (using the standard openquake strings) :param list imts: List of Intensity Measures """ self.gmpe_list = _check_gsim_list(gmpe_list) self.number_gmpes = len(self.gmpe_list) self.types = OrderedDict([(gmpe, {}) for gmpe in self.gmpe_list]) self.residuals = [] self.modelled = [] self.imts = imts self.unique_indices = {} self.gmpe_sa_limits = {} self.gmpe_scalars = {} for gmpe in self.gmpe_list: gmpe_dict_1 = OrderedDict([]) gmpe_dict_2 = OrderedDict([]) self.unique_indices[gmpe] = {} # Get the period range and the coefficient types gmpe_i = GSIM_LIST[gmpe]() for c in dir(gmpe_i): if 'COEFFS' in c: pers = [sa.period for sa in getattr(gmpe_i,c).sa_coeffs] min_per, max_per = (min(pers), max(pers)) self.gmpe_sa_limits[gmpe] = (min_per, max_per) for c in dir(gmpe_i): if 'COEFFS' in c: self.gmpe_scalars[gmpe] = getattr(gmpe_i,c).non_sa_coeffs.keys() for imtx in self.imts: if "SA(" in imtx: period = imt.from_string(imtx).period if period < min_per or period > max_per: print("IMT %s outside period range for GMPE %s" % (imtx, gmpe)) gmpe_dict_1[imtx] = None gmpe_dict_2[imtx] = None continue gmpe_dict_1[imtx] = {} gmpe_dict_2[imtx] = {} self.unique_indices[gmpe][imtx] = [] self.types[gmpe][imtx] = [] for res_type in \ self.gmpe_list[gmpe].DEFINED_FOR_STANDARD_DEVIATION_TYPES: gmpe_dict_1[imtx][res_type] = [] gmpe_dict_2[imtx][res_type] = [] self.types[gmpe][imtx].append(res_type) gmpe_dict_2[imtx]["Mean"] = [] self.residuals.append([gmpe, gmpe_dict_1]) self.modelled.append([gmpe, gmpe_dict_2]) self.residuals = OrderedDict(self.residuals) self.modelled = OrderedDict(self.modelled) self.database = None self.number_records = None self.contexts = None def get_residuals(self, database, nodal_plane_index=1, component="Geometric", normalise=True): """ Calculate the residuals for a set of ground motion records """ # Contexts is a list of dictionaries contexts = database.get_contexts(nodal_plane_index) self.database = SMRecordSelector(database) self.contexts = [] for context in contexts: # Get the observed strong ground motions context = self.get_observations(context, component) # Get the expected ground motions context = self.get_expected_motions(context) context = self.calculate_residuals(context, normalise) for gmpe in self.residuals.keys(): for imtx in self.residuals[gmpe].keys(): if not context["Residual"][gmpe][imtx]: continue for res_type in self.residuals[gmpe][imtx].keys(): if res_type == "Inter event": inter_ev = \ context["Residual"][gmpe][imtx][res_type] if np.all( np.fabs(inter_ev - inter_ev[0]) < 1.0E-12): # Single inter-event residual self.residuals[gmpe][imtx][res_type].append( inter_ev[0]) # Append indices self.unique_indices[gmpe][imtx].append( np.array([0])) else: # Inter event residuals per-site e.g. Chiou # & Youngs (2008; 2014) case self.residuals[gmpe][imtx][res_type].extend( inter_ev.tolist()) self.unique_indices[gmpe][imtx].append( np.arange(len(inter_ev))) else: self.residuals[gmpe][imtx][res_type].extend( context["Residual"][gmpe][imtx][res_type].tolist()) self.modelled[gmpe][imtx][res_type].extend( context["Expected"][gmpe][imtx][res_type].tolist()) self.modelled[gmpe][imtx]["Mean"].extend( context["Expected"][gmpe][imtx]["Mean"].tolist()) self.contexts.append(context) for gmpe in self.residuals.keys(): for imtx in self.residuals[gmpe].keys(): if not self.residuals[gmpe][imtx]: continue for res_type in self.residuals[gmpe][imtx].keys(): self.residuals[gmpe][imtx][res_type] = np.array( self.residuals[gmpe][imtx][res_type]) self.modelled[gmpe][imtx][res_type] = np.array( self.modelled[gmpe][imtx][res_type]) self.modelled[gmpe][imtx]["Mean"] = np.array( self.modelled[gmpe][imtx]["Mean"]) def get_observations(self, context, component="Geometric"): """ Get the obsered ground motions from the database """ select_records = self.database.select_from_event_id(context["EventID"]) observations = OrderedDict([(imtx, []) for imtx in self.imts]) selection_string = "IMS/H/Spectra/Response/Acceleration/" for record in select_records: fle = h5py.File(record.datafile, "r") for imtx in self.imts: if imtx in SCALAR_IMTS: if imtx == "PGA": observations[imtx].append( get_scalar(fle, imtx, component) / 981.0) else: observations[imtx].append( get_scalar(fle, imtx, component)) elif "SA(" in imtx: target_period = imt.from_string(imtx).period spectrum = fle[selection_string + component + "/damping_05"].value periods = fle["IMS/H/Spectra/Response/Periods"].value observations[imtx].append(get_interpolated_period( target_period, periods, spectrum) / 981.0) else: raise "IMT %s is unsupported!" % imtx fle.close() for imtx in self.imts: observations[imtx] = np.array(observations[imtx]) context["Observations"] = observations context["Num. Sites"] = len(select_records) return context def get_expected_motions(self, context): """ Calculate the expected ground motions from the context """ # TODO Rake hack will be removed!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! if not context["Rupture"].rake: context["Rupture"].rake = 0.0 expected = OrderedDict([(gmpe, {}) for gmpe in self.gmpe_list]) # Period range for GSIM for gmpe in self.gmpe_list: expected[gmpe] = OrderedDict([(imtx, {}) for imtx in self.imts]) for imtx in self.imts: gsim = self.gmpe_list[gmpe] if "SA(" in imtx: period = imt.from_string(imtx).period if period < self.gmpe_sa_limits[gmpe][0] or\ period > self.gmpe_sa_limits[gmpe][1]: expected[gmpe][imtx] = None continue mean, stddev = gsim.get_mean_and_stddevs( context["Sites"], context["Rupture"], context["Distances"], imt.from_string(imtx), self.types[gmpe][imtx]) expected[gmpe][imtx]["Mean"] = mean for i, res_type in enumerate(self.types[gmpe][imtx]): expected[gmpe][imtx][res_type] = stddev[i] context["Expected"] = expected return context def calculate_residuals(self, context, normalise=True): """ Calculate the residual terms """ # Calculate residual residual = {} for gmpe in self.gmpe_list: residual[gmpe] = OrderedDict([]) for imtx in self.imts: residual[gmpe][imtx] = {} obs = np.log(context["Observations"][imtx]) if not context["Expected"][gmpe][imtx]: residual[gmpe][imtx] = None continue mean = context["Expected"][gmpe][imtx]["Mean"] total_stddev = context["Expected"][gmpe][imtx]["Total"] residual[gmpe][imtx]["Total"] = (obs - mean) / total_stddev if "Inter event" in self.residuals[gmpe][imtx].keys(): inter, intra = self._get_random_effects_residuals( obs, mean, context["Expected"][gmpe][imtx]["Inter event"], context["Expected"][gmpe][imtx]["Intra event"], normalise) residual[gmpe][imtx]["Inter event"] = inter residual[gmpe][imtx]["Intra event"] = intra context["Residual"] = residual return context def _get_random_effects_residuals(self, obs, mean, inter, intra, normalise=True): """ Calculates the random effects residuals using the inter-event residual formula described in Abrahamson & Youngs (1992) Eq. 10 """ nvals = float(len(mean)) inter_res = ((inter ** 2.) * sum(obs - mean)) /\ (nvals * (inter ** 2.) + (intra ** 2.)) intra_res = obs - (mean + inter_res) if normalise: return inter_res / inter, intra_res / intra else: return inter_res, intra_res def get_residual_statistics(self): """ Retreives the mean and standard deviation values of the residuals """ statistics = OrderedDict([(gmpe, OrderedDict([])) for gmpe in self.gmpe_list]) for gmpe in self.gmpe_list: for imtx in self.imts: if not self.residuals[gmpe][imtx]: continue statistics[gmpe][imtx] = {} for res_type in self.types[gmpe][imtx]: data = { "Mean": np.mean( self.residuals[gmpe][imtx][res_type]), "Std Dev": np.std( self.residuals[gmpe][imtx][res_type])} statistics[gmpe][imtx][res_type] = data return statistics def pretty_print(self, filename=None, sep=","): """ Print the information to screen or to file """ if filename: fid = open(filename, "w") else: fid = sys.stdout fid.write("Ground Motion Residuals\n") # Print headers event = self.contexts[0] header_set = [] header_set.extend([key for key in event["Distances"].__dict__]) header_set.extend([key for key in event["Sites"].__dict__]) header_set.extend(["{:s}-Obs.".format(imt) for imt in self.imts]) for imt in self.imts: for gmpe in self.gmpe_list: if not event["Expected"][gmpe][imt]: continue for key in event["Expected"][gmpe][imt].keys(): header_set.append( "{:s}-{:s}-{:s}-Exp.".format(imt, gmpe, key)) for imt in self.imts: for gmpe in self.gmpe_list: if not event["Residual"][gmpe][imt]: continue for key in event["Residual"][gmpe][imt].keys(): header_set.append( "{:s}-{:s}-{:s}-Res.".format(imt, gmpe, key)) header_set = self._extend_header_set(header_set) fid.write("%s\n" % sep.join(header_set)) for event in self.contexts: self._pprint_event(fid, event, sep) if filename: fid.close() def _pprint_event(self, fid, event, sep): """ Pretty print the information for each event """ # Print rupture info rupture_str = sep.join([ "{:s}{:s}{:s}".format(key, sep, str(val)) for key, val in event["Rupture"].__dict__.items()]) fid.write("Rupture: %s %s %s\n" % (str(event["EventID"]), sep, rupture_str)) # For each record for i in range(event["Num. Sites"]): data = [] # Distances for key in event["Distances"].__dict__: data.append("{:.4f}".format( getattr(event["Distances"], key)[i])) # Sites for key in event["Sites"].__dict__: data.append("{:.4f}".format(getattr(event["Sites"], key)[i])) # Observations for imt in self.imts: data.append("{:.8e}".format(event["Observations"][imt][i])) # Expected for imt in self.imts: for gmpe in self.gmpe_list: if not event["Expected"][gmpe][imt]: continue for key in event["Expected"][gmpe][imt].keys(): data.append("{:.8e}".format( event["Expected"][gmpe][imt][key][i])) # Residuals for imt in self.imts: for gmpe in self.gmpe_list: if not event["Expected"][gmpe][imt]: continue for key in event["Residual"][gmpe][imt].keys(): data.append("{:.8e}".format( event["Residual"][gmpe][imt][key][i])) self._extend_data_print(data, event, i) fid.write("%s\n" % sep.join(data)) def _extend_header_set(self, header_set): """ Additional headers to add to the pretty print - does nothing here but overwritten in subclasses """ return header_set def _extend_data_print(self, data, event, i): """ Additional data to add to the pretty print - also does nothing here but overwritten in subclasses """ return data def _get_magnitudes(self): """ Returns an array of magnitudes equal in length to the number of residuals """ magnitudes = np.array([]) for ctxt in self.contexts: magnitudes = np.hstack([ magnitudes, ctxt["Rupture"].mag * np.ones(len(ctxt["Distances"].repi))]) return magnitudes
class Residuals(object): """ Class to derive sets of residuals for a list of ground motion residuals according to the GMPEs """ def __init__(self, gmpe_list, imts): """ :param list gmpe_list: List of GMPE names (using the standard openquake strings) :param list imts: List of Intensity Measures """ self.gmpe_list = gmpe_list self.number_gmpes = len(self.gmpe_list) self.types = OrderedDict([(gmpe, {}) for gmpe in self.gmpe_list]) self.residuals = [] self.modelled = [] self.imts = imts for gmpe in gmpe_list: if not gmpe in GSIM_LIST: raise ValueError("%s not supported in OpenQuake" % gmpe) gmpe_dict_1 = {} gmpe_dict_2 = {} for imtx in self.imts: gmpe_dict_1[imtx] = {} gmpe_dict_2[imtx] = {} self.types[gmpe][imtx] = [] for res_type in \ GSIM_LIST[gmpe].DEFINED_FOR_STANDARD_DEVIATION_TYPES: gmpe_dict_1[imtx][res_type] = [] gmpe_dict_2[imtx][res_type] = [] self.types[gmpe][imtx].append(res_type) gmpe_dict_2[imtx]["Mean"] = [] self.residuals.append([gmpe, gmpe_dict_1]) self.modelled.append([gmpe, gmpe_dict_2]) self.residuals = OrderedDict(self.residuals) self.modelled = OrderedDict(self.modelled) self.database = None self.number_records = None self.contexts = None def get_residuals(self, database, nodal_plane_index=1, component="Geometric", normalise=True): """ Calculate the residuals for a set of ground motion records """ # Contexts is a list of dictionaries contexts = database.get_contexts(nodal_plane_index) self.database = SMRecordSelector(database) self.contexts = [] for context in contexts: # Get the observed strong ground motions context = self.get_observations(context, component) # Get the expected ground motions context = self.get_expected_motions(context) context = self.calculate_residuals(context, normalise) for gmpe in self.residuals.keys(): for imtx in self.residuals[gmpe].keys(): for res_type in self.residuals[gmpe][imtx].keys(): self.residuals[gmpe][imtx][res_type].extend( context["Residual"][gmpe][imtx][res_type].tolist()) self.modelled[gmpe][imtx][res_type].extend( context["Expected"][gmpe][imtx][res_type].tolist()) self.modelled[gmpe][imtx]["Mean"].extend( context["Expected"][gmpe][imtx]["Mean"].tolist()) self.contexts.append(context) for gmpe in self.residuals.keys(): for imtx in self.residuals[gmpe].keys(): for res_type in self.residuals[gmpe][imtx].keys(): self.residuals[gmpe][imtx][res_type] = np.array( self.residuals[gmpe][imtx][res_type]) self.modelled[gmpe][imtx][res_type] = np.array( self.modelled[gmpe][imtx][res_type]) self.modelled[gmpe][imtx]["Mean"] = np.array( self.modelled[gmpe][imtx]["Mean"]) def get_observations(self, context, component="Geometric"): """ Get the obsered ground motions from the database """ select_records = self.database.select_from_event_id(context["EventID"]) observations = OrderedDict([(imtx, []) for imtx in self.imts]) selection_string = "IMS/H/Spectra/Response/Acceleration/" for record in select_records: fle = h5py.File(record.datafile, "r") for imtx in self.imts: if imtx in SCALAR_IMTS: if imtx == "PGA": observations[imtx].append( get_scalar(fle, imtx, component) / 981.0) else: observations[imtx].append( get_scalar(fle, imtx, component)) elif "SA(" in imtx: target_period = imt.from_string(imtx).period spectrum = fle[selection_string + component + "/damping_05"].value periods = fle["IMS/H/Spectra/Response/Periods"].value observations[imtx].append(get_interpolated_period( target_period, periods, spectrum) / 981.0) else: raise "IMT %s is unsupported!" % imtx fle.close() for imtx in self.imts: observations[imtx] = np.array(observations[imtx]) context["Observations"] = observations return context def get_expected_motions(self, context): """ Calculate the expected ground motions from the context """ # TODO Rake hack will be removed!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! if not context["Rupture"].rake: context["Rupture"].rake = 0.0 expected = OrderedDict([(gmpe, {}) for gmpe in self.gmpe_list]) for gmpe in self.gmpe_list: expected[gmpe] = OrderedDict([(imtx, {}) for imtx in self.imts]) for imtx in self.imts: for res_type in self.types[gmpe][imtx]: gsim = GSIM_LIST[gmpe]() mean, stddev = gsim.get_mean_and_stddevs( context["Sites"], context["Rupture"], context["Distances"], imt.from_string(imtx), [res_type]) expected[gmpe][imtx]["Mean"] = mean expected[gmpe][imtx][res_type] = stddev[0] context["Expected"] = expected return context def calculate_residuals(self, context, normalise=True): """ Calculate the residual terms """ # Calculate residual residual = {} for gmpe in self.gmpe_list: residual[gmpe] = {} for imtx in self.imts: residual[gmpe][imtx] = {} obs = np.log(context["Observations"][imtx]) mean = context["Expected"][gmpe][imtx]["Mean"] total_stddev = context["Expected"][gmpe][imtx]["Total"] residual[gmpe][imtx]["Total"] = (obs - mean) / total_stddev if "Inter event" in self.residuals[gmpe][imtx].keys(): inter, intra = self._get_random_effects_residuals( obs, mean, context["Expected"][gmpe][imtx]["Inter event"], context["Expected"][gmpe][imtx]["Intra event"], normalise) residual[gmpe][imtx]["Inter event"] = inter residual[gmpe][imtx]["Intra event"] = intra context["Residual"] = residual return context def _get_random_effects_residuals(self, obs, mean, inter, intra, normalise=True): """ Calculates the random effects residuals using the inter-event residual formula described in Abrahamson & Youngs (1992) Eq. 10 """ nvals = float(len(mean)) inter_res = ((inter ** 2.) * sum(obs - mean)) /\ (nvals * (inter ** 2.) + (intra ** 2.)) intra_res = obs - (mean + inter_res) if normalise: return inter_res / inter, intra_res / intra else: return inter_res, intra_res def get_residual_statistics(self): """ Retreives the mean and standard deviation values of the residuals """ statistics = OrderedDict([(gmpe, {}) for gmpe in self.gmpe_list]) for gmpe in self.gmpe_list: for imtx in self.imts: statistics[gmpe][imtx] = {} for res_type in self.types[gmpe][imtx]: data = { "Mean": np.mean( self.residuals[gmpe][imtx][res_type]), "Std Dev": np.std( self.residuals[gmpe][imtx][res_type])} statistics[gmpe][imtx][res_type] = data return statistics
class Residuals(object): """ Class to derive sets of residuals for a list of ground motion residuals according to the GMPEs """ def __init__(self, gmpe_list, imts): """ :param list gmpe_list: List of GMPE names (using the standard openquake strings) :param list imts: List of Intensity Measures """ self.gmpe_list = gmpe_list self.number_gmpes = len(self.gmpe_list) self.types = OrderedDict([(gmpe, {}) for gmpe in self.gmpe_list]) self.residuals = [] self.modelled = [] self.imts = imts for gmpe in gmpe_list: if not gmpe in GSIM_LIST: raise ValueError("%s not supported in OpenQuake" % gmpe) gmpe_dict_1 = {} gmpe_dict_2 = {} for imtx in self.imts: gmpe_dict_1[imtx] = {} gmpe_dict_2[imtx] = {} self.types[gmpe][imtx] = [] for res_type in \ GSIM_LIST[gmpe].DEFINED_FOR_STANDARD_DEVIATION_TYPES: gmpe_dict_1[imtx][res_type] = [] gmpe_dict_2[imtx][res_type] = [] self.types[gmpe][imtx].append(res_type) gmpe_dict_2[imtx]["Mean"] = [] self.residuals.append([gmpe, gmpe_dict_1]) self.modelled.append([gmpe, gmpe_dict_2]) self.residuals = OrderedDict(self.residuals) self.modelled = OrderedDict(self.modelled) self.database = None self.number_records = None self.contexts = None def get_residuals(self, database, nodal_plane_index=1, component="Geometric", normalise=True): """ Calculate the residuals for a set of ground motion records """ # Contexts is a list of dictionaries contexts = database.get_contexts(nodal_plane_index) self.database = SMRecordSelector(database) self.contexts = [] for context in contexts: #print context # Get the observed strong ground motions context = self.get_observations(context, component) # Get the expected ground motions context = self.get_expected_motions(context) context = self.calculate_residuals(context, normalise) for gmpe in self.residuals.keys(): for imtx in self.residuals[gmpe].keys(): for res_type in self.residuals[gmpe][imtx].keys(): self.residuals[gmpe][imtx][res_type].extend( context["Residual"][gmpe][imtx][res_type].tolist()) self.modelled[gmpe][imtx][res_type].extend( context["Expected"][gmpe][imtx][res_type].tolist()) self.modelled[gmpe][imtx]["Mean"].extend( context["Expected"][gmpe][imtx]["Mean"].tolist()) self.contexts.append(context) for gmpe in self.residuals.keys(): for imtx in self.residuals[gmpe].keys(): for res_type in self.residuals[gmpe][imtx].keys(): self.residuals[gmpe][imtx][res_type] = np.array( self.residuals[gmpe][imtx][res_type]) self.modelled[gmpe][imtx][res_type] = np.array( self.modelled[gmpe][imtx][res_type]) self.modelled[gmpe][imtx]["Mean"] = np.array( self.modelled[gmpe][imtx]["Mean"]) def get_observations(self, context, component="Geometric"): """ Get the obsered ground motions from the database """ select_records = self.database.select_from_event_id(context["EventID"]) observations = OrderedDict([(imtx, []) for imtx in self.imts]) selection_string = "IMS/H/Spectra/Response/Acceleration/" for record in select_records: fle = h5py.File(record.datafile, "r") for imtx in self.imts: if imtx in SCALAR_IMTS: if imtx == "PGA": observations[imtx].append( get_scalar(fle, imtx, component) / 981.0) else: observations[imtx].append( get_scalar(fle, imtx, component)) elif "SA(" in imtx: target_period = imt.from_string(imtx).period spectrum = fle[selection_string + component + "/damping_05"].value periods = fle["IMS/H/Spectra/Response/Periods"].value observations[imtx].append(get_interpolated_period( target_period, periods, spectrum) / 981.0) else: raise "IMT %s is unsupported!" % imtx fle.close() for imtx in self.imts: observations[imtx] = np.array(observations[imtx]) context["Observations"] = observations return context def get_expected_motions(self, context): """ Calculate the expected ground motions from the context """ # TODO Rake hack will be removed!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! if not context["Rupture"].rake: context["Rupture"].rake = 0.0 expected = OrderedDict([(gmpe, {}) for gmpe in self.gmpe_list]) for gmpe in self.gmpe_list: expected[gmpe] = OrderedDict([(imtx, {}) for imtx in self.imts]) for imtx in self.imts: for res_type in self.types[gmpe][imtx]: gsim = GSIM_LIST[gmpe]() mean, stddev = gsim.get_mean_and_stddevs( context["Sites"], context["Rupture"], context["Distances"], imt.from_string(imtx), [res_type]) expected[gmpe][imtx]["Mean"] = mean expected[gmpe][imtx][res_type] = stddev[0] context["Expected"] = expected return context def calculate_residuals(self, context, normalise=True): """ Calculate the residual terms """ # Calculate residual residual = {} for gmpe in self.gmpe_list: residual[gmpe] = {} for imtx in self.imts: residual[gmpe][imtx] = {} obs = np.log(context["Observations"][imtx]) mean = context["Expected"][gmpe][imtx]["Mean"] total_stddev = context["Expected"][gmpe][imtx]["Total"] residual[gmpe][imtx]["Total"] = (obs - mean) / total_stddev if "Inter event" in self.residuals[gmpe][imtx].keys(): inter, intra = self._get_random_effects_residuals( obs, mean, context["Expected"][gmpe][imtx]["Inter event"], context["Expected"][gmpe][imtx]["Intra event"], normalise) residual[gmpe][imtx]["Inter event"] = inter residual[gmpe][imtx]["Intra event"] = intra context["Residual"] = residual return context def _get_random_effects_residuals(self, obs, mean, inter, intra, normalise=True): """ Calculates the random effects residuals using the inter-event residual formula described in Abrahamson & Youngs (1992) Eq. 10 """ nvals = float(len(mean)) inter_res = ((inter ** 2.) * sum(obs - mean)) /\ (nvals * (inter ** 2.) + (intra ** 2.)) intra_res = obs - (mean + inter_res) if normalise: return inter_res / inter, intra_res / intra else: return inter_res, intra_res def get_residual_statistics(self): """ Retreives the mean and standard deviation values of the residuals """ statistics = OrderedDict([(gmpe, {}) for gmpe in self.gmpe_list]) for gmpe in self.gmpe_list: for imtx in self.imts: statistics[gmpe][imtx] = {} for res_type in self.types[gmpe][imtx]: data = { "Mean": np.mean( self.residuals[gmpe][imtx][res_type]), "Std Dev": np.std( self.residuals[gmpe][imtx][res_type])} statistics[gmpe][imtx][res_type] = data return statistics