def calc_simple_ratios(self, label): """ Calculate simplified (original Enrich) ratios scores. This method does not produce standard errors. """ if self.store_check("/main/{}/scores".format(label)): return log_message( logging_callback=logging.info, msg="Calculating simple ratios ({})".format(label), extra={"oname": self.name}, ) c_last = "c_{}".format(self.store_timepoints()[-1]) df = self.store_select( key="/main/{}/counts".format(label), columns=["c_0", "{}".format(c_last)] ) # perform operations on the numpy values of the # dataframe for easier broadcasting num = df[c_last].values.astype("float") / df[c_last].sum(axis="index") denom = df["c_0"].values.astype("float") / df["c_0"].sum(axis="index") ratios = num / denom # make it a data frame again ratios = pd.DataFrame(data=ratios, index=df.index, columns=["ratio"]) ratios["score"] = np.log2(ratios["ratio"]) ratios["SE"] = np.nan ratios = ratios[["score", "SE", "ratio"]] # re-order columns self.store_put( key="/main/{}/scores".format(label), value=ratios, data_columns=ratios.columns, )
def calc_ratios(self, label): """ Calculate frequency ratios and standard errors between the last timepoint and the input. Ratios can be calculated using one of three methods: - wt - complete - full """ if self.store_check("/main/{}/scores".format(label)): return log_message( logging_callback=logging.info, msg="Calculating ratios ({})".format(label), extra={"oname": self.name}, ) c_last = "c_{}".format(self.store_timepoints()[-1]) df = self.store_select(key="/main/{}/counts".format(label), columns=["c_0", "{}".format(c_last)]) if self.logr_method == "wt": if VARIANTS in self.store_labels(): wt_label = VARIANTS elif IDENTIFIERS in self.store_labels(): wt_label = IDENTIFIERS else: raise ValueError("Failed to use wild type log " "ratio method, suitable data " "table not present [{}]".format(self.name)) shared_counts = self.store_select( key="/main/{}/counts".format(wt_label), columns=["c_0", "{}".format(c_last)], where="index='{}'".format(WILD_TYPE_VARIANT), ) # wild type not found if len(shared_counts) == 0: raise ValueError("Failed to use wild type log " "ratio method, wild type " "sequence not present [{}]".format(self.name)) shared_counts = shared_counts.values + 0.5 elif self.logr_method == "complete": shared_counts = (self.store_select( key="/main/{}/counts".format(label), columns=["c_0", "{}".format(c_last)], ).sum(axis="index").values + 0.5) elif self.logr_method == "full": shared_counts = (self.store_select( key="/main/{}/counts_unfiltered".format(label), columns=["c_0", "{}".format(c_last)], ).sum(axis="index", skipna=True).values + 0.5) else: raise ValueError('Invalid log ratio method "{}" ' "[{}]".format(self.logr_method, self.name)) ratios = np.log(df[["c_0", c_last]].values + 0.5) - np.log(shared_counts) ratios = ratios[:, 1] - ratios[:, 0] # selected - input ratios = pd.DataFrame(ratios, index=df.index, columns=["logratio"]) shared_variance = np.sum(1.0 / shared_counts) summed = np.sum(1.0 / (df[["c_0", c_last]].values + 0.5), axis=1) ratios["variance"] = summed + shared_variance ratios["score"] = ratios["logratio"] ratios["SE"] = np.sqrt(ratios["variance"]) # re-order columns ratios = ratios[["score", "SE", "logratio", "variance"]] self.store_put( key="/main/{}/scores".format(label), value=ratios, data_columns=ratios.columns, )
def calc_log_ratios(self, label): """ Calculate the log ratios that will be fit using the linear models. """ if self.store_check("/main/{}/log_ratios".format(label)): return log_message( logging_callback=logging.info, msg="Calculating log ratios ({})".format(label), extra={"oname": self.name}, ) ratios = self.store_select("/main/{}/counts".format(label)) index = ratios.index c_n = ["c_{}".format(x) for x in self.store_timepoints()] ratios = np.log(ratios + 0.5) # perform operations on the numpy values of the data # frame for easier broadcasting ratios = ratios[c_n].values if self.logr_method == "wt": if VARIANTS in self.store_labels(): wt_label = VARIANTS elif IDENTIFIERS in self.store_labels(): wt_label = IDENTIFIERS else: raise ValueError( "Failed to use wild type log ratio method, " "suitable data table not " "present [{}]".format(self.name) ) wt_counts = self.store_select( key="/main/{}/counts".format(wt_label), columns=c_n, where="index='{}'".format(WILD_TYPE_VARIANT), ) if len(wt_counts) == 0: # wild type not found raise ValueError( "Failed to use wild type log ratio method, " "wild type sequence not " "present [{}]".format(self.name) ) ratios = ratios - np.log(wt_counts.values + 0.5) elif self.logr_method == "complete": ratios = ratios - np.log( self.store_select(key="/main/{}/counts".format(label), columns=c_n) .sum(axis="index") .values + 0.5 ) elif self.logr_method == "full": ratios = ratios - np.log( self.store_select( key="/main/{}/counts_unfiltered".format(label), columns=c_n ) .sum(axis="index", skipna=True) .values + 0.5 ) else: raise ValueError( 'Invalid log ratio method "{}" [{}]'.format(self.logr_method, self.name) ) # make it a data frame again columns = ["L_{}".format(x) for x in self.store_timepoints()] ratios = pd.DataFrame(data=ratios, index=index, columns=columns) self.store_put( key="/main/{}/log_ratios".format(label), value=ratios, data_columns=ratios.columns, )
def calc_regression(self, label): """ Calculate least squares regression for *label*. If *weighted* is ``True``, calculates weighted least squares; else ordinary least squares. Regression results are stored in ``'/main/label/scores'`` """ req_tables = ["/main/{}/log_ratios".format(label)] if self.weighted: req_tables.append("/main/{}/weights".format(label)) for req_table in req_tables: if not self.store_check(req_table): raise ValueError( "Required table {} does not " "exist [{}].".format(req_table, self.name) ) if self.store_check("/main/{}/scores".format(label)): return elif "/main/{}/scores".format(label) in list(self.store_keys()): # need to remove the current keys because we are using append self.store_remove("/main/{}/scores".format(label)) method = "WLS" if self.weighted else "OLS" log_message( logging_callback=logging.info, msg="Calculating {} regression coefficients " "({})".format(method, label), extra={"oname": self.name}, ) longest = ( self.store_select( key="/main/{}/log_ratios".format(label), columns=["index"] ) .index.map(len) .max() ) chunk = 1 # -------------------- REG COMPUTATION --------------------------- # selection = ["/main/{}/log_ratios".format(label)] if self.weighted: selection.append("/main/{}/weights".format(label)) selection = self.store_select_as_multiple(keys=selection, chunk=True) for data in selection: log_message( logging_callback=logging.info, msg="Calculating {} for chunk {} ({} rows)".format( method, chunk, len(data.index) ), extra={"oname": self.name}, ) result = data.apply( self.row_apply_function, axis="columns", args=[self.store_timepoints(), self.weighted], ) # append is required because it takes the # "min_itemsize" argument, and put doesn't self.store_append( key="/main/{}/scores".format(label), value=result, min_itemsize={"index": longest}, ) chunk += 1 # ----------------------- POST ------------------------------------ # # need to read from the file, calculate percentiles, and rewrite it log_message( logging_callback=logging.info, msg="Calculating slope standard error " "percentiles ({})".format(label), extra={"oname": self.name}, ) data = self.store_get("/main/{}/scores".format(label)) data["score"] = data["slope"] data["SE"] = data["SE_slope"] data["SE_pctile"] = [ stats.percentileofscore(data["SE"], x, "weak") for x in data["SE"] ] # reorder columns reorder_selector = [ "score", "SE", "SE_pctile", "slope", "intercept", "SE_slope", "t", "pvalue_raw", ] data = data[reorder_selector] self.store_put( key="/main/{}/scores".format(label), value=data, data_columns=data.columns )
def calc_weights(self, label): """ Calculate the regression weights (1 / variance). """ if self.store_check("/main/{}/weights".format(label)): return log_message( logging_callback=logging.info, msg="Calculating regression weights ({})".format(label), extra={"oname": self.name}, ) variances = self.store_select("/main/{}/counts".format(label)) c_n = ["c_{}".format(x) for x in self.store_timepoints()] index = variances.index # perform operations on the numpy values of the # data frame for easier broadcasting # var_left = 1.0 / (variances[c_n].values + 0.5) # var_right = 1.0 / (variances[['c_0']].values + 0.5) # variances = var_left + var_right variances = 1.0 / (variances[c_n].values + 0.5) # -------------------------- WT NORM ----------------------------- # if self.logr_method == "wt": if VARIANTS in self.store_labels(): wt_label = VARIANTS elif IDENTIFIERS in self.store_labels(): wt_label = IDENTIFIERS else: raise ValueError( "Failed to use wild type log ratio method, " "suitable data table not present [{}]".format(self.name) ) wt_counts = self.store_select( key="/main/{}/counts".format(wt_label), columns=c_n, where="index='{}'".format(WILD_TYPE_VARIANT), ) # wild type not found if len(wt_counts) == 0: raise ValueError( "Failed to use wild type log ratio method, wild type " "sequence not present [{}]".format(self.name) ) variances = variances + 1.0 / (wt_counts.values + 0.5) # ---------------------- COMPLETE NORM ----------------------------- # elif self.logr_method == "complete": variances = variances + 1.0 / ( self.store_select(key="/main/{}/counts".format(label), columns=c_n) .sum(axis="index") .values + 0.5 ) # ------------------------- FULL NORM ----------------------------- # elif self.logr_method == "full": variances = variances + 1.0 / ( self.store_select( key="/main/{}/counts_unfiltered".format(label), columns=c_n ) .sum(axis="index", skipna=True) .values + 0.5 ) # ---------------------------- WUT? ------------------------------- # else: raise ValueError( 'Invalid log ratio method "{}" [{}]'.format(self.logr_method, self.name) ) # weights are reciprocal of variances variances = 1.0 / variances # make it a data frame again variances = pd.DataFrame( data=variances, index=index, columns=["W_{}".format(x) for x in self.store_timepoints()], ) self.store_put( key="/main/{}/weights".format(label), value=variances, data_columns=variances.columns, )