def run_bootstrap(self, bootstrap_idx): betas, betas_resc = [], [] # Select the appropriate bootstrap from each task and stash the data into X and Y for k in range(self._n_tasks): X = self._task_design[k].get_bootstrap(self._task_bootstraps[k][bootstrap_idx]) Y = self._task_response[k].get_bootstrap(self._task_bootstraps[k][bootstrap_idx]) # Make sure that the priors align to the expression matrix priors_data = self._task_priors[k].reindex(labels=self._targets, axis=0). \ reindex(labels=self._regulators, axis=1). \ fillna(value=0) if self.clr_only: # Create a mock prior with no information if clr_only is set priors_data = pd.DataFrame(0, index=priors_data.index, columns=priors_data.columns) MPControl.sync_processes(pref="bbsr_pre") Debug.vprint('Calculating MI, Background MI, and CLR Matrix', level=0) clr_matrix, _ = self.mi_driver().run(Y, X, return_mi=False) Debug.vprint('Calculating task {k} betas using BBSR'.format(k=k), level=0) t_beta, t_br = BBSR(X, Y, clr_matrix, priors_data, prior_weight=self.prior_weight, no_prior_weight=self.no_prior_weight, nS=self.bsr_feature_num).run() betas.append(t_beta) betas_resc.append(t_br) return betas, betas_resc
def context_likelihood_mi(x, y, bins=DEFAULT_NUM_BINS, logtype=DEFAULT_LOG_TYPE, return_mi=True): """ Wrapper to calculate the Context Likelihood of Relatedness and Mutual Information for two data sets that have common condition rows. The y argument will be used to calculate background MI for the x & y MI. As an implementation detail, y will be cast to a dense array if it is sparse. X can be sparse with no internal copy. This function handles unpacking and packing the InferelatorData. :param x: An N x G InferelatorData object :type x: InferelatorData [N x G] :param y: An N x K InferelatorData object :type y: InferelatorData [N x K] :param logtype: The logarithm function to use when calculating information. Defaults to natural log (np.log) :type logtype: np.log func :param bins: Number of bins for discretizing continuous variables :type bins: int :param return_mi: Boolean for returning a MI object. Defaults to True :type return_mi: bool :return clr, mi: CLR and MI InferelatorData objects. Returns (CLR, None) if return_mi is False. :rtype InferelatorData, InferelatorData: """ assert check.argument_integer(bins, allow_none=True) assert min(x.shape) > 0 assert min(y.shape) > 0 assert check.indexes_align((x.sample_names, y.sample_names)) # Create dense output matrix and copy the inputs mi_r = x.gene_names mi_c = y.gene_names # Build a [G x K] mutual information array mi = mutual_information(x.expression_data, y.expression_data, bins, logtype=logtype) array_set_diag(mi, 0., mi_r, mi_c) # Build a [K x K] mutual information array mi_bg = mutual_information(y.expression_data, y.expression_data, bins, logtype=logtype) array_set_diag(mi_bg, 0., mi_c, mi_c) # Calculate CLR clr = calc_mixed_clr(mi, mi_bg) MPControl.sync_processes(pref=SYNC_CLR_KEY) mi = pd.DataFrame(mi, index=mi_r, columns=mi_c) clr = pd.DataFrame(clr, index=mi_r, columns=mi_c) return clr, mi if return_mi else None
def run_bootstrap(self, bootstrap): X = self.design.get_bootstrap(bootstrap) Y = self.response.get_bootstrap(bootstrap) utils.Debug.vprint('Calculating betas using MEN', level=0) MPControl.sync_processes("pre-bootstrap") return ElasticNet(X, Y, self.random_seed, parameters=self.elastic_net_parameters).run()
def run_bootstrap(self, bootstrap_idx): betas, betas_resc = [], [] # Select the appropriate bootstrap from each task and stash the data into X and Y for k in range(self._n_tasks): X = self._task_design[k].iloc[:, self._task_bootstraps[k][bootstrap_idx]].loc[self._regulators, :] Y = self._task_response[k].iloc[:, self._task_bootstraps[k][bootstrap_idx]].loc[self._targets, :] MPControl.sync_processes(pref="en_pre") utils.Debug.vprint('Calculating task {k} betas using MEN'.format(k=k), level=0) t_beta, t_br = ElasticNet(X, Y, random_seed=self.random_seed).run() betas.append(t_beta) betas_resc.append(t_br) return betas, betas_resc
def run(self): """ Execute regression separately on each response variable in the data :return: pd.DataFrame [G x K], pd.DataFrame [G x K] Returns the regression betas and beta error reductions for all threads if this is the master thread (rank 0) Returns None, None if it's a subordinate thread """ run_data = self.regress() if MPControl.is_master: pileup_data = self.pileup_data(run_data) else: pileup_data = None, None MPControl.sync_processes("post_pileup") return pileup_data
def run_regression(self): betas = [] rescaled_betas = [] MPControl.sync_processes("pre_regression") for idx, bootstrap in enumerate(self.get_bootstraps()): Debug.vprint('Bootstrap {} of {}'.format((idx + 1), self.num_bootstraps), level=0) np.random.seed(self.random_seed + idx) current_betas, current_rescaled_betas = self.run_bootstrap( bootstrap) if self.is_master(): betas.append(current_betas) rescaled_betas.append(current_rescaled_betas) MPControl.sync_processes("post_bootstrap") return betas, rescaled_betas
def run_bootstrap(self, bootstrap_idx): x, y = [], [] # Select the appropriate bootstrap from each task and stash the data into X and Y for k in range(self.n_tasks): x.append(self.task_design[k]. iloc[:, self.task_bootstraps[k][bootstrap_idx]].transpose()) y.append(self.task_response[k]. iloc[:, self.task_bootstraps[k][bootstrap_idx]].transpose()) MPControl.sync_processes(pref="amusr_pre") regress = AMuSR_regression(x, y, tfs=self.regulators, genes=self.targets, priors=self.priors_data, prior_weight=self.prior_weight) return regress.run()
def run_bootstrap(self, bootstrap_idx): betas, betas_resc = [], [] # Select the appropriate bootstrap from each task and stash the data into X and Y for k in range(self._n_tasks): X = self._task_design[ k].iloc[:, self._task_bootstraps[k][bootstrap_idx]].loc[ self._regulators, :] Y = self._task_response[ k].iloc[:, self._task_bootstraps[k][bootstrap_idx]].loc[ self._targets, :] # Make sure that the priors align to the expression matrix priors_data = self._task_priors[k].reindex(labels=self._targets, axis=0).fillna(value=0) priors_data = priors_data.reindex(labels=self._regulators, axis=1).fillna(value=0) MPControl.sync_processes(pref="bbsr_pre") utils.Debug.vprint('Calculating MI, Background MI, and CLR Matrix', level=0) clr_matrix, mi_matrix = self.mi_driver( sync_in_tmp_path=self.mi_sync_path).run(X, Y) mi_matrix = None utils.Debug.vprint( 'Calculating task {k} betas using BBSR'.format(k=k), level=0) t_beta, t_br = BBSR(X, Y, clr_matrix, priors_data, prior_weight=self.prior_weight, no_prior_weight=self.no_prior_weight, nS=self.bsr_feature_num).run() betas.append(t_beta) betas_resc.append(t_br) return betas, betas_resc
def run(self, x_df, y_df, bins=None, logtype=DEFAULT_LOG_TYPE): """ Wrapper to calculate the CLR and MI for two data sets that have common condition columns :param x_df: pd.DataFrame :param y_df: pd.DataFrame :param logtype: np.log func :param bins: int Number of bins for discretizing continuous variables :return clr, mi: pd.DataFrame, pd.DataFrame CLR and MI DataFrames """ assert check.argument_integer(bins, allow_none=True) assert check.indexes_align((x_df.columns, y_df.columns)) assert x_df.shape[0] > 0 assert x_df.shape[1] > 0 assert y_df.shape[0] > 0 assert y_df.shape[1] > 0 if bins is not None: self.bins = bins mi = mutual_information(y_df, x_df, self.bins, temp_dir=self.temp_dir, logtype=logtype) mi_bg = mutual_information(x_df, x_df, self.bins, temp_dir=self.temp_dir, logtype=logtype) clr = calc_mixed_clr(utils.df_set_diag(mi, 0), utils.df_set_diag(mi_bg, 0)) MPControl.sync_processes(pref=SYNC_CLR_KEY) return clr, mi
def run_bootstrap(self, bootstrap): X = self.design.iloc[:, bootstrap] Y = self.response.iloc[:, bootstrap] utils.Debug.vprint('Calculating betas using MEN', level=0) MPControl.sync_processes("pre-bootstrap") return ElasticNet(X, Y).run()
def test_sync(self): with self.assertRaises(RuntimeError): MPControl.sync_processes()
def test_dask_cluster_sync(self): self.assertTrue(MPControl.sync_processes())
def test_dask_local_sync(self): self.assertTrue(MPControl.sync_processes())
def test_mp_sync(self): self.assertTrue(MPControl.sync_processes())
def test_kvs_sync(self): self.assertEqual(MPControl.sync_processes(), None)