class BestModel(luigi.Task): __version__ = '0.3' features = luigi.DictParameter() guide_start = luigi.IntParameter() guide_length = luigi.IntParameter() pam_start = luigi.IntParameter() pam_length = luigi.IntParameter() activity_column = luigi.Parameter() kmer_column = luigi.Parameter() requires = task.Requires() example = True if example: cv_lasso = task.Requirement( CrossValidate, model_str='lasso', param_grid={'alpha': np.logspace(-3, 0, 100).tolist()}) else: cv_gb = task.Requirement(CrossValidate, model_str='GB', param_grid={ 'max_depth': [int(x) for x in np.linspace(2, 40, 30)], 'max_features': np.linspace(0.01, 0.3, 50).tolist(), 'min_samples_split': np.linspace(0.01, 0.4, 50).tolist(), 'subsample': np.linspace(0.6, 1, 50).tolist(), 'alpha': np.linspace(0.5, 0.99, 50).tolist() }) # cv_nn = task.Requirement(CrossValidate, model_str = 'NN', # param_grid = {'alpha':np.logspace(-4, -0.01, 100).tolist(), # 'learning_rate_init': np.linspace(0.001, 0.3, 50).tolist()}) # cv_gb = task.Requirement(CrossValidate, model_str = 'GB', # param_grid = {'alpha': [0.5]}) output = task.SaltedOutput(base_dir='data/models', ext='.pickle', format=luigi.format.Nop) def run(self): reqs = self.requires() best_fit = None for model, cv_x in reqs.items(): with cv_x.output().open('rb') as f: cv_model = pickle.load(f) score = cv_model.best_score_ curr_estimator = cv_model.best_estimator_ if best_fit is None: best_estimator = curr_estimator best_fit = score elif best_fit < score: best_estimator = curr_estimator best_fit = score with self.output().open('wb') as f: pickle.dump(best_estimator, f)
class CrossValidate(luigi.Task): __version__ = '0.5' model_str = luigi.Parameter() folds = luigi.IntParameter(default=10) param_grid = luigi.DictParameter() requires = task.Requires() scaler = task.Requirement(Standardize, activity_column='percentile', kmer_column='X30mer') featurized = task.Requirement(FeaturizeTrain) output = task.SaltedOutput(base_dir='data/cv', ext='.pickle', format=luigi.format.Nop) def run(self): reqs = self.requires() featurized = reqs['featurized'] with featurized.output().open('r') as f: featurized_df = pd.read_csv(f) with reqs['scaler'].output().open('rb') as f: scaler = pickle.load(f) y = featurized_df['activity'] X = featurized_df[featurized_df.columns.difference( ['activity', 'kmer'])] X_train = scaler.transform(X) #X_train = X if self.model_str == 'GB': model = ensemble.GradientBoostingRegressor() elif self.model_str == 'RF': model = ensemble.RandomForestRegressor() elif self.model_str == 'lasso': model = linear_model.Lasso() elif self.model_str == 'EN': model = linear_model.ElasticNet() elif self.model_str == 'NN': model = neural_network.MLPRegressor() grid_search = model_selection.RandomizedSearchCV( model, dict(self.param_grid), cv=self.folds, scoring='neg_mean_squared_error', n_iter=20, n_jobs=1) grid_search.fit(X_train, y) # Use path because we have to write binary (stack: localTarget pickle) with self.output().open('wb') as f: pickle.dump(grid_search, f)
class AnalyzePredictions(luigi.Task): __version__ = '0.1' guide_start = luigi.IntParameter() guide_length = luigi.IntParameter() pam_start = luigi.IntParameter() pam_length = luigi.IntParameter() requires = task.Requires() azimuth_predictions = task.Requirement() rs2_predictions = task.Requirement(PredictModel, features = {'Pos. Ind. 1mer': True, 'Pos. Ind. 2mer': True, 'Pos. Ind. 3mer': False, 'Pos. Dep. 1mer': True, 'Pos. Dep. 2mer': True, 'Pos. Dep. 3mer': False, 'GC content': True, 'Tm': True}) dimer_predictions = task.Requirement(PredictModel, features = {'Pos. Ind. 1mer': False, 'Pos. Ind. 2mer': False, 'Pos. Ind. 3mer': False, 'Pos. Dep. 1mer': False, 'Pos. Dep. 2mer': True, 'Pos. Dep. 3mer': False, 'GC content': True, 'Tm': False}) output = task.SaltedOutput(base_dir='data/predictions', ext='.csv') def run(self): reqs = self.requires() with reqs['model'].output().open('rb') as f: model = pickle.load(f) with reqs['test_mat'].output().open('r') as f: test_mat = pd.read_csv(f) with reqs['scaler'].output().open('rb') as f: scaler = pickle.load(f) y = test_mat['activity'] X = test_mat[test_mat.columns.difference(['activity', 'kmer'])] X_train = scaler.transform(X) predictions = model.predict(X_train) prediction_mat = pd.DataFrame({'kmer': test_mat['kmer'], 'true': y, 'predicted': predictions}) with self.output().open('w') as f: prediction_mat.to_csv(f, index=False)
class PredictModel(luigi.Task): __version__ = '0.2' features = luigi.DictParameter() guide_start = luigi.IntParameter() guide_length = luigi.IntParameter() pam_start = luigi.IntParameter() pam_length = luigi.IntParameter() true_val = luigi.BoolParameter(default=True) requires = task.Requires() model = task.Requirement(BestModel, activity_column='percentile', kmer_column='X30mer') test_mat = task.Requirement(FeaturizeAchillesTest, activity_column='sgRNA.measured.value', kmer_column='X30mer') scaler = task.Requirement(Standardize, activity_column='percentile', kmer_column='X30mer') output = task.SaltedOutput(base_dir='data/predictions', ext='.csv') def run(self): reqs = self.requires() with reqs['model'].output().open('rb') as f: model = pickle.load(f) with reqs['test_mat'].output().open('r') as f: test_mat = pd.read_csv(f) with reqs['scaler'].output().open('rb') as f: scaler = pickle.load(f) y = test_mat['activity'] X = test_mat[test_mat.columns.difference(['activity', 'kmer'])] X_train = scaler.transform(X) #X_train = X predictions = model.predict(X_train) prediction_mat = pd.DataFrame({ 'kmer': test_mat['kmer'], 'true': y, 'predicted': predictions }) with self.output().open('w') as f: prediction_mat.to_csv(f, index=False)
class ModelCoefficients(luigi.Task): __version__ = '0.3' features = luigi.DictParameter() guide_start = luigi.IntParameter() guide_length = luigi.IntParameter() pam_start = luigi.IntParameter() pam_length = luigi.IntParameter() requires = task.Requires() model = task.Requirement(BestModel, activity_column='percentile', kmer_column='X30mer') scaler = task.Requirement(Standardize, activity_column='percentile', kmer_column='X30mer') train_mat = task.Requirement(FeaturizeTrain, activity_column='percentile', kmer_column='X30mer') output = task.SaltedOutput(base_dir='data/models', ext='.csv') def run(self): reqs = self.requires() with reqs['model'].output().open('rb') as f: model = pickle.load(f) with reqs['train_mat'].output().open('r') as f: train_mat = pd.read_csv(f) with reqs['scaler'].output().open('r') as f: scaler = pickle.load(f) X = train_mat[train_mat.columns.difference(['activity', 'kmer'])] if model.__class__ == ensemble.GradientBoostingRegressor: importances = model.feature_importances_ elif model.__class__ == linear_model.Lasso: importances = score_coefs(scaler.transform(X), train_mat['activity'], model.coef_, model.intercept_) feature_importances = pd.DataFrame({ 'feature': X.keys(), 'importance': importances }) with self.output().open('w') as f: feature_importances.to_csv(f, index=False)
class FilteredAchillesData(luigi.Task): __version__ = '0.1' requires = task.Requires() achilles_file = task.Requirement(AchillesTestData) oof_gv2_file = task.Requirement(OofGv2) output = task.SaltedOutput(base_dir='./data/filtered', ext='.csv') def run(self): with self.achilles_file.output().open('r') as f: achilles_data = pd.read_csv(f) with self.oof_gv2_file.output().open('r') as f: oof_gv2_data = pd.read_csv(f) achilles_oof_data = (pd.merge( achilles_data, oof_gv2_data.drop_duplicates(), how='inner', on='X30mer').drop(['X', 'Unnamed: 0'], axis=1).drop_duplicates()) with self.output().open('w') as f: achilles_oof_data.to_csv(f)
class FilteredRS2Data(luigi.Task): __version__ = '0.1' requires = task.Requires() rs2_file = task.Requirement(RS2CombData) oof_fc_file = task.Requirement(OofFc) oof_res_file = task.Requirement(OofRes) output = task.SaltedOutput(base_dir='./data/filtered', ext='.csv') def run(self): with self.rs2_file.output().open('r') as f: rs2_data = pd.read_csv(f) with self.oof_fc_file.output().open('r') as f: oof_fc_data = pd.read_csv(f) with self.oof_res_file.output().open('r') as f: oof_res_data = pd.read_csv(f) rs2_oof_data = (pd.merge( oof_res_data[['30mer', 'OOF mutation rate']], oof_fc_data[['30mer', 'OOF mutation rate']], how='outer').merge(rs2_data, how='inner', on='30mer').drop_duplicates()) with self.output().open('w') as f: rs2_oof_data.to_csv(f)
class Fasta(luigi.Task): __version__ = '0.1' seq_col = luigi.Parameter() requires = task.Requires() seq_data = task.Requirement(RS2CombData) output = task.SaltedOutput(base_dir='./data/raw', ext='.FASTA') def run(self): reqs = self.requires() with reqs['seq_data'].output().open('r') as f: seq_data = pd.read_csv(f) seqs = seq_data[self.seq_col] with self.output().open('w') as f: for seq in seqs: f.write('>' + seq + '\n') f.write(seq + '\n')
class FeaturizeTrain(BaseFeaturize): # ["Cd28", "Cd3e", "CD45", "Cd5", "Cd43", "H2-K", "Thy1", # "CD13_TF-1", "CD13_NB4", "CD33_NB4", "CD33_MOLM-13", # "CD15_MOLM-13", "CCDC101_AZD", "CUL3_PLX", # "HPRT1_6TG", "MED12_AZD", "MED12_PLX", "NF1_PLX", "NF2_PLX", # "TADA1_AZD", "TADA2B_AZD"] filtered = task.Requirement(FilteredRS3Data, assays = ["CD45", "Cd28", "Cd5", "Cd43", "H2-K", "Thy1", "CD13_TF-1", "CD33_MOLM-13", "CD15_MOLM-13", "CCDC101_AZD", "HPRT1_6TG", "MED12_AZD", "NF1_PLX", "NF2_PLX", "TADA1_AZD", "TADA2B_AZD"], assays_end = ["CD45", "Cd5", "Cd43", "H2-K", "Thy1", "CD33_MOLM-13", "HPRT1_6TG", "MED12_AZD", "NF1_PLX", "NF2_PLX", "TADA2B_AZD"] , assays_start = ["CD13_TF-1","CD15_MOLM-13", "CD45"], perc_pep_end = 80, perc_pep_start = 20)
class FilteredRS3Data(luigi.Task): __version__ = '0.3' requires = task.Requires() rs3_file = task.Requirement(RS3Train) assays = luigi.ListParameter() assays_end = luigi.ListParameter() assays_start = luigi.ListParameter() perc_pep_end = luigi.IntParameter() perc_pep_start = luigi.IntParameter() output = task.SaltedOutput(base_dir='./data/filtered', ext='.csv') def run(self): with self.rs3_file.output().open('r') as f: rs3_data = pd.read_csv(f) filtered_rs3_data = rs3_data[(rs3_data.Assay_ID.isin(self.assays)) & ( ((rs3_data.Target_Cut < self.perc_pep_end) & (rs3_data.Assay_ID.isin(self.assays_end))) | ~rs3_data.Assay_ID.isin(self.assays_end)) & ( ((rs3_data.Target_Cut > self.perc_pep_start) & rs3_data.Assay_ID.isin(self.assays_start)) | ~rs3_data.Assay_ID.isin(self.assays_start))] with self.output().open('w') as f: filtered_rs3_data.to_csv(f)
class Standardize(luigi.Task): __version__ = '0.1' activity_column = luigi.Parameter() kmer_column = luigi.Parameter() features = luigi.DictParameter() guide_start = luigi.IntParameter() guide_length = luigi.IntParameter() pam_start = luigi.IntParameter() pam_length = luigi.IntParameter() requires = task.Requires() featurized = task.Requirement(FeaturizeTrain) output = task.SaltedOutput(base_dir='data/featurized', ext='.csv', format=luigi.format.Nop) def run(self): reqs = self.requires() with reqs['featurized'].output().open('r') as f: test_mat = pd.read_csv(f) X = test_mat[test_mat.columns.difference(['activity', 'kmer'])] scaler = preprocessing.StandardScaler().fit(X) with self.output().open('wb') as f: pickle.dump(scaler, f)
class FeaturizeAchillesTest(BaseFeaturize): filtered = task.Requirement(Gv2Test)
class FeaturizeDoenchTest(BaseFeaturize): filtered = task.Requirement(DoenchTestData)