def viz_errs_outliers_venn(X_test, preds, Y_test, num_feats_reduced=5): '''Compare outliers to errors in venn-diagram ''' feat_names = data.get_feature_names(X_test) X_feat = X_test[feat_names] if num_feats_reduced is not None: pca = decomposition.PCA(n_components=num_feats_reduced) X_reduced = pca.fit_transform(X_feat) else: X_reduced = X_feat R, C = 2, 2 titles = [ 'isolation forest', 'local outlier factor', 'elliptic envelop', 'one-class svm' ] plt.figure(figsize=(6, 5), dpi=200) for i in range(4): plt.subplot(R, C, i + 1) plt.title(titles[i]) if i == 0: clf = IsolationForest(n_estimators=10, warm_start=True) elif i == 1: clf = LocalOutlierFactor(novelty=True) elif i == 2: clf = EllipticEnvelope() elif i == 3: clf = OneClassSVM() clf.fit(X_reduced) # fit 10 trees is_outlier = clf.predict(X_reduced) == -1 is_err = preds != Y_test idxs = np.arange(is_outlier.size) venn2([set(idxs[is_outlier]), set(idxs[is_err])], set_labels=['outliers', 'errors'])
def test_reg(df, model, feat_names=None, outcome_def='Y_max_log', out_name='results/regression/test.pkl', seed=42): np.random.seed(seed) if not feat_names: feat_names = data.get_feature_names(df) feat_names = [ x for x in feat_names if not x.startswith('sc_') and not x.startswith('nmf_') and not x in [ 'center_max', 'left_max', 'right_max', 'up_max', 'down_max', 'X_max_around_Y_peak', 'X_max_after_Y_peak' ] and not x.startswith('pc_') and not 'log' in x and not 'binary' in x # and not 'slope' in x ] X = df[feat_names] # X = (X - X.mean()) / X.std() # normalize the data test_preds = model.predict(X) results = {'preds': test_preds} if outcome_def in df.keys(): y = df[outcome_def].values results['r2'] = r2_score(y, test_preds) results['pearsonr'] = pearsonr(y, test_preds) results['kendalltau'] = kendalltau(y, test_preds) return results
def normalize_and_predict(m0, feat_names_selected, dset_name, normalize_by_train, exclude_easy_tracks=False, outcome_def='y_consec_thresh'): df_new = data.get_data(dset=dset_name, use_processed=True, use_processed_dicts=True, outcome_def=outcome_def, previous_meta_file=oj( DIR_PROCESSED, 'metadata_clath_aux+gak_a7d2.pkl')) if exclude_easy_tracks: df_new = df_new[ df_new['valid']] # exclude test cells, short/long tracks, hotspots # impute (only does anything for dynamin data) df_new = df_new.fillna(df_new.median()) X_new = df_new[data.get_feature_names(df_new)] if normalize_by_train: X_new = (X_new - X_mean_train) / X_std_train else: X_new = (X_new - X_new.mean()) / X_new.std() y_new = df_new[outcome_def].values preds_new = m0.predict(X_new[feat_names_selected]) preds_proba_new = m0.predict_proba(X_new[feat_names_selected])[:, 1] Y_maxes = df_new['Y_max'] return df_new, y_new, preds_new, preds_proba_new, Y_maxes
def add_binary_features(df, outcome_def): '''binarize features at the difference between the mean of each class ''' feat_names = data.get_feature_names(df) threshes = (df[df[outcome_def] == 1].mean() + df[df[outcome_def] == 0].mean()) / 2 for i, k in tqdm(enumerate(feat_names)): thresh = threshes.loc[k] df[k + '_binary'] = df[k] >= thresh return df
def normalize(df, outcome_def): X = df[data.get_feature_names(df)] X_mean = X.mean() X_std = X.std() ks = list(X.keys()) norms = {ks[i]: {'mu': X_mean[i], 'std': X_std[i]} for i in range(len(ks))} X = (X - X_mean) / X_std y = df[outcome_def].values return X, y, norms
def add_pcs(df): '''adds 10 pcs based on feature names ''' feat_names = data.get_feature_names(df) X = df[feat_names] X = (X - X.mean()) / X.std() pca = decomposition.PCA(whiten=True) pca.fit(X[df.valid]) X_reduced = pca.transform(X) for i in range(10): df['pc_' + str(i)] = X_reduced[:, i] return df
def input_fn_dataset(df): features = df[data.get_feature_names()] labels = df[data.get_target_name()] dataset = tf.data.Dataset.from_tensor_slices((dict(features), labels)) if param('shuffle_data'): dataset = dataset.shuffle(buffer_size=256) dataset = dataset.repeat(param('epochs')) dataset = dataset.batch(param('batch_size')) iterator = dataset.make_one_shot_iterator() batch_features, batch_labels = iterator.get_next() return batch_features, batch_labels
def input_fn_pandas(df): x = df[data.get_feature_names()] y = df[data.get_target_name()] return tf.estimator.inputs.pandas_input_fn( x, y, batch_size=param('batch_size'), num_epochs=param('epochs'), shuffle=param('shuffle_data'), queue_capacity=1000, num_threads=4, target_column=data.get_target_name() )
def get_feature_columns(): feature_names = data.get_feature_names() classifier_type = param('classifier_type') if classifier_type == 'linear': return [ tf.feature_column.categorical_column_with_vocabulary_list(name, data.get_feature_values(name)) for name in feature_names ] elif classifier_type == 'dnn': return [ tf.feature_column.indicator_column( tf.feature_column.categorical_column_with_vocabulary_list(name, data.get_feature_values(name)) ) for name in feature_names ] raise Exception('Unsupported classifier type: {}'.format(classifier_type))
def load_and_train(dset, outcome_def, out_dir, feat_names=None, use_processed=True): df = pd.read_pickle(f'../data/tracks/tracks_{dset}.pkl') if dset == 'clath_aux_dynamin': df = df[df.catIdx.isin([1, 2])] df = df[df.lifetime > 15] else: df = df[df['valid'] == 1] df = features.add_basic_features(df) df = log_transforms(df) df = add_sig_mean(df) df_train = df[df.cell_num.isin(config.DSETS[dset]['train'])] df_test = df[df.cell_num.isin(config.DSETS[dset]['test'])] df_train = df_train.dropna() #outcome_def = 'Z_sig_mean' #out_dir = 'results/regression/Sep15' os.makedirs(out_dir, exist_ok=True) if not feat_names: feat_names = data.get_feature_names(df_train) feat_names = [ x for x in feat_names if not x.startswith('sc_') and not x.startswith('nmf_') and not x in [ 'center_max', 'left_max', 'right_max', 'up_max', 'down_max', 'X_max_around_Y_peak', 'X_max_after_Y_peak' ] and not x.startswith('pc_') and not 'log' in x and not 'binary' in x # and not 'slope' in x ] for model_type in tqdm(['linear', 'gb', 'rf', 'svm', 'ridge']): out_name = f'{model_type}' #print(out_name) if use_processed and os.path.exists(f'{out_dir}/{out_name}.pkl'): continue train_reg(df_train, feat_names=feat_names, model_type=model_type, outcome_def=outcome_def, out_name=f'{out_dir}/{out_name}.pkl')
def add_trend_filtering(df): df_tf = deepcopy(df) for i in range(len(df)): df_tf['X'].iloc[i] = trend_filtering.trend_filtering( y=df['X'].iloc[i], vlambda=len(df['X'].iloc[i]) * 5, order=1) df_tf = add_features(df_tf) feat_names = data.get_feature_names(df_tf) feat_names = [ x for x in feat_names if not x.startswith('sc_') and not x.startswith('nmf_') and not x in [ 'center_max', 'left_max', 'right_max', 'up_max', 'down_max', 'X_max_around_Y_peak', 'X_max_after_Y_peak', 'X_max_diff_after_Y_peak', 'X_tf' ] and not x.startswith('pc_') # and not 'local' in x # and not 'X_peak' in x # and not 'slope' in x # and not x in ['fall_final', 'fall_slope', 'fall_imp', 'fall'] ] for feat in feat_names: df[feat + '_tf_smooth'] = df_tf[feat] return df
res[f'{feat}_{metric}'].append(test_preds[metric]) else: res[f'{feat}_{metric}'].append(test_preds[metric][0]) res_df = pd.DataFrame(res, index=datasets) res_df.to_pickle('results/regression/regression_results.pkl') if __name__ == '__main__': print("loading data...") all_data = load_all_datasets() print("training regression models...") df = all_data[orig_dset] df_train = df[df.cell_num.isin(config.DSETS[orig_dset]['train'])] df_train = df_train[df_train['valid'] == 1] feat_names = data.get_feature_names(df_train) feat_names = [x for x in feat_names if not x.startswith('sc_') and not x.startswith('nmf_') and not x in ['center_max', 'left_max', 'right_max', 'up_max', 'down_max', 'X_max_around_Y_peak', 'X_max_after_Y_peak', 'X_quantiles', 'X_d1', 'X_d2', 'X_d3', 'slope_end' ] and not x.startswith('pc_') and not 'log' in x and not 'binary' in x # and not 'slope' in x ] train_and_save_models(df_train, feat_names, outcome_def) print("testing regression models...") test_and_save_results()
from src import train # from src.viz import * import config if __name__ == '__main__': # some settings outcome_def = 'y_consec_thresh' out_dir = oj('/scratch/users/vision/chandan/abc', 'nov16') dset_key = 'clath_aux+gak_a7d2' dset = config.DSETS[dset_key] binarize = False # True # get data df = data.get_data(dset=dset_key) df = df[df['valid']] # exclude test cells, short/long tracks, hotspots feat_names = data.get_feature_names(df) feat_names = data.select_final_feats(feat_names, binarize=binarize) print('num feats', len(feat_names)) print(feat_names) # run os.makedirs(out_dir, exist_ok=True) feature_selection_nums = [ 2, 3, 4, 5, 6, 7, 8 ] #, 9, 15, len(feat_names)] #[3, 5, 7, 12, 16]: # number of feature to select [4, 9, 11, 23, 35, 39] for calibrated in [True, False]: for feature_selection_num in feature_selection_nums: for feature_selection in ['select_lasso', 'select_rf' ]: # select_lasso, select_rf, None if feature_selection is None and feature_selection_num > feature_selection_nums[ 0]: # don't do extra computation