def get_frac_correct(df_train, df_test, pipeline_str=None, num_groups=4, energy_key='MC_log_energy'): '''Calculates the fraction of correctly identified samples in each energy bin for each composition in comp_list. In addition, the statisitcal error for the fraction correctly identified is calculated.''' # Input validation if energy_key not in ['MC_log_energy', 'reco_log_energy']: raise ValueError( "Invalid energy_key ({}) entered. Must be either " "'MC_log_energy' or 'reco_log_energy'.".format(energy_key)) if pipeline_str is None: pipeline_str = 'BDT_comp_IC86.2012_{}-groups'.format(num_groups) # Fit pipeline and get mask for correctly identified events feature_list, feature_labels = comp.get_training_features() pipeline = comp.get_pipeline(pipeline_str) comp_target_str = 'comp_target_{}'.format(num_groups) pipeline.fit(df_train[feature_list], df_train[comp_target_str]) test_predictions = pipeline.predict(df_test[feature_list]) correctly_identified_mask = (test_predictions == df_test[comp_target_str]) data = {} for composition in comp_list + ['total']: comp_mask = df_test['comp_group_{}'.format(num_groups)] == composition # Get number of MC comp in each energy bin num_MC_energy, _ = np.histogram(df_test.loc[comp_mask, energy_key], bins=energybins.log_energy_bins) num_MC_energy_err = np.sqrt(num_MC_energy) # Get number of correctly identified comp in each energy bin combined_mask = comp_mask & correctly_identified_mask num_reco_energy, _ = np.histogram(df_test.loc[combined_mask, energy_key], bins=energybins.log_energy_bins) num_reco_energy_err = np.sqrt(num_reco_energy) # Calculate correctly identified fractions as a function of energy frac_correct, frac_correct_err = comp.ratio_error( num_reco_energy, num_reco_energy_err, num_MC_energy, num_MC_energy_err) data['frac_correct_{}'.format(composition)] = frac_correct data['frac_correct_err_{}'.format(composition)] = frac_correct_err return data
def get_BDT_scores(config): df_data = comp.load_dataframe(datatype='data', config=config) feature_list, feature_labels = comp.get_training_features() df_data.loc[:, feature_list].dropna(axis=0, how='any', inplace=True) # Load saved pipeline model_file = os.path.join(comp.paths.project_home, 'models/GBDT_IC86.2012.pkl') pipeline = joblib.load(model_file)['pipeline'] # Get BDT scores for each data event X_data = comp.dataframe_functions.dataframe_to_array(df_data, feature_list) classifier_scores = pipeline.decision_function(X_data) print('{} complete!'.format(config)) return classifier_scores
def save_anisotropy_dataframe(config, outfile): print('Loading data...') data_df = comp.load_dataframe(datatype='data', config=config, verbose=False) keep_columns = [ 'lap_zenith', 'lap_azimuth', 'start_time_mjd', 'pred_comp', 'lap_log_energy' ] comp_list = ['light', 'heavy'] pipeline_str = 'GBDT' pipeline = comp.get_pipeline(pipeline_str) feature_list, feature_labels = comp.get_training_features() data_df.loc[:, feature_list].dropna(axis=0, how='any', inplace=True) print('Loading simulation...') if 'IC86' in config: sim_config = 'IC86.2012' else: sim_config = 'IC79' sim_df = comp.load_dataframe(datatype='sim', config=sim_config, verbose=False, split=False) X_train, y_train = comp.dataframe_functions.dataframe_to_X_y( sim_df, feature_list) print('Training classifier...') pipeline = pipeline.fit(X_train, y_train) X_data = comp.dataframe_functions.dataframe_to_array(data_df, feature_list) data_pred = pd.Series(pipeline.predict(X_data), dtype=int) data_df['pred_comp'] = data_pred.apply( comp.dataframe_functions.label_to_comp) # print('decision_function = {}'.format(pipeline.decision_function(X_data))) # data_df['score'] = pipeline.decision_function(X_data) print('Saving anisotropy DataFrame for {}'.format(config)) with pd.HDFStore(outfile, 'w') as store: store.put('dataframe', data_df.loc[:, keep_columns], format='table') return
comp_list = comp.get_comp_list(num_groups=num_groups) energybins = comp.get_energybins(config) num_ebins = len(energybins.log_energy_midpoints) data_dir = os.path.join(comp.paths.comp_data_dir, config, 'unfolding', 'datachallenge') # Load simulation and train composition classifier df_sim_train, df_sim_test = comp.load_sim(config=config, energy_reco=False, log_energy_min=None, log_energy_max=None, test_size=0.5, verbose=True) feature_list, feature_labels = comp.get_training_features() print('Loading energy regressor...') energy_pipeline = comp.load_trained_model( 'linearregression_energy_{}'.format(config)) # energy_pipeline = comp.load_trained_model('RF_energy_{}'.format(config)) for df in [df_sim_train, df_sim_test]: df['reco_log_energy'] = energy_pipeline.predict( df[feature_list].values) df['reco_energy'] = 10**df['reco_log_energy'] print('Loading or fitting composition classifier...') if any([ args.weights_model, args.energy_spectrum_weights, args.compositon_weights ]):
comp_list = comp.get_comp_list(num_groups=num_groups) energybins = comp.get_energybins(config=config) log_energy_min = energybins.log_energy_min log_energy_max = energybins.log_energy_max # Load training data and fit model df_sim_train, df_sim_test = comp.load_sim( config=config, energy_reco=True, log_energy_min=None, log_energy_max=None, # log_energy_min=log_energy_min, # log_energy_max=log_energy_max, test_size=0.5) features, feature_labels = comp.get_training_features(args.features) # Add random training feature if specified if args.random_feature: np.random.seed(2) df_sim_train['random'] = np.random.random(size=len(df_sim_train)) features.append('random') feature_labels.append('random') X_train = df_sim_train[features].values y_train = df_sim_train['comp_target_{}'.format(num_groups)].values # Will need energy for each event to make classification performance vs. energy plot log_energy_train = df_sim_train['reco_log_energy'].values pipeline_str = '{}_comp_{}_{}-groups'.format(args.pipeline, config, num_groups)
def get_classified_fractions(df_train, df_test, pipeline_str=None, num_groups=4, energy_key='MC_log_energy'): '''Calculates the fraction of correctly identified samples in each energy bin for each composition in comp_list. In addition, the statisitcal error for the fraction correctly identified is calculated.''' # Input validation if energy_key not in ['MC_log_energy', 'reco_log_energy']: raise ValueError( "Invalid energy_key ({}) entered. Must be either " "'MC_log_energy' or 'reco_log_energy'.".format(energy_key)) if pipeline_str is None: pipeline_str = 'BDT_comp_IC86.2012_{}-groups'.format(num_groups) # Fit pipeline and get mask for correctly identified events feature_list, feature_labels = comp.get_training_features() if 'CustomClassifier' in pipeline_str: pipeline = comp.get_pipeline(pipeline_str) else: pipeline = comp.load_trained_model(pipeline_str) comp_target_str = 'comp_target_{}'.format(num_groups) if 'CustomClassifier' in pipeline_str: test_predictions = pipeline.predict( df_test['comp_target_{}'.format(num_groups)]) else: test_predictions = pipeline.predict(df_test[feature_list]) pred_comp = np.array( comp.decode_composition_groups(test_predictions, num_groups=num_groups)) data = {} for true_composition, identified_composition in product( comp_list, comp_list): true_comp_mask = df_test['comp_group_{}'.format( num_groups)] == true_composition ident_comp_mask = pred_comp == identified_composition # Get number of MC comp in each energy bin num_true_comp, _ = np.histogram(df_test.loc[true_comp_mask, energy_key], bins=energybins.log_energy_bins) num_true_comp_err = np.sqrt(num_true_comp) # Get number of correctly identified comp in each energy bin combined_mask = true_comp_mask & ident_comp_mask num_identified_comp, _ = np.histogram(df_test.loc[combined_mask, energy_key], bins=energybins.log_energy_bins) num_identified_comp_err = np.sqrt(num_identified_comp) # Calculate correctly identified fractions as a function of energy frac_identified, frac_identified_err = comp.ratio_error( num_identified_comp, num_identified_comp_err, num_true_comp, num_true_comp_err) data['true_{}_identified_{}'.format( true_composition, identified_composition)] = frac_identified data['true_{}_identified_{}_err'.format( true_composition, identified_composition)] = frac_identified_err return data