def get_frac_correct(df_train,
df_test,
pipeline_str=None,
num_groups=4,
energy_key='MC_log_energy'):
'''Calculates the fraction of correctly identified samples in each energy bin
for each composition in comp_list. In addition, the statisitcal error for the
fraction correctly identified is calculated.'''
# Input validation
if energy_key not in ['MC_log_energy', 'reco_log_energy']:
raise ValueError(
"Invalid energy_key ({}) entered. Must be either "
"'MC_log_energy' or 'reco_log_energy'.".format(energy_key))
if pipeline_str is None:
pipeline_str = 'BDT_comp_IC86.2012_{}-groups'.format(num_groups)
# Fit pipeline and get mask for correctly identified events
feature_list, feature_labels = comp.get_training_features()
pipeline = comp.get_pipeline(pipeline_str)
comp_target_str = 'comp_target_{}'.format(num_groups)
pipeline.fit(df_train[feature_list], df_train[comp_target_str])
test_predictions = pipeline.predict(df_test[feature_list])
correctly_identified_mask = (test_predictions == df_test[comp_target_str])
data = {}
for composition in comp_list + ['total']:
comp_mask = df_test['comp_group_{}'.format(num_groups)] == composition
# Get number of MC comp in each energy bin
num_MC_energy, _ = np.histogram(df_test.loc[comp_mask, energy_key],
bins=energybins.log_energy_bins)
num_MC_energy_err = np.sqrt(num_MC_energy)
# Get number of correctly identified comp in each energy bin
combined_mask = comp_mask & correctly_identified_mask
num_reco_energy, _ = np.histogram(df_test.loc[combined_mask,
energy_key],
bins=energybins.log_energy_bins)
num_reco_energy_err = np.sqrt(num_reco_energy)
# Calculate correctly identified fractions as a function of energy
frac_correct, frac_correct_err = comp.ratio_error(
num_reco_energy, num_reco_energy_err, num_MC_energy,
num_MC_energy_err)
data['frac_correct_{}'.format(composition)] = frac_correct
data['frac_correct_err_{}'.format(composition)] = frac_correct_err
return data
def get_BDT_scores(config):
df_data = comp.load_dataframe(datatype='data', config=config)
feature_list, feature_labels = comp.get_training_features()
df_data.loc[:, feature_list].dropna(axis=0, how='any', inplace=True)
# Load saved pipeline
model_file = os.path.join(comp.paths.project_home,
'models/GBDT_IC86.2012.pkl')
pipeline = joblib.load(model_file)['pipeline']
# Get BDT scores for each data event
X_data = comp.dataframe_functions.dataframe_to_array(df_data, feature_list)
classifier_scores = pipeline.decision_function(X_data)
print('{} complete!'.format(config))
return classifier_scores
def save_anisotropy_dataframe(config, outfile):
print('Loading data...')
data_df = comp.load_dataframe(datatype='data',
config=config,
verbose=False)
keep_columns = [
'lap_zenith', 'lap_azimuth', 'start_time_mjd', 'pred_comp',
'lap_log_energy'
]
comp_list = ['light', 'heavy']
pipeline_str = 'GBDT'
pipeline = comp.get_pipeline(pipeline_str)
feature_list, feature_labels = comp.get_training_features()
data_df.loc[:, feature_list].dropna(axis=0, how='any', inplace=True)
print('Loading simulation...')
if 'IC86' in config:
sim_config = 'IC86.2012'
else:
sim_config = 'IC79'
sim_df = comp.load_dataframe(datatype='sim',
config=sim_config,
verbose=False,
split=False)
X_train, y_train = comp.dataframe_functions.dataframe_to_X_y(
sim_df, feature_list)
print('Training classifier...')
pipeline = pipeline.fit(X_train, y_train)
X_data = comp.dataframe_functions.dataframe_to_array(data_df, feature_list)
data_pred = pd.Series(pipeline.predict(X_data), dtype=int)
data_df['pred_comp'] = data_pred.apply(
comp.dataframe_functions.label_to_comp)
# print('decision_function = {}'.format(pipeline.decision_function(X_data)))
# data_df['score'] = pipeline.decision_function(X_data)
print('Saving anisotropy DataFrame for {}'.format(config))
with pd.HDFStore(outfile, 'w') as store:
store.put('dataframe', data_df.loc[:, keep_columns], format='table')
return
comp_list = comp.get_comp_list(num_groups=num_groups)
energybins = comp.get_energybins(config)
num_ebins = len(energybins.log_energy_midpoints)
data_dir = os.path.join(comp.paths.comp_data_dir, config, 'unfolding',
'datachallenge')
# Load simulation and train composition classifier
df_sim_train, df_sim_test = comp.load_sim(config=config,
energy_reco=False,
log_energy_min=None,
log_energy_max=None,
test_size=0.5,
verbose=True)
feature_list, feature_labels = comp.get_training_features()
print('Loading energy regressor...')
energy_pipeline = comp.load_trained_model(
'linearregression_energy_{}'.format(config))
# energy_pipeline = comp.load_trained_model('RF_energy_{}'.format(config))
for df in [df_sim_train, df_sim_test]:
df['reco_log_energy'] = energy_pipeline.predict(
df[feature_list].values)
df['reco_energy'] = 10**df['reco_log_energy']
print('Loading or fitting composition classifier...')
if any([
args.weights_model, args.energy_spectrum_weights,
args.compositon_weights
]):
comp_list = comp.get_comp_list(num_groups=num_groups)
energybins = comp.get_energybins(config=config)
log_energy_min = energybins.log_energy_min
log_energy_max = energybins.log_energy_max
# Load training data and fit model
df_sim_train, df_sim_test = comp.load_sim(
config=config,
energy_reco=True,
log_energy_min=None,
log_energy_max=None,
# log_energy_min=log_energy_min,
# log_energy_max=log_energy_max,
test_size=0.5)
features, feature_labels = comp.get_training_features(args.features)
# Add random training feature if specified
if args.random_feature:
np.random.seed(2)
df_sim_train['random'] = np.random.random(size=len(df_sim_train))
features.append('random')
feature_labels.append('random')
X_train = df_sim_train[features].values
y_train = df_sim_train['comp_target_{}'.format(num_groups)].values
# Will need energy for each event to make classification performance vs. energy plot
log_energy_train = df_sim_train['reco_log_energy'].values
pipeline_str = '{}_comp_{}_{}-groups'.format(args.pipeline, config,
num_groups)
def get_classified_fractions(df_train,
df_test,
pipeline_str=None,
num_groups=4,
energy_key='MC_log_energy'):
'''Calculates the fraction of correctly identified samples in each energy bin
for each composition in comp_list. In addition, the statisitcal error for the
fraction correctly identified is calculated.'''
# Input validation
if energy_key not in ['MC_log_energy', 'reco_log_energy']:
raise ValueError(
"Invalid energy_key ({}) entered. Must be either "
"'MC_log_energy' or 'reco_log_energy'.".format(energy_key))
if pipeline_str is None:
pipeline_str = 'BDT_comp_IC86.2012_{}-groups'.format(num_groups)
# Fit pipeline and get mask for correctly identified events
feature_list, feature_labels = comp.get_training_features()
if 'CustomClassifier' in pipeline_str:
pipeline = comp.get_pipeline(pipeline_str)
else:
pipeline = comp.load_trained_model(pipeline_str)
comp_target_str = 'comp_target_{}'.format(num_groups)
if 'CustomClassifier' in pipeline_str:
test_predictions = pipeline.predict(
df_test['comp_target_{}'.format(num_groups)])
else:
test_predictions = pipeline.predict(df_test[feature_list])
pred_comp = np.array(
comp.decode_composition_groups(test_predictions,
num_groups=num_groups))
data = {}
for true_composition, identified_composition in product(
comp_list, comp_list):
true_comp_mask = df_test['comp_group_{}'.format(
num_groups)] == true_composition
ident_comp_mask = pred_comp == identified_composition
# Get number of MC comp in each energy bin
num_true_comp, _ = np.histogram(df_test.loc[true_comp_mask,
energy_key],
bins=energybins.log_energy_bins)
num_true_comp_err = np.sqrt(num_true_comp)
# Get number of correctly identified comp in each energy bin
combined_mask = true_comp_mask & ident_comp_mask
num_identified_comp, _ = np.histogram(df_test.loc[combined_mask,
energy_key],
bins=energybins.log_energy_bins)
num_identified_comp_err = np.sqrt(num_identified_comp)
# Calculate correctly identified fractions as a function of energy
frac_identified, frac_identified_err = comp.ratio_error(
num_identified_comp, num_identified_comp_err, num_true_comp,
num_true_comp_err)
data['true_{}_identified_{}'.format(
true_composition, identified_composition)] = frac_identified
data['true_{}_identified_{}_err'.format(
true_composition, identified_composition)] = frac_identified_err
return data