def run(self, x, y):
""" Single training run of the given model. It is assumed that the
input data is preprocessed, normalized, and good to go.
:param x: input data (batch_size, width, height, channels)
:param y: labels
"""
# Set indices for train and validation
x_idx = np.arange(x.shape[0])
train_idx, val_idx = train_test_split(
x_idx, random_state=self.config['random_seed'])
# Train the model
self.history = self.model.fit(
x=normalize_image_data(x[train_idx]),
y=y[train_idx],
validation_data=(normalize_image_data(x[val_idx]), y[val_idx]),
**self.config['fit_args'],
).history
# Calculate metrics for the model
if self.model_type == "classification":
self.classification_metrics(x[val_idx], y[val_idx])
elif self.model_type == "regression":
self.regression_metrics(x[val_idx], y[val_idx])
# Store indices for training and validation in config output
# Need conversion to list as numpy arrays aren't json serializable.
# To make the indices in config more uniform in format, we treat
# a non-kfold experiment like a 1-fold experiment.
self.indices['fold_0'] = {
'train_idx': train_idx.tolist(),
'val_idx': val_idx.tolist(),
}
def run_kfold(self, x, y, f1_print=False):
""" Train the model using kfold cross-validation.
It is assumed that the input data is preprocessed,
and good to go.
:param x: input data (batch_size, width, height, channels)
:param y: labels / targets
:param f1_print: whether to print f1 score after each fold or not.
"""
# Store accuracy for each fold for all models
results = {}
# Create KFold data generator
kf = KFold(random_state=self.config['random_seed'],
**self.config['kfold_args'])
original_model = tf.keras.models.clone_model(self.model)
# Run k-fold cross-validation
fold = 0 # Track which fold
for train_idx, val_idx in kf.split(x, y):
# Reinitialize model
self.model = tf.keras.models.clone_model(original_model)
self.model.compile(optimizer=Adam(
learning_rate=self.config['compile_args']['adam_lr']),
loss=self.config['compile_args']['loss'],
metrics=self.config['compile_args']['metrics'])
# Train model
history = self.model.fit(
x=normalize_image_data(x[train_idx]),
y=y[train_idx],
validation_data=(normalize_image_data(x[val_idx]), y[val_idx]),
**self.config['fit_args'],
).history
# Calculate metrics for the model
if self.model_type == "classification":
self.classification_metrics(x[val_idx], y[val_idx], fold)
elif self.model_type == "regression":
self.regression_metrics(x[val_idx], y[val_idx], fold)
# Store train and val indices for the fold
foldkey = 'fold_' + str(fold)
self.indices[foldkey] = {
'train_idx': train_idx.tolist(),
'val_idx': val_idx.tolist(),
}
# Store the history object
results[foldkey] = history
if f1_print:
print("\n", foldkey, " F1-score: ",
self.metrics_kfold[foldkey]['f1_score'])
fold += 1
self.history_kfold = results
def classification_metrics(self, x_val, y_val, fold=None):
""" Calculates f1_score, matthews_corrcoef, confusion matrix and
roc area under curve, accuracy metrics and stores them in the
metrics attribute.
The values are calculated based on the validation data.
Recall that the default positive class for f1_score is 1
"""
# Get prediction and make class labels based on threshold of 0.5
y_out = self.model.predict(normalize_image_data(x_val))
y_pred = y_out > 0.5
confmat = confusion_matrix(y_val, y_pred)
metrics = {}
metrics['accuracy_score'] = accuracy_score(y_val, y_pred)
metrics['confusion_matrix'] = {
'TN': int(confmat[0, 0]),
'FP': int(confmat[0, 1]),
'FN': int(confmat[1, 0]),
'TP': int(confmat[1, 1]),
}
metrics['f1_score'] = f1_score(y_val, y_pred)
metrics['matthews_corrcoef'] = matthews_corrcoef(y_val, y_pred)
metrics['roc_auc_score'] = roc_auc_score(y_val, y_out)
if fold is not None:
foldkey = 'fold_' + str(fold)
self.metrics_kfold[foldkey] = metrics
else:
self.metrics = metrics
def regression_metrics(self, x_val, y_val, fold=None):
""" Calculates regression metrics on the validation data.
"""
# Get prediction and make class labels based on threshold of 0.5
y_pred = self.model.predict(normalize_image_data(x_val))
metrics = {}
metrics['r2_score'] = r2_score(y_val, y_pred)
metrics['mse'] = mean_squared_error(y_val, y_pred)
metrics['rmse'] = mean_squared_error(y_val, y_pred, squared=False)
metrics['mae'] = mean_absolute_error(y_val, y_pred)
if fold is not None:
foldkey = 'fold_' + str(fold)
self.metrics_kfold[foldkey] = metrics
else:
self.metrics = metrics
'data': "200k",
}
# ================== Import Data ==================
DATA_PATH = get_git_root() + "data/simulated/"
images = np.load(DATA_PATH + f"images_{config['data']}.npy")
images = images.reshape(images.shape[0], 16, 16, 1)
positions = np.load(DATA_PATH + "positions_200k.npy")
labels = np.load(DATA_PATH + "labels_200k.npy")
single_indices, double_indices, close_indices = event_indices(positions)
train_idx, val_idx, non1, non2 = train_test_split(
double_indices, double_indices, random_state=config['random_seed'])
# log-scale the images if desireable
config['scaling'] = "minmax"
# set tf random seed
with tf.device(get_tf_device(20)):
reg = ak.ImageRegressor(
overwrite=True,
max_trials=100,
)
# Feed the structured data regressor with training data.
reg.fit(
normalize_image_data(images[train_idx]),
normalize_position_data(positions[train_idx]),
validation_data=(normalize_image_data(images[val_idx]), normalize_position_data(positions[val_idx])),
epochs=10,
)
predicted_y = reg.predict(normalize_image_data(images[val_idx]))
print(reg.evaluate(normalize_image_data(images[val_idx]), normalize_position_data(positions[val_idx])))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
# Run experiment
experiment = Experiment(model=model,
config=config,
model_type="classification",
experiment_name=search_name)
experiment.run(
normalize_image_data(images[train_idx]),
labels[train_idx],
normalize_image_data(images[val_idx]),
labels[val_idx],
)
experiment.save()
id_param[experiment.id] = {
'batch_size': b_size,
}
search_path = get_git_root() + "experiments/searches/"
with open(search_path + search_name + ".json", "w") as fp:
json.dump(id_param, fp, indent=2)
'ENERGY_MODEL': "7eb0ab25bf53.h5",
'POSITIONS_MODEL': "7eb0ab25bf53.h5",
}
# Note this is not "real data" this is a test sanity check the models
#NPFILE = "../../data/simulated/CeBr10k_1.npy"
NPFILE = "anodedata_500k.npy"
print(config['DATA_PATH'] + NPFILE)
events, images = import_real_data(
config['DATA_PATH'] + config['DATA_FILENAME']) # Images not normalized
#images = np.load(config['DATA_PATH'] + NPFILE)
images = images.reshape(images.shape[0], 16, 16, 1)
images = normalize_image_data(images) # Normalize images
descriptors = list(
set([event['event_descriptor'] for event in events.values()]))
# Load models
model = tf.keras.models.load_model(config['MODEL_PATH'] + config['CLASSIFIER'])
# Classify events
prediction = model.predict(images)
event_classification = (prediction > 0.5).astype(int)
for event_id in events.keys():
if event_classification[events[event_id]['image_idx']] == 0:
events[event_id]['event_class'] = "single"
else:
events[event_id]['event_class'] = "double"
# import real data
config_real = {
'DATA_PATH': "../../data/real/",
'DATA_FILENAME': "anodedata_500k.txt",
'MODEL_PATH': "../../models/",
'RESULTS_PATH': "../../results/",
'CLASSIFIER': "367e35da671b.h5",
'ENERGY_MODEL': "2137bd6d101c.h5",
'POSITIONS_MODEL': "337cafc233f7.h5",
}
events, images_real = import_real_data(
config_real['DATA_PATH'] +
config_real['DATA_FILENAME']) # Images not normalized
images_real = images_real.reshape(images_real.shape[0], 16, 16, 1)
images_real = normalize_image_data(images_real) # Normalize images
# log-scale the images if desireable
config['scaling'] = "minmax"
if "np.log" in config['scaling']:
images = np.log1p(images)
# set tf random seed
tf.random.set_seed(config['random_seed'])
experiments = {}
with tf.device(get_tf_device(20)):
models = {}
# Logistic
model = Sequential()
model.add(InputLayer(input_shape=(256, )))
model.add(Dense(1, activation='sigmoid'))
for config in configs:
images = np.load(config['DATA_PATH'] + config['IMAGE_FILE'])
labels = np.load(config['DATA_PATH'] + config['LABEL_FILE'])
#print("DEBUG: ", labels)
print("DEBUG: ")
print("CLASSIFIER: ", config['CLASSIFIER'])
print("IMAGE_FILE: ", config['IMAGE_FILE'])
if config['ML_METHOD'] == 'CNN':
images = images.reshape(images.shape[0],16,16,1)
else:
images = images.reshape(images.shape[0], 256)
model = tf.keras.models.load_model(config['MODEL_PATH'] + config['CLASSIFIER'])
pred = model.predict(normalize_image_data(images))
result = pred > 0.5
accuracy = accuracy_score(labels, result)
confmat = confusion_matrix(labels, result)
f1 = f1_score(labels, result)
mcc = matthews_corrcoef(labels, result)
print("Model:", config['NAME'])
print("Confusion matrix:\n", confmat)
print("Accuracy:", accuracy)
print("F1-score:", f1)
print("MCC:", mcc)
prediction_model = tf.keras.Model(inputs=inputs, outputs=outputs)
model.compile(
optimizer='adam',
loss='mse',
)
prediction_model.compile(
optimizer='adam',
loss='mse',
)
print(model.summary())
# Run experiment
experiment = Experiment(model=model,
config=config,
model_type="regression",
experiment_name=search_name)
experiment.run(
normalize_image_data(images[single_indices]),
normalize_position_data(positions[single_indices])[:, :2],
)
experiment.save()
mpath = experiment.config['path_args']['models'] + experiment.id + ".h5"
prediction_model.save(mpath)
heatmaps, coords = prediction_model.predict(
images[single_indices][experiment.indices['fold_0']['val_idx']])
np.save("dsnt_heatmaps_pred.npy", heatmaps)
np.save("dsnt_coords_pred.npy", coords)
print("Finished experiment.")
print("Name:", search_name)
print("id:", experiment.id)
Conv2D(32,
kernel_size=(3, 3),
activation='relu',
input_shape=(16, 16, 1),
padding=padding))
model.add(Conv2D(64, (3, 3), activation='relu', padding=padding))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3), activation='relu', padding=padding))
model.add(Conv2D(64, (3, 3), activation='relu', padding=padding))
model.add(Flatten())
model.add(Dense(256, activation='relu'))
model.add(Dense(4, activation='linear'))
model.compile(
loss='mse',
optimizer='adam',
)
print(model.summary())
# Run experiment
experiment = Experiment(model=model,
config=config,
model_type="regression",
experiment_name=search_name)
experiment.run(
normalize_image_data(images[double_indices]),
normalize_position_data(positions[double_indices]),
)
experiment.save()
mpath = experiment.config['path_args']['models'] + experiment.id + ".h5"
model.save(mpath)
