def evaluate(self, model, custom_objects=None, **kwargs):
'''
evaluate
'''
name = kwargs['name'] if kwargs.has_key('name') else 'model'
title = kwargs['title'] if kwargs.has_key('title') else 'QGJets'
if isinstance(model, str):
filepath = model
del model
model = tf.keras.models.load_model(
filepath=filepath,
custom_objects=custom_objects)
ckpt_name = os.path.basename(filepath).replace(".hdf5", "")
epoch = kh.utils.misc.parse_str(ckpt_name, target="epoch")
name += "_epoch-{:02d}".format(epoch)
title += " Epoch {}".format(epoch)
roc_curve = ROCCurve(
name=name,
title=title,
directory=self.log_dir.roc_curve.path)
model_response = BinaryClassifierResponse(
name=name,
title=title,
directory=self.log_dir.model_response.path)
##########################
# training data
###########################
print("TRAINING SET")
for_loop_seq = tqdm(self.train_iter) if HAS_TQDM else self.train_iter
for x, y in for_loop_seq:
y_true = y[0]
y_score = model.predict_on_batch(x)
model_response.append(is_train=True,
y_true=y_true,
y_score=y_score)
#############################
# test data
########################
print("TEST SET")
for_loop_seq = tqdm(self.test_iter) if HAS_TQDM else self.test_iter
for x, y in for_loop_seq:
y_true = y[0]
y_score = model.predict_on_batch(x)
model_response.append(is_train=False,
y_true=y_true,
y_score=y_score)
roc_curve.append(y_true=y_true, y_score=y_score)
roc_curve.finish()
model_response.finish()
def evaluate(checkpoint_path,
train_iter,
test_iter,
log_dir):
model = load_model(checkpoint_path)
ckpt_name = os.path.basename(checkpoint_path).replace(".hdf5", "")
epoch = parse_str(ckpt_name, target="epoch")
name = "model_epoch-{:02d}".format(epoch)
title = "Quark/Gluon Jet Discrimination (Epoch {})".format(epoch)
roc_curve = ROCCurve(
name=name,
title=title,
directory=log_dir.roc_curve.path)
model_response = BinaryClassifierResponse(
name=name,
title=title,
directory=log_dir.model_response.path)
##########################
# training data
###########################
print("TRAINING SET")
for batch_idx, batch in enumerate(train_iter, 1):
y_score = model.predict_on_batch([batch.x])
model_response.append(is_train=True,
y_true=batch.y,
y_score=y_score)
#############################
# Test on dijet dataset
########################
print("TEST SET")
for batch_idx, batch in enumerate(test_iter, 1):
y_score = model.predict_on_batch([batch.x])
model_response.append(is_train=False,
y_true=batch.y,
y_score=y_score)
roc_curve.append(y_true=batch.y, y_score=y_score)
roc_curve.finish()
model_response.finish()
def evaluate(model, train_iter, test_iter, log_dir, custom_objects={}):
name = "model"
title = "Quark/Gluon Jet Discrimination (Ensemble: ConvNet + GRU)"
roc_curve = ROCCurve(name=name,
title=title,
directory=log_dir.roc_curve.path)
model_response = BinaryClassifierResponse(
name=name, title=title, directory=log_dir.model_response.path)
##########################
# training data
###########################
print("TRAINING SET")
for batch_idx, batch in enumerate(train_iter, 1):
if batch_idx == 1 or batch_idx % 10 == 0:
print("\t[{}/{}] {:.3f} %".format(
batch_idx, len(train_iter), 100 * batch_idx / len(train_iter)))
y_score = model.predict_on_batch(
[batch.x_img, batch.x_kin, batch.x_pid, batch.x_len])
model_response.append(is_train=True, y_true=batch.y, y_score=y_score)
#############################
# Test on dijet dataset
########################
print("TEST SET")
for batch_idx, batch in enumerate(test_iter, 1):
if batch_idx == 1 or batch_idx % 10 == 0:
print("\t[{}/{}] {:.3f} %".format(batch_idx, len(test_iter), 100 *
batch_idx / len(test_iter)))
y_score = model.predict_on_batch(
[batch.x_img, batch.x_kin, batch.x_pid, batch.x_len])
model_response.append(is_train=False, y_true=batch.y, y_score=y_score)
roc_curve.append(y_true=batch.y, y_score=y_score)
roc_curve.finish()
model_response.finish()