def plot_truth_vs_prediction(
y_pred: Union[JOINTS_25D, JOINTS_3D],
y_true: Union[JOINTS_25D, JOINTS_3D],
image: torch.Tensor,
experiment: Experiment,
):
"""Generates the graphics with input image, predicetd labels and the ground truth.
Args:
y_pred (Union[JOINTS_25D, JOINTS_3D]): Output from the model as a tensor. shape (21 x 3)
y_true (Union[JOINTS_25D, JOINTS_3D]): ground truth. shape(21 x 3)
image (torch.Tensor): Input image to the model.
experiment (Experiment): Comet ml experiment object.
"""
img = cv2.cvtColor(np.array(transforms.ToPILImage()(image)),
cv2.COLOR_BGR2RGB)
width, height, _ = img.shape
fig = plt.figure(figsize=(10, 10))
ax1 = fig.add_subplot(121)
plt.imshow(img)
plot_hand(ax1, y_true)
ax1.title.set_text("True joints")
ax2 = fig.add_subplot(122)
plot_hand(ax2, y_true, alpha=0.2, linestyle=":")
plot_hand(ax2, y_pred)
ax2.set_xlim([0, width])
ax2.set_ylim([height, 0])
ax2.title.set_text("Predicted joints")
if experiment is not None:
experiment.log_figure(figure=plt)
plt.close()
def train(path):
name = os.path.splitext(os.path.basename(path))[0]
print('Processing: ', name)
features = pd.read_csv(path, index_col=None)
selected_features_names = [name for name, desc in selected_features]
features = features[selected_features_names]
split_idx = 1200
features = features.drop(['sound.files'], axis=1)
noise_only_df, df = features.iloc[:split_idx], features.iloc[split_idx:]
y = df.pop('petrel')
X = df.values
y_noise = noise_only_df.pop('petrel')
X_noise = noise_only_df.values
X_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.25, random_state=42, stratify=y)
hyperparams = {
'n_estimators': [100, 300, 500, 1000],
'learning_rate': [0.1],
'gamma': [0.0, 0.5],
'max_depth': [2, 3, 4],
'min_child_weight': [1, 2],
'subsample': [1.0, 0.8],
'reg_alpha': [0.0, 0.1],
'reg_lambda': [1, 2, 3]
}
#
# hyperparams = {
# 'n_estimators': [100],
# 'learning_rate': [0.1],
# 'gamma': [0.0],
# 'max_depth': [2],
# 'min_child_weight': [1],
# 'subsample': [1.0],
# 'reg_alpha': [0.0],
# 'reg_lambda': [1]
# }
clf = model_selection.GridSearchCV(estimator=xg.XGBClassifier(objective='binary:logistic', n_jobs=-1),
param_grid=hyperparams,
cv=4)
fit_params = clf.fit(X_train, y_train)
estimator = fit_params.best_estimator_
joblib.dump(estimator, name + '_model.pkl')
test_pred = estimator.predict(X_test)
metrics = calculate_metrics(test_pred, y_test)
noise_pred = estimator.predict(X_noise)
noise_detection_accuracy = accuracy_score(y_noise, noise_pred)
experiment = Experiment(api_key="4PdGdUZmGf6P8QsMa5F2zB4Ui",
project_name="storm petrels",
workspace="tracewsl")
experiment.set_name(name)
experiment.log_parameter('name', name)
experiment.log_multiple_params(fit_params.best_params_)
experiment.log_multiple_metrics(metrics)
experiment.log_metric('Noise detection accuracy', noise_detection_accuracy)
experiment.log_figure('Confusion matrix', get_confusion_matrix_figure(test_pred, y_test))
experiment.log_figure('Feature importnace', get_feature_importance_figure(estimator, list(df.columns.values)))
def plot_simclr_images(img1: np.array, img2: np.array,
comet_logger: Experiment):
fig = plt.figure(figsize=(10, 10))
ax = fig.add_subplot(121)
plt.imshow(
cv2.cvtColor(np.array(transforms.ToPILImage()(img1.cpu())),
cv2.COLOR_BGR2RGB))
ax.set_title("Image 1")
ax = fig.add_subplot(122)
plt.imshow(
cv2.cvtColor(np.array(transforms.ToPILImage()(img2.cpu())),
cv2.COLOR_BGR2RGB))
ax.set_title("Image 2")
if comet_logger is not None:
comet_logger.log_figure(figure=plt)
plt.close()
def log(self, experiment=None):
''' Export all logs in the Comet.ml environment.
See https://www.comet.ml/ for more details
'''
# Initialize Comet.ml experience (naming, tags) for automatic logging
project_name = 'Optimization' if self.comet_optimize else 'Summary'
experiment_name = '{} - {} '.format(self.model_name, str(self.batch_size)) + ('ES+' if self.train_after_es else '')
experiment_tags = [ self.model_name, self.monitor_val ] + (['ES+'] if self.train_after_es else []) + (['Pre-train'] if self.pretraining else [])
if experiment == None:
experiment = Experiment(api_key='cSZq9kuH2I87ezvm2dEWTx6op', project_name=project_name, log_code=False, auto_param_logging=False, auto_metric_logging=False)
experiment.set_name(experiment_name)
experiment.add_tags(experiment_tags)
# Export hyperparameters
experiment.log_parameters(self.dataloader_params)
experiment.log_parameters(self.training_params)
# Export metrics values
experiment.log_metrics({'Average accuracy' : np.mean(self.test_score['accuracy']), 'Std accuracy' : np.std(self.test_score['accuracy'])})
# Export metrics graphs for each pilot (accuracy, loss, confusion matrix)
[ experiment.log_figure(figure_name='Confusion matrix {}'.format(pilot_idx), figure=plot_cm(self.conf_matrices, pilot_idx)) for pilot_idx in range(1,self.n_pilots+1)]
[ experiment.log_figure(figure_name='Loss pilot {}'.format(pilot_idx), figure=plot_loss(self.histories[pilot_idx-1], pilot_idx)) for pilot_idx in range(1,self.n_pilots+1)]
fig, ax = plt.subplots(figsize=(10,6))
plot_full_barchart(self.test_score, n_pilots=self.n_pilots, title=' {} ConvNet model'.format(self.model_name), fig=fig)
experiment.log_figure(figure_name='Accuracy barchart', figure=fig)
if self.train_after_es:
[ experiment.log_figure(figure_name='Loss pilot {} (ES+)'.format(pilot_idx), figure=plot_loss(self.histories_es[pilot_idx-1], pilot_idx)) for pilot_idx in range(1,self.n_pilots+1)]
# Export model weights for each pilot
[ experiment.log_asset('{}{}.h5'.format(self.weights_savename_prefix, pilot_idx)) for pilot_idx in range(1,self.n_pilots+1)]
experiment.end()
import matplotlib as mpl
mpl.use('TkAgg')
import matplotlib.pyplot as plt
import numpy as np
from comet_ml import Experiment
experiment = Experiment(api_key="API_KEY",
project_name='matplotlib-demos',
auto_param_logging=False)
t = np.arange(0.0, 2.0, 0.01)
s = 1 + np.sin(2*np.pi*t)
plt.plot(t, s)
plt.xlabel('time (s)')
plt.ylabel('voltage (mV)')
plt.title('About as simple as it gets, folks')
plt.grid(True)
experiment.log_figure(figure=plt)
class Logger:
"""
Logs/plots results to comet.
Args:
exp_config (dict): experiment configuration hyperparameters
model_config (dict): model configuration hyperparameters
data_config (dict): data configuration hyperparameters
"""
def __init__(self, exp_config, model_config, data_config):
self.exp_config = exp_config
self.experiment = Experiment(**exp_config['comet_config'])
self.experiment.disable_mp()
self._log_hyper_params(exp_config, model_config, data_config)
self._epoch = 0
def _log_hyper_params(self, exp_config, model_config, data_config):
"""
Log the hyper-parameters for the experiment.
Args:
exp_config (dict): experiment configuration hyperparameters
model_config (dict): model configuration hyperparameters
data_config (dict): data configuration hyperparameters
"""
def flatten_arg_dict(arg_dict):
flat_dict = {}
for k, v in arg_dict.items():
if type(v) == dict:
flat_v = flatten_arg_dict(v)
for kk, vv in flat_v.items():
flat_dict[k + '_' + kk] = vv
else:
flat_dict[k] = v
return flat_dict
self.experiment.log_parameters(flatten_arg_dict(exp_config))
self.experiment.log_parameters(flatten_arg_dict(model_config))
self.experiment.log_parameters(flatten_arg_dict(data_config))
def log(self, results, train_val):
"""
Plot the results in comet.
Args:
results (dict): dictionary of metrics to plot
train_val (str): either 'train' or 'val'
"""
objectives, grads, params, images, metrics = results
for metric_name, metric in objectives.items():
self.experiment.log_metric(metric_name + '_' + train_val, metric,
self._epoch)
print(metric_name, ':', metric.item())
if train_val == 'train':
for grad_metric_name, grad_metric in grads.items():
self.experiment.log_metric('grads_' + grad_metric_name,
grad_metric, self._epoch)
for param_name, param in params.items():
self.experiment.log_metric(param_name + '_' + train_val, param,
self._epoch)
for image_name, imgs in images.items():
self.plot_images(imgs, image_name, train_val)
for metric_name, metric in metrics.items():
self.experiment.log_metric(metric_name + '_' + train_val, metric,
self._epoch)
if train_val == 'val':
self._epoch += 1
def plot_images(self, images, title, train_val):
"""
Plot a tensor of images.
Args:
images (torch.Tensor): a tensor of shape [steps, b, c, h, w]
title (str): title for the images, e.g. reconstructions
train_val (str): either 'train' or 'val'
"""
# add a channel dimension if necessary
if len(images.shape) == 4:
s, b, h, w = images.shape
images = images.view(s, b, 1, h, w)
s, b, c, h, w = images.shape
if b > 10:
images = images[:, :10]
# swap the steps and batch dimensions
images = images.transpose(0, 1).contiguous()
images = images.view(-1, c, h, w)
# grid = make_grid(images.clamp(0, 1), nrow=s).numpy()
grid = make_grid(images, nrow=s).numpy()
if c == 1:
grid = grid[0]
cmap = 'gray'
else:
grid = np.transpose(grid, (1, 2, 0))
cmap = None
plt.imshow(grid, cmap=cmap)
plt.axis('off')
self.experiment.log_figure(figure=plt,
figure_name=title + '_' + train_val)
plt.close()
def save(self, model):
"""
Save the model weights in comet.
Args:
model (nn.Module): the model to be saved
"""
if self._epoch % self.exp_config['checkpoint_interval'] == 0:
print('Checkpointing the model...')
state_dict = model.state_dict()
cpu_state_dict = {k: v.cpu() for k, v in state_dict.items()}
# save the state dictionary
ckpt_path = os.path.join('./ckpt_epoch_' + str(self._epoch) +
'.ckpt')
torch.save(cpu_state_dict, ckpt_path)
self.experiment.log_asset(ckpt_path)
os.remove(ckpt_path)
print('Done.')
def load(self, model):
"""
Load the model weights.
"""
assert self.exp_config[
'checkpoint_exp_key'] is not None, 'Checkpoint experiment key must be set.'
print('Loading checkpoint from ' +
self.exp_config['checkpoint_exp_key'] + '...')
comet_api = comet_ml.papi.API(
rest_api_key=self.exp_config['rest_api_key'])
exp = comet_api.get_experiment(
workspace=self.exp_config['comet_config']['workspace'],
project_name=self.exp_config['comet_config']['project_name'],
experiment=self.exp_config['checkpoint_exp_key'])
# asset_list = comet_api.get_experiment_asset_list(self.exp_config['checkpoint_exp_key'])
asset_list = exp.get_asset_list()
# get most recent checkpoint
ckpt_assets = [
asset for asset in asset_list if 'ckpt' in asset['fileName']
]
asset_times = [asset['createdAt'] for asset in ckpt_assets]
asset = asset_list[asset_times.index(max(asset_times))]
print('Checkpoint Name:', asset['fileName'])
ckpt = exp.get_asset(asset['assetId'])
state_dict = torch.load(io.BytesIO(ckpt))
model.load(state_dict)
print('Done.')
class Trainer2D:
def __init__(self, config):
self.experiment = Experiment(api_key="CQ4yEzhJorcxul2hHE5gxVNGu",
project_name="HIP")
self.experiment.log_parameters(vars(config))
self.config = config
self.log_step = config.log_step
self.model = conv2d.Conv2DPatches(image_size=config.image_size)
print(self.model)
self.d = get_dataloader2D(config)
self.train_loader, self.test_loader = self.d
self.train_loader_jig, self.test_loader_jig = get_dataloader2DJigSaw(
config)
self.net_optimizer = optim.Adam(self.model.parameters(), config.lr,
[0.5, 0.9999])
if torch.cuda.is_available():
self.model = self.model.cuda()
self.criterion_c = nn.CrossEntropyLoss()
self.criterion_d = nn.MSELoss()
self.epochs = config.epochs
if torch.cuda.is_available():
print("Using CUDA")
self.model = self.model.cuda()
# self.model = self.model.cuda()
self.pre_model_path = "./artifacts/pre_models/" + str(
config.lr) + ".pth"
self.model_path = "./artifacts/models/" + str(config.lr) + ".pth"
self.image_size = config.image_size
def pre_train(self):
if os.path.isfile(self.pre_model_path):
print("Using pre-trained model for solving the jigsaw puzzle")
self.model = torch.load(self.pre_model_path)
else:
print("Starting pre-training and solving the jigsaw puzzle")
for epoch in range(0):
print("Starting epoch {}".format(epoch))
train_loader = iter(self.train_loader_jig)
with self.experiment.train():
for i in range(len(train_loader)):
self.net_optimizer.zero_grad()
data, indexes, _ = train_loader.next()
# print(landmarks)
# print(landmarks.shape)
data, indexes = self.to_var(data), self.to_var(
indexes).float()
B, L, H, W = data.size()
B, L, S = indexes.size()
print(data.size())
print(indexes.size())
jig_out, _ = self.model(data, True)
loss = self.criterion_d(jig_out, indexes.view(-1, S))
loss.backward()
self.net_optimizer.step()
# self.plots(y_slices, landmarks[:, :, [0, 2]], detected_points)
self.experiment.log_metric("pre-loss", loss.item())
print("loss: {}".format(loss.item()))
torch.save(self.model, self.pre_model_path)
def train(self):
if os.path.isfile(self.model_path):
print("Using pre-trained model")
self.model = torch.load(self.model_path)
if False:
pass
else:
print("Starting training")
if torch.cuda.is_available():
self.model = self.model.cuda()
for epoch in range(self.epochs):
print("Starting epoch {}".format(epoch))
train_loader = iter(self.train_loader)
with self.experiment.train():
for i in range(len(train_loader)):
self.net_optimizer.zero_grad()
data, landmarks, _ = train_loader.next()
# print(landmarks)
data, landmarks = self.to_var(data), self.to_var(
landmarks)
B, L, H, W = data.size()
B, L, S = landmarks.size()
y = landmarks[:, :, 1].view(B, L)
y_slices = torch.zeros([B, L, H, W],
dtype=torch.float32)
if torch.cuda.is_available():
y_slices = y_slices.cuda()
for i in range(B):
y_slices[i] = data[i, y[i]]
jig_out, detected_points = self.model(y_slices)
landmarks = landmarks.float() / self.image_size
loss = self.criterion_d(detected_points,
landmarks[:, :, [0, 2]])
loss.backward()
self.net_optimizer.step()
# self.plots(y_slices, landmarks[:, :, [0, 2]], detected_points)
self.experiment.log_metric("loss", loss.item())
print("loss: {}".format(loss.item()))
if epoch % self.log_step == 0:
with self.experiment.test():
self.evaluate()
evaluator = Evaluator(self, self.test_loader)
evaluator.report()
torch.save(self.model, self.model_path)
evaluator = Evaluator(self, self.test_loader)
evaluator.report()
def evaluate(self):
test_loader = iter(self.test_loader)
with self.experiment.test():
loss = 0
for i in range(len(test_loader)):
self.net_optimizer.zero_grad()
data, landmarks, _ = test_loader.next()
data, landmarks = self.to_var(data), self.to_var(landmarks)
B, L, H, W = data.size()
B, L, S = landmarks.size()
y = landmarks[:, :, 1].view(B, L)
y_slices = torch.zeros([B, L, H, W], dtype=torch.float32)
if torch.cuda.is_available():
y_slices = y_slices.cuda()
for i in range(B):
y_slices[i] = data[i, y[i]]
jig_out, detected_points = self.model(y_slices)
landmarks = landmarks.float() / self.image_size
loss += self.criterion_d(detected_points,
landmarks[:, :, [0, 2]]).item()
self.plots(y_slices.cpu(), landmarks[:, :, [0, 2]],
detected_points)
self.experiment.log_metric("loss", loss / len(test_loader))
def plots(self, slices, real, predicted):
figure, axes = plt.subplots(nrows=4, ncols=4, figsize=(15, 15))
slices = slices[0].cpu().detach().numpy()
real = real[0].cpu().detach().numpy()
predicted = predicted[0].cpu().detach().numpy()
real *= self.image_size
predicted *= self.image_size
s = 0
# print(real.size())
# print(predicted.size())
for i in range(4):
for j in range(4):
axes[i, j].imshow(slices[s])
x, z = real[s]
axes[i, j].scatter(x, z, color="red")
x, z = predicted[s]
axes[i, j].scatter(x, z, color="blue")
s += 1
self.experiment.log_figure(figure=plt)
plt.savefig("artifacts/predictions/img.png")
plt.show()
def to_var(self, x):
"""Converts numpy to variable."""
if torch.cuda.is_available():
x = x.cuda()
return Variable(x, requires_grad=False)
def to_data(self, x):
"""Converts variable to numpy."""
if torch.cuda.is_available():
x = x.cpu()
return x.data.numpy()
def predict(self, x):
if torch.cuda.is_available():
self.model = self.model.cuda()
x = x.cuda()
_, x = self.model(x)
return x
class Dashboard:
"""Record training/evaluation statistics to comet
:param Path log_dir
:param list taskid_to_name
"""
def __init__(self, config, paras, log_dir, train_type, resume=False):
self.log_dir = log_dir
self.expkey_f = Path(self.log_dir, 'exp_key')
self.global_step = 1
if resume:
assert self.expkey_f.exists(
), f"Cannot find comet exp key in {self.log_dir}"
with open(Path(self.log_dir, 'exp_key'), 'r') as f:
exp_key = f.read().strip()
self.exp = ExistingExperiment(
previous_experiment=exp_key,
project_name=COMET_PROJECT_NAME,
workspace=COMET_WORKSPACE,
auto_output_logging=None,
auto_metric_logging=None,
display_summary_level=0,
)
else:
self.exp = Experiment(
project_name=COMET_PROJECT_NAME,
workspace=COMET_WORKSPACE,
auto_output_logging=None,
auto_metric_logging=None,
display_summary_level=0,
)
#TODO: is there exists better way to do this?
with open(self.expkey_f, 'w') as f:
print(self.exp.get_key(), file=f)
self.exp.log_other('seed', paras.seed)
self.log_config(config)
if train_type == 'evaluation':
if paras.pretrain:
self.exp.set_name(
f"{paras.pretrain_suffix}-{paras.eval_suffix}")
self.exp.add_tags([
paras.pretrain_suffix, config['solver']['setting'],
paras.accent, paras.algo, paras.eval_suffix
])
if paras.pretrain_model_path:
self.exp.log_other("pretrain-model-path",
paras.pretrain_model_path)
else:
self.exp.log_other("pretrain-runs",
paras.pretrain_runs)
self.exp.log_other("pretrain-setting",
paras.pretrain_setting)
self.exp.log_other("pretrain-tgt-accent",
paras.pretrain_tgt_accent)
else:
self.exp.set_name(paras.eval_suffix)
self.exp.add_tags(
["mono", config['solver']['setting'], paras.accent])
else:
self.exp.set_name(paras.pretrain_suffix)
self.exp.log_others({
f"accent{i}": k
for i, k in enumerate(paras.pretrain_accents)
})
self.exp.log_other('accent', paras.tgt_accent)
self.exp.add_tags([
paras.algo, config['solver']['setting'], paras.tgt_accent
])
#TODO: Need to add pretrain setting
##slurm-related
hostname = os.uname()[1]
if len(hostname.split('.')) == 2 and hostname.split(
'.')[1] == 'speech':
logger.notice(f"Running on Battleship {hostname}")
self.exp.log_other('jobid', int(os.getenv('SLURM_JOBID')))
else:
logger.notice(f"Running on {hostname}")
def log_config(self, config):
#NOTE: depth at most 2
for block in config:
for n, p in config[block].items():
if isinstance(p, dict):
self.exp.log_parameters(p, prefix=f'{block}-{n}')
else:
self.exp.log_parameter(f'{block}-{n}', p)
def set_status(self, status):
self.exp.log_other('status', status)
def step(self, n=1):
self.global_step += n
def set_step(self, global_step=1):
self.global_step = global_step
def log_info(self, prefix, info):
self.exp.log_metrics({k: float(v)
for k, v in info.items()},
prefix=prefix,
step=self.global_step)
def log_other(self, name, value):
self.exp.log_metric(name, value, step=self.global_step)
def log_step(self):
self.exp.log_other('step', self.global_step)
def add_figure(self, fig_name, data):
self.exp.log_figure(figure_name=fig_name,
figure=data,
step=self.global_step)
def check(self):
if not self.exp.alive:
logger.warning("Comet logging stopped")
distrfracs = np.array(distrfracs)
order = np.argsort(distrfracs)
# apply sort order
distrfracs = distrfracs[order]
testlasts = np.array(testlasts)[order]
trainlasts = np.array(trainlasts)[order]
radmeans = np.array(radmeans)[order]
# plot train and test accuracy
plt.fill_between(np.log10(distrfracs),
trainlasts.min(axis=1) - 5e-3,
trainlasts.max(axis=1) + 0e-3,
label='train')
plt.fill_between(np.log10(distrfracs),
testlasts.min(axis=1) - 0e-3,
testlasts.max(axis=1) + 5e-3,
label='test')
xlabel('poison fraction (log)')
ylabel('accuracy')
legend(loc='lower left', frameon=False)
print(experiment.log_figure()['web'])
# plot volume (HWHM)
plt.clf()
plt.fill_between(np.log10(distrfracs), radmeans.min(axis=1),
radmeans.max(axis=1))
xlabel('poison fraction (log)')
ylabel('half-width-half-min')
print(experiment.log_figure()['web'])
for i in range(hyper_params["N_OBS"]):
pval_selected = pval_sort[i]
index_original = int(numpy.where(pval_selected == pval)[0])
if index_original in idx_noisy:
index.append(True)
else:
index.append(False)
x_line = numpy.arange(0, hyper_params["N_OBS"], step=1)
y_line = numpy.linspace(0, 1, hyper_params["N_OBS"])
y_adj = numpy.arange(
0, hyper_params["N_OBS"], step=1
) / hyper_params["N_OBS"] * 0.05 # 0.05 means the alpha value of my test
zoom = 40 # nb of points to zoom
fig, (ax1, ax2) = plt.subplots(2, 1)
ax1.scatter(numpy.arange(0, len(pval), 1), pval_sort, c=index)
ax1.plot(x_line, y_line, color="green")
ax1.plot(x_line, y_adj, color="red")
ax1.set_title('Entire dataset')
ax1.set_xticklabels([])
ax2.scatter(numpy.arange(0, zoom, 1), pval_sort[0:zoom], c=index[0:zoom])
ax2.plot(x_line[0:zoom], y_line[0:zoom], color="green")
ax2.plot(x_line[0:zoom], y_adj[0:zoom], color="red")
ax2.set_title('Zoomed in')
ax2.set_xticklabels([])
experiment.log_figure(figure_name="chi2_test", figure=fig, overwrite=True)
plt.show()
best_train_acc = train_acc
if (test_acc > best_test_acc):
best_test_acc = test_acc
experiment.log_metric("best_train_acc", best_train_acc, epoch=epoch + 1)
experiment.log_metric("best_test_acc", best_test_acc, epoch=epoch + 1)
experiment.log_metric("train_acc", train_acc, epoch=epoch + 1)
experiment.log_metric("test_acc", test_acc, epoch=epoch + 1)
plt.plot(training_loss_list, color='blue', label='Training')
plt.plot(testing_loss_list, color='red', label='Testing', alpha=.5)
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.title('Loss plot')
plt.legend()
plt.savefig("./loss_plot_" + model_name + ".png", format='png')
experiment.log_figure(figure=plt, figure_name='loss_plot', overwrite=True)
plt.close()
plt.plot(training_acc_list, color='blue', label='Training')
plt.plot(testing_acc_list, color='red', label='Testing', alpha=.5)
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.title('Accuracy plot')
plt.legend()
plt.savefig("./accuracy_plot_" + model_name + ".png", format='png')
experiment.log_figure(figure=plt,
figure_name='accuracy_plot',
overwrite=True)
plt.close()
melted = merged_df.melt(id_vars = ['AID','Classifier','Iteration Number','Embedding','test_train','Percent_lib_scanned'],var_name = 'Metric',value_name='Score')
mcc_auc_plot = melted[(melted['Metric'].isin(['auc','mcc']))&
(melted['test_train']=='test')]
active_find_plot = melted[(melted['Metric'].isin(['Percent Active Found','Percent Batch Pred Active']))&
(melted['test_train']=='test')]
prec_rec_plot = melted[(melted['Metric'].isin(['prec_Active','rec_Active']))&
(melted['test_train']=='test')]
mcc_auc_plot.Score = mcc_auc_plot.Score.astype(float)
active_find_plot.Score = active_find_plot.Score.astype(float)
prec_rec_plot.Score = prec_rec_plot.Score.astype(float)
#%%
'''Plot 3 figures: AUC/MCC,%found/totalfound,recall/precision @0.5'''
g = sns.relplot(x="Iteration Number", y="Score", hue='Classifier',style="Metric", col="AID", col_wrap=3, data=mcc_auc_plot,kind='line',legend='full',markers= True,ci = None )
exp.log_figure()
g = sns.relplot(x="Iteration Number", y="Score", hue='Classifier',style="Metric", col="AID", col_wrap=3, data=active_find_plot,kind='line',legend='full',markers= True,ci = None )
exp.log_figure()
g = sns.relplot(x="Iteration Number", y="Score", hue='Classifier',style="Metric", col="AID", col_wrap=3, data=prec_rec_plot,kind='line',legend='full',markers= True,ci = None )
exp.log_figure()
g = sns.relplot(x="Percent_lib_scanned", y="Score", hue='Classifier',style="Metric", col="AID", col_wrap=3, data=active_find_plot,kind='line',legend='full',markers= True,ci = None )
exp.log_figure()
'''Plot prec/recall curves for all points '''
df = merged_df[merged_df['test_train']=='test']
# Initialize the figure
fig, axs = plt.subplots(9, 10)
plt.style.use('seaborn-darkgrid')
# create an aid dict
AID_list =['AID_1345083','AID_624255','AID_449739','AID_995','AID_938','AID_628','AID_596','AID_893','AID_894']
ax1.plot(x_line, y_adj, color="red")
ax1.set_title(
f'Entire test dataset with {int(hyper_params["TEST_NOISE"] * 100)}% of noise'
)
ax1.set_xticklabels([])
ax2.scatter(numpy.arange(0, zoom, 1),
pval[pval_order][0:zoom],
c=index[pval_order].reshape(-1)[0:zoom])
ax2.plot(x_line[0:zoom], y_line[0:zoom], color="green")
ax2.plot(x_line[0:zoom], y_adj[0:zoom], color="red")
ax2.set_title('Zoomed in')
ax2.set_xticklabels([])
experiment.log_figure(figure_name="empirical_test_hypothesis",
figure=fig,
overwrite=True)
plt.show()
# Compute some stats
precision, recall = test_performances(pval, index, hyper_params["ALPHA"])
print(f"Precision: {precision}")
print(f"Recall: {recall}")
experiment.log_metric("precision", precision)
experiment.log_metric("recall", recall)
# Show some examples
fig, axs = plt.subplots(5, 5)
fig.tight_layout()
axs = axs.ravel()
def comet_Fold(save_path, embedding_type, model_type, bin_labels):
from comet_ml import Experiment
exp = Experiment(api_key="sqMrI9jc8kzJYobRXRuptF5Tj",
project_name="80_10_baseline",
workspace="gdreiman1",
disabled=False)
exp.log_code = True
import warnings
warnings.filterwarnings('ignore')
import pickle
import pandas as pd
import numpy as np
import sklearn as sklearn
from sklearn.metrics import precision_recall_fscore_support as prf
from sklearn.linear_model import SGDClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import StandardScaler, LabelEncoder
import matplotlib.pyplot as plt
import seaborn as sns
'''Comet Saving Zone'''
def comet_addtional_info(exp, save_path, metrics_dict, X_test, y_test,
embedding_type, model_type):
#get AID number
import ntpath
#get base file name
folder, base = ntpath.split(save_path)
#split file name at second _ assumes file save in AID_xxx_endinfo.pkl
AID, _, end_info = base.rpartition('_')
exp.add_tag(AID)
#save data location, AID info, and version info
exp.log_dataset_info(name=AID, version=end_info, path=save_path)
#save some informatvie tags:
tags = [AID, end_info, model_type]
exp.add_tags(tags)
exp.add_tag(embedding_type)
#save metrics_dict in data_folder with comet experiement number associated
exp_num = exp.get_key()
model_save = folder + '\\' + model_type + '_' + exp_num + 'metrics_dict.pkl'
pickle_on = open(model_save, 'wb')
pickle.dump(metrics_dict, pickle_on)
pickle_on.close()
#log trained model location
exp.log_other('Metrics Dict Path', model_save)
#tell comet that the experiement is over
exp.end()
def get_Scaled_Data(train_ind, test_ind, X_mfp, activity_table, labels,
bin_labels):
#get start and end index for molchars
MC_start = activity_table.columns.get_loc('Chi0')
#need to add 1 bc exclusive indexing
MC_end = activity_table.columns.get_loc('VSA_EState9') + 1
# standardize data
scaler = StandardScaler(copy=False)
#return requested datatype
if embedding_type == 'MFPMolChars':
X_train_molchars_std = scaler.fit_transform(
np.array(activity_table.iloc[train_ind,
MC_start:MC_end]).astype(float))
X_test_molchars_std = scaler.transform(
np.array(activity_table.iloc[test_ind,
MC_start:MC_end]).astype(float))
X_train = np.concatenate(
(X_mfp[train_ind, :], X_train_molchars_std), axis=1)
X_test = np.concatenate((X_mfp[test_ind, :], X_test_molchars_std),
axis=1)
elif embedding_type == 'MFP':
X_train = X_mfp[train_ind, :]
X_test = X_mfp[test_ind, :]
elif embedding_type == 'MolChars':
X_train_molchars_std = scaler.fit_transform(
np.array(activity_table.iloc[train_ind,
MC_start:MC_end]).astype(float))
X_test_molchars_std = scaler.transform(
np.array(activity_table.iloc[test_ind,
MC_start:MC_end]).astype(float))
X_train = X_train_molchars_std
X_test = X_test_molchars_std
y_train = labels[train_ind]
y_test = labels[test_ind]
#remapping active to 1 and everything else to zero
bin_y_train, bin_y_test = np.array([
1 if x == 0 else 0 for x in y_train
]), np.array([1 if x == 0 else 0 for x in y_test])
if bin_labels == True:
y_test = bin_y_test
y_train = bin_y_train
return X_train, X_test, y_train, y_test
def train_SVM(X_train, X_test, y_train, y_test, split_ID):
sgd_linear_SVM = SGDClassifier(loss='hinge',
penalty='l2',
alpha=0.0001,
l1_ratio=0.15,
fit_intercept=True,
max_iter=500000,
tol=0.001,
shuffle=True,
verbose=0,
epsilon=0.1,
n_jobs=-1,
random_state=None,
learning_rate='optimal',
eta0=0.0,
power_t=0.5,
early_stopping=False,
validation_fraction=0.1,
n_iter_no_change=5,
class_weight='balanced',
warm_start=False,
average=False)
sgd_linear_SVM_model = sgd_linear_SVM.fit(X_train, y_train)
sgd_lSVM_preds = sgd_linear_SVM_model.predict(X_test)
prec, rec, f_1, supp = prf(y_test, sgd_lSVM_preds, average=None)
class_rep = sklearn.metrics.classification_report(
y_test, sgd_lSVM_preds)
exp.log_other('Classification Report' + split_ID, class_rep)
mcc = sklearn.metrics.matthews_corrcoef(y_test, sgd_lSVM_preds)
#if first iteration, report model parameters to comet
if split_ID == '0':
exp.log_parameters(sgd_linear_SVM_model.get_params())
return prec, rec, f_1, supp, mcc
def train_RF(X_train, X_test, y_train, y_test, split_ID):
rf = RandomForestClassifier(n_estimators=100,
random_state=2562,
class_weight="balanced_subsample",
n_jobs=-1)
rand_for = rf.fit(X_train, y_train)
rf_preds = rand_for.predict(X_test)
prec, rec, f_1, supp = prf(y_test, rf_preds, average=None)
class_rep = sklearn.metrics.classification_report(y_test, rf_preds)
exp.log_other('Classification Report' + split_ID, class_rep)
mcc = sklearn.metrics.matthews_corrcoef(y_test, rf_preds)
#if first iteration, report model parameters to comet
if split_ID == '0':
exp.log_parameters(rand_for.get_params())
return prec, rec, f_1, supp, mcc
def train_LGBM(X_train, X_test, y_train, y_test, split_ID):
import lightgbm as lgb
#make model class
lgbm_model = lgb.LGBMClassifier(boosting_type='gbdt',
num_leaves=31,
max_depth=-1,
learning_rate=0.1,
n_estimators=500,
subsample_for_bin=200000,
objective='binary',
is_unbalance=True,
min_split_gain=0.0,
min_child_weight=0.001,
min_child_samples=20,
subsample=1.0,
subsample_freq=0,
colsample_bytree=1.0,
reg_alpha=0.0,
reg_lambda=0.0,
random_state=None,
n_jobs=-1,
silent=True,
importance_type='split')
#train model
lgbm = lgbm_model.fit(X_train, y_train)
lgbm_preds = lgbm.predict(X_test)
prec, rec, f_1, supp = prf(y_test, lgbm_preds, average=None)
class_rep = sklearn.metrics.classification_report(y_test, lgbm_preds)
exp.log_other('Classification Report' + split_ID, class_rep)
mcc = sklearn.metrics.matthews_corrcoef(y_test, lgbm_preds)
#if first iteration, report model parameters to comet
if split_ID == '0':
exp.log_parameters(lgbm.get_params())
return prec, rec, f_1, supp, mcc
#from https://stackoverflow.com/questions/6027558/flatten-nested-dictionaries-compressing-keys
import collections
def flatten(d, parent_key='', sep='_'):
items = []
for k, v in d.items():
new_key = parent_key + sep + k if parent_key else k
if isinstance(v, collections.MutableMapping):
items.extend(flatten(v, new_key, sep=sep).items())
else:
items.append((new_key, v))
return dict(items)
#get data cleaned
pickle_off = open(save_path, 'rb')
activity_table = pickle.load(pickle_off)
pickle_off.close()
#get length of MFP
fp_length = len(activity_table.iloc[5]['MFP'])
#reshape mfp
X_mfp = np.concatenate(np.array(activity_table['MFP'])).ravel()
X_mfp = X_mfp.reshape((-1, fp_length))
le = LabelEncoder()
labels = le.fit_transform(activity_table['PUBCHEM_ACTIVITY_OUTCOME'])
#split data:
from sklearn.model_selection import StratifiedShuffleSplit
#this is outer 5fold cross validation i.e. 80/20 split
big_splitter = StratifiedShuffleSplit(n_splits=5,
test_size=0.2,
random_state=2562)
#inner replicateing the start with 10% of data (or 12.5% of 80% intial split)
little_splitter = StratifiedShuffleSplit(n_splits=8,
test_size=0.2,
train_size=0.125,
random_state=2562)
#this holds all the metrics values that will be stored in comet
metric_dict = {}
metric_names = [
'prec_Inactive', 'prec_Active', 'rec_Inactive', 'rec_Active',
'f_1_Inactive', 'f_1_Active', 'supp_Inactive', 'supp_Active', 'mcc'
]
def calc_and_save_metrics(X_train, X_test, y_train, y_test, split_index,
metric_names, metric_dict_list, map_name):
'''Takes in test and train data + labels, computes metrics and saves them
as a dict inside of the provided list. Returns this list.'''
prec, rec, f_1, supp, mcc = classifier_train(X_train, X_test, y_train,
y_test, split_index)
results_array = np.concatenate((prec, rec, f_1, supp)).tolist() + [mcc]
metric_dict_list.append(
dict(
zip(['ID', 'Split Info'] + metric_names,
[split_index, map_name] + results_array)))
return metric_dict_list
#determine model type
classifier_dict = {'SVM': train_SVM, 'RF': train_RF}
#set dummy variable to func that trains specified model
classifier_train = classifier_dict[model_type]
metric_dict_list = []
#using labels as a dummy for X
for split_num, [train_ind,
test_ind] in enumerate(big_splitter.split(labels, labels)):
#indexs which split the data comes from X.X ie big.little
split_index = str(split_num)
map_name = 'Split' + split_index + ' 80% train'
#get test/train index
X_train, X_test, y_train, y_test = get_Scaled_Data(
train_ind, test_ind, X_mfp, activity_table, labels, bin_labels)
#train model and get back classwise metrics
metric_dict_list = calc_and_save_metrics(X_train, X_test, y_train,
y_test, split_index,
metric_names,
metric_dict_list, map_name)
#add split_index to metric names this assumes 0 = inactive 1 = active!!
for little_split_num, [little_train_ind, little_test_ind] in enumerate(
little_splitter.split(labels[train_ind], labels[train_ind])):
split_index = str(split_num) + '.' + str(little_split_num)
#get test/train index
X_train, X_test, y_train, y_test = get_Scaled_Data(
little_train_ind, test_ind, X_mfp, activity_table, labels,
bin_labels)
map_name = 'Split' + str(split_num) + ' 10% train'
#train model and get back classwise metrics
#check if train_split contains both postive and negative labels
if len(set(y_train)) == 2:
metric_dict_list = calc_and_save_metrics(
X_train, X_test, y_train, y_test, split_index,
metric_names, metric_dict_list, map_name)
# now convert metric_dict_list to df:
metrics_df = pd.DataFrame(metric_dict_list)
#set Split_ID to index
metrics_df.set_index('ID', drop=True, inplace=True)
#now plot all the columns
#first make a new df column to ID things as either split
cols_to_plot = list(metrics_df.columns.values)[1:]
#turn off plotting
plt.ioff()
for metric in cols_to_plot:
#make sns boxplot
ax = sns.boxplot(x='Split Info', y=metric, data=metrics_df)
ax.set_xticklabels(ax.get_xticklabels(), rotation=30)
plt.tight_layout()
#log the plot
exp.log_figure()
''' now we're going to go through and calculate means and stds for 3 diff groups
1) the 5 80% train runs
2) the 5 sets of 8 10% runs
3) the 40 total 10% runs
we save each in a list as a pd Series with a name explaining the contents'''
avg_std_list = []
#get 5 80% mean and std
avg_std_list.append(
(metrics_df[~metrics_df.index.str.contains(r'\.')].mean(
axis=0, numeric_only=True)).rename('50_80_mean'))
avg_std_list.append((metrics_df[~metrics_df.index.str.contains(r'\.')].std(
axis=0, numeric_only=True)).rename('50_80_std'))
#get 40 10% mean and std
avg_std_list.append((metrics_df[metrics_df.index.str.contains(r'\.')].mean(
axis=0, numeric_only=True)).rename('40_10_mean'))
avg_std_list.append((metrics_df[metrics_df.index.str.contains(r'\.')].std(
axis=0, numeric_only=True)).rename('40_10_std'))
#get the splitwise 10% mean and std
for split_num in range(5):
id_split = str(split_num) + r'\.'
avg_std_list.append(
(metrics_df[metrics_df.index.str.contains(id_split)].mean(
axis=0,
numeric_only=True)).rename(str(split_num) + '_10_mean'))
avg_std_list.append(
(metrics_df[metrics_df.index.str.contains(id_split)].std(
axis=0, numeric_only=True)).rename(str(split_num) + '_10_std'))
#now add list of dicts of averages to metrics df
metrics_df = metrics_df.append(avg_std_list)
#convert metrics_df to metric dict and log it
metric_dict = metrics_df.to_dict('index')
exp.log_metrics(flatten(metric_dict))
#save metric_df to current folder
comet_addtional_info(exp, save_path, metrics_df, X_test, y_test,
embedding_type, model_type)
def train_cifar10(batch_size: int,
learning_rate: float,
epochs: int,
experiment: Experiment,
model: Sequential = get_model(),
initial_epoch: int = 0,
training_datagen: ImageDataGenerator = ImageDataGenerator(),
scheduler: Callable[[int], float] = None,
early_stopping_th: Optional[int] = 250,
data_portion: float = 1.0,
find_lr: bool = False) -> None:
preprocessing_fnc = training_datagen.preprocessing_function
name = experiment.get_key()
log_path, model_path = get_output_paths(name)
data = get_cifar10_data(data_portion=data_portion)
training_datagen.fit(data.x_train)
log_images(data.x_train, training_datagen, experiment)
log_input_images(data.x_train, data.y_train, training_datagen, experiment)
opt = Adam(lr=learning_rate)
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
log_model_plot(experiment, model)
csv_cb = CSVLogger(log_path)
keep_best_cb = KeepBest('val_acc')
callbacks = [csv_cb,
keep_best_cb] # [csv_cb, early_stopping_cb, keep_best_cb]
if early_stopping_th is not None:
early_stopping_cb = EarlyStopping('val_acc',
patience=early_stopping_th,
restore_best_weights=True,
verbose=2)
callbacks.append(early_stopping_cb)
if scheduler is not None:
scheduler.experiment_log(experiment=experiment,
epochs=list(range(epochs)))
callbacks.append(LearningRateScheduler(scheduler))
if find_lr:
lrf = LearningRateFinder(model=model)
lrf.lrMult = (10e-1 / learning_rate)**(
1.0 / (epochs * len(data.x_train) / batch_size))
callbacks = [
LambdaCallback(
on_batch_end=lambda batch, logs: lrf.on_batch_end(batch, logs))
]
model.fit_generator(training_datagen.flow(data.x_train,
data.y_train,
batch_size=batch_size),
steps_per_epoch=len(data.x_train) / batch_size,
epochs=epochs,
validation_data=(preprocessing_fnc(data.x_dev),
data.y_dev),
shuffle=True,
callbacks=callbacks,
verbose=2,
initial_epoch=initial_epoch)
model.save(model_path)
experiment.log_asset(model_path)
experiment.log_asset(log_path)
if find_lr:
experiment.log_figure('lr vs acc', lrf.plot_loss())
log_final_metrics(experiment, model, data, preprocessing_fnc)
def run_exp(self):
''' Run the experiment on given number of pilots. '''
# Pre-train
pretrain_weights_filename = '{}{}.h5'.format(self.weights_savename_prefix, '_all')
if self.pretraining:
print('Pretraining...')
_ = self.pretrain(pretrain_weights_filename, self.patience)
for pilot_idx in range(1, self.n_pilots + 1):
if self.log_pilots:
experiment = Experiment(api_key='cSZq9kuH2I87ezvm2dEWTx6op', project_name='pilot{}'.format(pilot_idx), log_code=False, auto_param_logging=False)
else:
experiment = None
# Load pilot data
X_train, y_train, X_valid, y_valid, X_test, y_test = self.load_data(pilot_idx, valid_ratio=0.2)
# Construct model & load pre-trained weights if available
weights_filename = '{}{}.h5'.format(self.weights_savename_prefix, pilot_idx)
self.model = self.build_model(load_weights=True if self.pretraining else False,
weights_filename=pretrain_weights_filename)
# Train
print('First phase training - Pilot {}'.format(pilot_idx))
hist, best_epoch = self.train(X_train, y_train, X_valid, y_valid,
save_filename='{}{}.h5'.format(self.weights_savename_prefix, pilot_idx),
patience=self.patience, experiment=experiment)
self.histories.append(hist)
# Test (before extra training)
self.test(pilot_idx, weights_filename, X_train, y_train, X_test, y_test, False)
# Extra-train
if self.train_after_es:
hist_es = self.extra_train(pilot_idx, X_valid, y_valid, weights_filename, hist.history['loss'][best_epoch], self.max_after_es_epochs) # New
self.histories_es.append(hist_es)
# Test (after extra training)
self.test(pilot_idx, weights_filename, X_train, y_train, X_test, y_test, True)
if self.log_pilots:
experiment.log_metrics({'Test accuracy' : self.test_score['accuracy'][-1]})
# Get t-SNE from intermediary outputs
layer_idxs = self.tsne_layer_idxs
get_output_functions = [ K.function([self.model.layers[0].input], [self.model.layers[idx].output]) for idx in layer_idxs]
# Training dataset
layer_outputs = [ get_output([X_train])[0] for get_output in get_output_functions ]
[ experiment.log_figure(figure_name='tsne_raw_train{}_layer{}'.format(pilot_idx, layer_idxs[idx]),
figure=tsne_plot(layer_outputs[idx], y_train, 20, title="t-SNE - Pilot {} - Layer {} (train)".format(pilot_idx, layer_idxs[idx])))
for idx in range(len(layer_idxs)) ]
# Testing dataset
layer_outputs = [ get_output([X_test])[0] for get_output in get_output_functions ]
[ experiment.log_figure(figure_name='tsne_raw_test{}_layer{}'.format(pilot_idx, layer_idxs[idx]),
figure=tsne_plot(layer_outputs[idx], y_test, 20, title="t-SNE - Pilot {} - Layer {} (test)".format(pilot_idx, layer_idxs[idx])))
for idx in range(len(layer_idxs)) ]
plt.close('all')
experiment.end()
# Export logs to Comet.ml
if self.comet_log:
self.log()
return self.test_score
class Logger:
"""
Logs/plots results to comet.
Args:
exp_config (dict): experiment configuration hyperparameters
model_config (dict): model configuration hyperparameters
data_config (dict): data configuration hyperparameters
"""
def __init__(self, exp_config, model_config, data_config):
self.experiment = Experiment(**exp_config['comet_config'])
self.experiment.disable_mp()
self._log_hyper_params(exp_config, model_config, data_config)
self._epoch = 0
def _log_hyper_params(self, exp_config, model_config, data_config):
"""
Log the hyper-parameters for the experiment.
Args:
exp_config (dict): experiment configuration hyperparameters
model_config (dict): model configuration hyperparameters
data_config (dict): data configuration hyperparameters
"""
def flatten_arg_dict(arg_dict):
flat_dict = {}
for k, v in arg_dict.items():
if type(v) == dict:
flat_v = flatten_arg_dict(v)
for kk, vv in flat_v.items():
flat_dict[k + '_' + kk] = vv
else:
flat_dict[k] = v
return flat_dict
self.experiment.log_parameters(flatten_arg_dict(exp_config))
self.experiment.log_parameters(flatten_arg_dict(model_config))
self.experiment.log_parameters(flatten_arg_dict(data_config))
def log(self, results, train_val):
"""
Plot the results in comet.
Args:
results (dict): dictionary of metrics to plot
train_val (str): either 'train' or 'val'
"""
objectives, grads, params, images, metrics = results
for metric_name, metric in objectives.items():
self.experiment.log_metric(metric_name + '_' + train_val, metric,
self._epoch)
print(metric_name, ':', metric.item())
if train_val == 'train':
for grad_metric_name, grad_metric in grads.items():
self.experiment.log_metric('grads_' + grad_metric_name,
grad_metric, self._epoch)
for param_name, param in params.items():
self.experiment.log_metric(param_name + '_' + train_val, param,
self._epoch)
for image_name, imgs in images.items():
self.plot_images(imgs, image_name, train_val)
for metric_name, metric in metrics.items():
self.experiment.log_metric(metric_name + '_' + train_val, metric,
self._epoch)
if train_val == 'val':
self._epoch += 1
def plot_images(self, images, title, train_val):
"""
Plot a tensor of images.
Args:
images (torch.Tensor): a tensor of shape [steps, b, c, h, w]
title (str): title for the images, e.g. reconstructions
train_val (str): either 'train' or 'val'
"""
# add a channel dimension if necessary
if len(images.shape) == 4:
s, b, h, w = images.shape
images = images.view(s, b, 1, h, w)
s, b, c, h, w = images.shape
if b > 10:
images = images[:, :10]
# swap the steps and batch dimensions
images = images.transpose(0, 1).contiguous()
images = images.view(-1, c, h, w)
grid = make_grid(images.clamp(0, 1), nrow=s).numpy()
if c == 1:
grid = grid[0]
cmap = 'gray'
else:
grid = np.transpose(grid, (1, 2, 0))
cmap = None
plt.imshow(grid, cmap=cmap)
self.experiment.log_figure(figure=plt,
figure_name=title + '_' + train_val)
plt.close()
def save(self, model):
"""
Save the model weights in comet.
Args:
model (nn.Module): the model to be saved
"""
pass
# if i % 2000 == 0: # print every 2000 mini-batches
# print('[%d, %5d] loss: %.3f' %
# (epoch, i, running_loss / 2000))
# running_loss = 0.0
end_time = time.time()
# print("Time", end_time-start_time)
loss_per_epoch.append(np.mean(epoch_loss))
model_scheduler.step(np.mean(epoch_loss))
plt.plot(loss_per_epoch, color='red', label='Training')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.title('Loss plot per epoch')
plt.legend()
plt.savefig("./loss_per_epoch_plot_baseline.png", format='png')
experiment.log_figure(figure=plt,
figure_name='loss_per_epoch_plot_baseline',
overwrite=True)
plt.close()
if (epoch % 5 == 0):
# test
correct = 0
total = 0
test_loss = 0
with torch.no_grad():
for data in testloader:
inputs, labels = data[0].to(
device, non_blocking=True), data[1].to(device,
non_blocking=True)
outputs = net(inputs)
os.makedirs('pickle', exist_ok=True); pickle.dump(dw1, open(join('pickle', args.ckpt), 'wb'))
along = 'along_eigvec'
else:
dw1 = evaluator.get_random_dir()
along = 'along_random_'+str(args.seed)
# span
cfeed = args.span/2 * np.linspace(-1, 1, 30)
cfeed_enum = list(enumerate(cfeed)); random.shuffle(cfeed_enum) # shuffle order so we see plot shape sooner on comet
# loop over all points along surface direction
name = 'span_' + str(args.span) + '/' + basename(args.ckpt) + '/' + along # name of experiment
xent = np.zeros(len(cfeed))
weights = evaluator.get_weights()
for i, (idx, c) in enumerate(cfeed_enum):
perturbedWeights = [w + c * d1 for w, d1 in zip(weights, dw1)]
evaluator.assign_weights(perturbedWeights)
xent[idx], acc, _ = evaluator.eval()
experiment.log_metric(name, xent[idx], idx)
print('progress:', i + 1, 'of', len(cfeed_enum), '| time:', time())
# save plot data and log the figure
xent = np.reshape(np.array(xent), cfeed.shape)
plt.plot(cfeed, xent)
experiment.log_figure(name)
unique = utils.timenow()
pickle.dump((cfeed, xent), open(unique, 'wb'))
experiment.log_asset(file_path=unique, file_name=name+'.pkl')
def comet_lgbm(save_path):
from comet_ml import Experiment
exp = Experiment(api_key="sqMrI9jc8kzJYobRXRuptF5Tj",
project_name="baseline", workspace="gdreiman1")
exp.log_code = True
import pickle
import pandas as pd
import lightgbm as lgb
import numpy as np
import sklearn
import matplotlib.pyplot as plt
from sklearn.metrics import precision_recall_fscore_support as prf
#%%
def single_roc(y_preds,y_true):
from sklearn.metrics import roc_curve, auc,precision_recall_curve
fpr, tpr, _ = roc_curve(y_true, y_preds)
roc_auc = auc(fpr, tpr)
plt.figure()
lw = 2
plt.plot(fpr, tpr, color='darkorange',
lw=lw, label='ROC curve (area = %0.2f)' % roc_auc)
plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic example')
precision, recall, thresholds = precision_recall_curve(y_true, y_preds)
plt.plot(recall, precision, color='blue',
lw=lw, label='Precision vs Recall')
# show the plot
plt.legend(loc="lower right")
plt.show()
def multi_roc(y_preds,y_true,name,n_classes):
import collections
nested_dict = lambda: collections.defaultdict(nested_dict)
data_store = nested_dict()
from sklearn.metrics import roc_curve, auc
from scipy import interp
from itertools import cycle
lw = 2
name_store = ['Active', 'Inactive', 'Inconclusive']
fpr = dict()
tpr = dict()
roc_auc = dict()
for i in range(n_classes):
fpr[i], tpr[i], _ = roc_curve(y_true[:, i], y_preds[:, i])
roc_auc[i] = auc(fpr[i], tpr[i])
# Compute micro-average ROC curve and ROC area
fpr["micro"], tpr["micro"], _ = roc_curve(y_true[:, i].ravel(), y_preds[:, i].ravel())
roc_auc["micro"] = auc(fpr["micro"], tpr["micro"])
# Compute macro-average ROC curve and ROC area
# First aggregate all false positive rates
all_fpr = np.unique(np.concatenate([fpr[i] for i in range(n_classes)]))
# Then interpolate all ROC curves at this points
mean_tpr = np.zeros_like(all_fpr)
for i in range(n_classes):
mean_tpr += interp(all_fpr, fpr[i], tpr[i])
# Finally average it and compute AUC
mean_tpr /= n_classes
fpr["macro"] = all_fpr
tpr["macro"] = mean_tpr
roc_auc["macro"] = auc(fpr["macro"], tpr["macro"])
# Plot all ROC curves
plt.figure()
plt.plot(fpr["micro"], tpr["micro"],
label='micro-average ROC curve (area = {0:0.2f})'
''.format(roc_auc["micro"]),
color='deeppink', linestyle=':', linewidth=4)
plt.plot(fpr["macro"], tpr["macro"],
label='macro-average ROC curve (area = {0:0.2f})'
''.format(roc_auc["macro"]),
color='navy', linestyle=':', linewidth=4)
colors = cycle(['aqua', 'darkorange', 'cornflowerblue','green'])
for i, color in zip(range(n_classes), colors):
plt.plot(fpr[i], tpr[i], color=color, lw=lw,
label='ROC curve of '+ name_store[i]+'(area = {1:0.2f})'
''.format(i, roc_auc[i]))
plt.plot([0, 1], [0, 1], 'k--', lw=lw)
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
#plt.title('Multi-class ROC for '+name+' Split= '+str(count+1))
plt.title('Multi-class ROC for '+name)
plt.legend(loc="lower right")
#plt.show()
#%%
#save_path = r'C:\Users\gdrei\Dropbox\UCL\Thesis\May_13\AID_1345083_processed.pkl'
model_type = 'lgbm'
#get data cleaned
pickle_off = open(save_path,'rb')
activity_table=pickle.load(pickle_off)
pickle_off.close()
#get length of MFP
fp_length = len(activity_table.iloc[5]['MFP'])
from sklearn.preprocessing import StandardScaler, LabelEncoder
scaler = StandardScaler(copy = False)
le = LabelEncoder()
labels = le.fit_transform(activity_table['PUBCHEM_ACTIVITY_OUTCOME'])
#split data:
from sklearn.model_selection import StratifiedShuffleSplit
splitter = StratifiedShuffleSplit(n_splits=1, test_size=0.5, train_size=None, random_state=2562)
X_mfp = np.concatenate(np.array(activity_table['MFP'])).ravel()
X_mfp = X_mfp.reshape((-1,fp_length))
for train_ind, test_ind in splitter.split(X_mfp,labels):
# standardize data
X_train_molchars_std = scaler.fit_transform(np.array(activity_table.iloc[train_ind,4:]))
X_test_molchars_std = scaler.transform(np.array(activity_table.iloc[test_ind,4:]))
X_train = np.concatenate((X_mfp[train_ind,:],X_train_molchars_std),axis = 1)
X_test = np.concatenate((X_mfp[test_ind,:],X_test_molchars_std),axis = 1)
y_train = labels[train_ind]
y_test = labels[test_ind]
#X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(X,labels,test_size = .5, shuffle = True, stratify = labels, random_state = 2562)
bin_y_train, bin_y_test = [1 if x ==2 else x for x in y_train],[1 if x ==2 else x for x in y_test]
#do light gbm
#need to make a lib svm file
train_data = lgb.Dataset(X_train,label=y_train)
test_data = lgb.Dataset(X_test,label=y_test)
#make model class
lgbm_model = lgb.LGBMClassifier(boosting_type='gbdt', num_leaves=31, max_depth=-1, learning_rate=0.1, n_estimators=500, subsample_for_bin=200000,
objective='binary', is_unbalance=True, min_split_gain=0.0, min_child_weight=0.001, min_child_samples=20, subsample=1.0,
subsample_freq=0, colsample_bytree=1.0, reg_alpha=0.0, reg_lambda=0.0, random_state=None, n_jobs=-1, silent=True,
importance_type='split')
#train model
trained_mod = lgbm_model.fit(X_train,y_train)
#predict classes and class_probs
test_class_preds = lgbm_model.predict(X_test)
test_prob_preds = lgbm_model.predict_proba(X_test)
#calculate Class report
class_rep = sklearn.metrics.classification_report(y_test,test_class_preds)
print(class_rep)
if len(set(y_test)) == 2:
single_roc(test_prob_preds[:,1],y_test)
prec,rec,f_1,supp = prf(y_test, test_class_preds, average=None)
else:
from tensorflow.keras.utils import to_categorical
multi_roc(test_prob_preds,to_categorical(y_test),'',3)
prec,rec,f_1,supp = prf(y_test, test_class_preds, average=None)
#%%
'''Comet Saving Zone'''
#get AID number
import ntpath
#get base file name
folder,base = ntpath.split(save_path)
#split file name at second _ assumes file save in AID_xxx_endinfo.pkl
AID, _,end_info = base.rpartition('_')
#save data location, AID info, and version info
exp.log_dataset_info(name = AID, version = end_info, path = save_path)
#save model params
exp.log_parameters(trained_mod.get_params())
#save metrics report to comet
if len(f_1) == 2:
for i,name in enumerate(['Active','Inactive']):
exp.log_metric('f1 class '+name, f_1[i])
exp.log_metric('Recall class'+name,rec[i])
exp.log_metric('Precision class'+name, prec[i])
else:
for i,name in enumerate(['Active','Inconclusive','Inactive']):
exp.log_metric('f1 class '+str(i), f_1[i])
exp.log_metric('Recall class'+str(i),rec[i])
exp.log_metric('Precision class'+str(i), prec[i])
#exp.log_metric('f1 class '+str(i), f_1[i])
#exp.log_metric('Recall class'+str(i),rec[i])
#exp.log_metric('Precision class'+str(i), prec[i])
exp.log_other('Classification Report',class_rep)
#save model in data_folder with comet experiement number associated
exp_num = exp.get_key()
model_save = folder+'\\'+model_type+'_'+exp_num+'.pkl'
pickle_on = open(model_save,'wb')
pickle.dump(trained_mod,pickle_on)
pickle_on.close()
#log trained model location
exp.log_other('Trained Model Path',model_save)
#save some informatvie tags:
tags = [AID,end_info,model_type]
exp.add_tags(tags)
#save ROC curve
exp.log_figure(figure_name = 'ROC-Pres/Recall',figure=plt)
plt.show()
#tell comet that the experiement is over
exp.end()
class experiment_logger:
'''
Interface for logging experiments on neptune, comet, or both.
Args: log_backend, project_name)
Other backends may also be added in the future
Currently defined methods:
add_params:
add_tags:
log_text: strings
log_metrics: numerical values
log_figure: pyplot figures
stop: end logging and close connection
'''
def __init__(self, log_backend, project_name):
'''
Parameters
----------
log_backend : STR
One of 'comet', 'neptune', 'all'
project_name : STR
one of available proyects ('yeast', 'jersey', 'wheat', 'debug', etc)
Returns
-------
None.
'''
self.proj_name = project_name
self.backend = log_backend
#Bool indicating wether neptune logging is enabled
self.neptune = log_backend=='neptune' or log_backend=='all'
#Bool indicating wether comet logging is enabled
self.comet = log_backend=='comet' or log_backend=='all'
if self.neptune:
neptune.init("dna-i/"+project_name,
api_token='eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vdWkubmVwdHVuZS5haSIsImFwaV91cmwiOiJodHRwczovL3VpLm5lcHR1bmUuYWkiLCJhcGlfa2V5IjoiMWYzMzhjMjItYjczNC00NzZhLWFlZTYtOTI2NzE5MzUwZmNkIn0=')
print("logging experiments on neptune project "+project_name)
neptune.create_experiment()
if self.comet:
self.comet_experiment = Experiment(api_key="V0OXnWOi4KVNS4OkwLjdnxSgK",
project_name=project_name, workspace="dna-i")
print("logging experiments on comet project "+project_name)
if not (self.neptune or self.comet):
raise ValueError('Logging Backend NOT Available')
def add_params(self, params, step=None ):
'''
Adds parameters to experiment log
Parameters
----------
params : Dict
Key-Value pairs
Returns
-------
None.
'''
if self.neptune:
for key, value in params.items():
neptune.set_property(key, value)
if step is not None:
neptune.set_property('step', step)
if self.comet:
self.comet_experiment.log_parameters(params,step=step)
def add_tags(self, tags):
'''
Adds parameters to experiment log
Parameters
----------
params : tags
list of tags (strings)
e.g.: ['tag1', 'tag2']
Returns
-------
None.
'''
if self.neptune:
neptune.append_tag(tags)
if self.comet:
self.comet_experiment.add_tags(tags)
def log_metrics(self, name, value, epoch=None):
'''
Logging pointwise metrics
Parameters
----------
name : STR
Metric key
value : Float/Integer/(Boolean/String)
Comet also allows Boolean/string
Tuples are lallowed
epoch: (OPT) INT
Epoch - or anything used as x axis when plotting metrics
Returns
-------
None.
'''
if self.neptune:
try:
if epoch is not None:
if type(value) is tuple:
print("Logging tuple as r and p-value")
for val, n in zip(value, [" (r)", " (p-val)"]):
neptune.log_metric(name + n,epoch,y=val)
else:
neptune.log_metric(name, epoch, y=value)
else:
if type(value) is tuple:
print("Logging tuple as r and p-value")
for val, n in zip(value, [" (r)", " (p-val)"]):
neptune.log_metric(name+n, val)
else:
neptune.log_metric(name, value)
except:
print("Metric type {} not supported by neptune.".format(type(value)))
print("logging as text")
self.log_text( "{}".format(value), key=name)
if self.comet:
try:
if epoch is not None:
if type(value) is tuple:
print("Logging tuple as r and p-value")
for val, n in zip(value, [" (r)", " (p-val)"]):
self.comet_experiment.log_metric(name+n, val, step=int(epoch))
else:
self.comet_experiment.log_metric(name, value, epoch=epoch)
else:
if type(value) is tuple:
print("Logging tuple as r and p-value")
for val, n in zip(value, [" (r)", " (p-val)"]):
self.comet_experiment.log_metric(name+n, val)
else:
self.comet_experiment.log_metric(name, value)
except:
print("Metric type {} not supported by comet.".format(type(value)))
if type(value) is tuple:
print("Logging tuple as x-y pairs")
for idx, val in enumerate(value):
self.comet_experiment.log_metric(name, val, epoch=idx)
else:
print("Logging as other.")
self.comet_experiment.log_other(name, value)
def log_text(self, string, key=None, epoch=None):
'''
Logs text strings
Parameters
----------
string : STR
text to log
key: STR
log_name needed for Neptune strings
epoch: INT
epoch or any other index
Returns
-------
None.
'''
if self.neptune:
if type(string) is str:
if key is None:
print('Neptune log_name needed for logging text')
print('Using a dummy name: text')
neptune.log_text('text', string)
if epoch is None:
neptune.log_text(key, string)
else:
neptune.log_text(key, epoch, y=string)
else:
print("Wrong type: logging text must be a string")
if self.comet:
if type(string) is str:
if key is not None:
print("Commet text logging does not support keys, prepending it to text")
string = key+ ', '+string
if epoch is None:
self.comet_experiment.log_text(string)
else:
self.comet_experiment.log_text(string, step=epoch)
else:
print("Wrong type: logging text must be a string")
def log_figure(self, figure=None, figure_name=None, step=None):
'''
Logs pyplot figure
Parameters
----------
figure : pyplot figure, optional in comet mandatory in neptune.
The default is None, uses global pyplot figure.
figure_name : STR, optional in comet mandatory in neptune.
The default is None.
step : INT, optional
An index. The default is None.
Returns
-------
None.
'''
if self.neptune:
if figure is not None:
if figure_name is None:
print("Figure name must be given to neptune logger")
print("Using dummy name: figure")
figure_name = 'figure'
if step is None:
neptune.log_image(figure_name, figure)
else:
neptune.log_image(figure_name, step, y=figure)
else:
print("A figure must be passed to neptune logger")
if self.comet:
self.comet_experiment.log_figure(figure_name=figure_name, figure=figure, step=step)
def stop(self):
if self.neptune:
neptune.stop()
if self.comet:
self.comet_experiment.end()
def add_table(self, filename, tabular_data=None, headers=False):
self.comet_experiment.log_table(filename, tabular_data, headers)
def log_image(self, image=None, figure_name=None, step=None):
'''
Logs pyplot figure
Parameters
----------
figure : pyplot figure, optional in comet mandatory in neptune.
The default is None, uses global pyplot figure.
figure_name : STR, optional in comet mandatory in neptune.
The default is None.
step : INT, optional
An index. The default is None.
Returns
-------
None.
'''
self.log_image(image, name=figure_name, overwrite=False, image_format="png", image_scale=1.0, \
image_shape=None, image_colormap=None, image_minmax=None, image_channels="last", \
copy_to_tmp=True, step=step)
def log_hist3d(self, values=None, figure_name=None, step=None):
'''
Logs pyplot figure
Parameters
----------
figure : pyplot figure, optional in comet mandatory in neptune.
The default is None, uses global pyplot figure.
figure_name : STR, optional in comet mandatory in neptune.
The default is None.
step : INT, optional
An index. The default is None.
Returns
-------
None.
'''
if self.neptune:
print("not implemented")
if self.comet:
self.comet_experiment.log_histogram_3d(values, name=figure_name, step=step)
def log_table(self, name=None, data=None, headers=False):
'''
Parameters
----------
name : str
Table name
data : array, list
headers : TYPE, optional
wether to use headers
Returns
-------
None.
'''
self.comet_experiment.log_table(name+'.csv', tabular_data= data, headers = headers )
# experiment.log_metric("d_loss", D_l)
# experiment.log_metric("g_loss", G_l)
# experiment.log_metric("current_resolution", current_resolution)
# experiment.log_metric("current_mode", 0 if current_mode == 'train' else 1)
if np.isnan(D_l) or np.isnan(G_l):
print('loss is NaN.')
exit()
if step % 1000 == 0:
print('epoch: {} step: {} G_loss: {} D_loss: {}'.format(epoch, step, G_l, D_l))
# Save figure
sampled_images = session.run([samples_for_all_resolutions[sizes.index(current_resolution)]])[0]
plot = plt.figure(figsize=(20, 10))
for m in range(3):
plt.subplot(1, 3, m + 1)
plt.imshow(sampled_images[m])
plt.savefig('./progress_images/epoch_{0}_{1}_{2}x{2}'.format(epoch, current_mode, current_resolution))
experiment.log_figure(figure_name='epoch_{0}_{1}_{2}x{2}'.format(epoch, current_mode, current_resolution))
plt.close()
if step % 5000 == 0:
save_path = saver.save(session, './checkpoints/model.ckpt', global_step=total_images_looked_at)
except tf.errors.OutOfRangeError:
pass
## embedding tsne visualizations
# country
categembs = sess.run(model.categembs)
with open('categembs.pkl', 'wb') as f:
pickle.dump(categembs, f)
with open('categembs.pkl', 'rb') as f:
categembs = pickle.load(f)
embs = categembs['country']
tsnes = TSNE(n_components=2).fit_transform(embs)
plt.figure(figsize=(8, 8))
plt.plot(*tsnes.T, '.')
for i, tsne in enumerate(tsnes):
plt.text(*tsne, ' ' + model.categs['country'][i], fontsize=8)
plt.gca().axis('equal')
plt.tight_layout()
print(experiment.log_figure(step=epoch))
# AS
embs = categembs['as'][:300]
tsnes = TSNE(n_components=2).fit_transform(embs)
plt.figure(figsize=(16, 16))
plt.plot(*tsnes.T, '.')
for i, tsne in enumerate(tsnes):
plt.text(*tsne, ' ' + model.categs['as'][i][3:23], fontsize=8)
plt.gca().axis('equal')
plt.tight_layout()
print(experiment.log_figure(step=epoch))
# subnet
embs = categembs['subnet']
tsnes = TSNE(n_components=2).fit_transform(embs)
progbar.add(valid_inc, [('Train Loss', metrics['train_loss']),
('Validation Loss', metrics['valid_loss']),
('Time (s)', step_time)])
#Plot on Comet
experiment.log_metrics(metrics, step=t)
# Plot on WandB
wandb.log(metrics, step=t)
if (t + 1) % save_inc == 0:
trainer.save_weights(model_path,
run_id=wandb.run.id,
experiment_key=experiment.get_key())
if not args.gcbc and not args.images:
z_enc, z_plan = produce_cluster_fig(next(plotting_dataset),
encoder,
planner,
TEST_DATA_PATHS[0],
num_take=dl.batch_size // 4)
#Comet
experiment.log_figure('z_enc', z_enc, step=t)
experiment.log_figure('z_plan', z_plan, step=t)
# WandB
wandb.log({'z_enc': z_enc, 'z_plan': z_plan}, step=t)
#latent_fig = project_enc_and_plan(ze, zp)
#latent_img = plot_to_image(latent_fig)
t += 1
input_shape=(1,))
model.add(rbflayer)
model.add(Dense(1))
model.compile(loss='mse',
optimizer=_OPTIMIZER)
model.fit(oDataSet.attributes[oData.Training_indexes],
oDataSet.labels[oData.Training_indexes],
batch_size=50,
epochs=epochs,
verbose=1)
y_pred = model.predict(oDataSet.attributes[oData.Testing_indexes])
y_true = oDataSet.labels[oData.Testing_indexes]
plt.plot(model.history.history['loss'])
experiment.log_figure(figure=plt, figure_name='Loss curve')
plt.show()
random_matrix = np.random.random((1000,1))
plt.scatter(random_matrix, model.predict(random_matrix), label = "Curva modelo")
plt.scatter(oDataSet.attributes, oDataSet.labels,label = "Curva dataset")
plt.legend(loc='upper left', bbox_to_anchor=(1.04, 1))
experiment.log_figure(figure=plt, figure_name="surface")
plt.show()
experiment.log_metric("test_accuracy", mean_squared_error(y_true, y_pred))
experiment.log_metric("test_accuracy_rmse", np.sqrt(mean_squared_error(y_true, y_pred)))
experiment.log_metric("beta", best_b)
experiment.log_metric("neurons", best_p)
model.save('model.h5')
experiment.log_asset("model.h5")
def train(normal_digit, anomalies, folder, file, p_train, p_test):
# Create an experiment
experiment = Experiment(project_name="deep-stats-thesis",
workspace="stecaron",
disabled=True)
experiment.add_tag("mnist_conv_ae")
# General parameters
DOWNLOAD_MNIST = True
PATH_DATA = os.path.join(os.path.expanduser("~"), 'Downloads/mnist')
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# Define training parameters
hyper_params = {
"EPOCH": 75,
"NUM_WORKERS": 10,
"BATCH_SIZE": 256,
"LR": 0.001,
"TRAIN_SIZE": 4000,
"TRAIN_NOISE": p_train,
"TEST_SIZE": 800,
"TEST_NOISE": p_test,
# on which class we want to learn outliers
"CLASS_SELECTED": [normal_digit],
# which class we want to corrupt our dataset with
"CLASS_CORRUPTED": anomalies,
"ALPHA": p_test,
"MODEL_NAME": "mnist_ae_model",
"LOAD_MODEL": False,
"LOAD_MODEL_NAME": "mnist_ae_model"
}
# Log experiment parameters
experiment.log_parameters(hyper_params)
# Load data
train_data, test_data = load_mnist(PATH_DATA, download=DOWNLOAD_MNIST)
# Train the autoencoder
model = ConvAutoEncoder2()
optimizer = torch.optim.Adam(model.parameters(), lr=hyper_params["LR"])
#loss_func = nn.MSELoss()
loss_func = nn.BCELoss()
# Build "train" and "test" datasets
id_maj_train = numpy.random.choice(numpy.where(
numpy.isin(train_data.train_labels, hyper_params["CLASS_SELECTED"]))[0],
int((1 - hyper_params["TRAIN_NOISE"]) *
hyper_params["TRAIN_SIZE"]),
replace=False)
id_min_train = numpy.random.choice(numpy.where(
numpy.isin(train_data.train_labels, hyper_params["CLASS_CORRUPTED"]))[0],
int(hyper_params["TRAIN_NOISE"] *
hyper_params["TRAIN_SIZE"]),
replace=False)
id_train = numpy.concatenate((id_maj_train, id_min_train))
id_maj_test = numpy.random.choice(numpy.where(
numpy.isin(test_data.test_labels, hyper_params["CLASS_SELECTED"]))[0],
int((1 - hyper_params["TEST_NOISE"]) *
hyper_params["TEST_SIZE"]),
replace=False)
id_min_test = numpy.random.choice(numpy.where(
numpy.isin(test_data.test_labels, hyper_params["CLASS_CORRUPTED"]))[0],
int(hyper_params["TEST_NOISE"] *
hyper_params["TEST_SIZE"]),
replace=False)
id_test = numpy.concatenate((id_min_test, id_maj_test))
train_data.data = train_data.data[id_train]
train_data.targets = train_data.targets[id_train]
test_data.data = test_data.data[id_test]
test_data.targets = test_data.targets[id_test]
train_data.targets = torch.from_numpy(
numpy.isin(train_data.train_labels,
hyper_params["CLASS_CORRUPTED"])).type(torch.int32)
test_data.targets = torch.from_numpy(
numpy.isin(test_data.test_labels,
hyper_params["CLASS_CORRUPTED"])).type(torch.int32)
train_loader = Data.DataLoader(dataset=train_data,
batch_size=hyper_params["BATCH_SIZE"],
shuffle=True,
num_workers=hyper_params["NUM_WORKERS"])
test_loader = Data.DataLoader(dataset=test_data,
batch_size=test_data.data.shape[0],
shuffle=False,
num_workers=hyper_params["NUM_WORKERS"])
model.train()
if hyper_params["LOAD_MODEL"]:
model = torch.load(hyper_params["LOAD_MODEL_NAME"])
else:
train_mnist(train_loader,
model,
criterion=optimizer,
n_epoch=hyper_params["EPOCH"],
experiment=experiment,
device=device,
model_name=hyper_params["MODEL_NAME"],
loss_func=loss_func,
loss_type="binary")
# Compute p-values
model.to(device)
pval, test_errors = compute_reconstruction_pval(
train_loader, model, test_loader, device)
pval_order = numpy.argsort(pval)
# Plot p-values
x_line = numpy.arange(0, len(test_data), step=1)
y_line = numpy.linspace(0, 1, len(test_data))
y_adj = numpy.arange(0, len(test_data),
step=1) / len(test_data) * hyper_params["ALPHA"]
zoom = int(0.2 * len(test_data)) # nb of points to zoom
#index = numpy.isin(test_data.test_labels, hyper_params["CLASS_CORRUPTED"]).astype(int)
index = numpy.array(test_data.targets).astype(int)
fig, (ax1, ax2) = plt.subplots(2, 1)
ax1.scatter(numpy.arange(0, len(pval), 1),
pval[pval_order],
c=index[pval_order].reshape(-1))
ax1.plot(x_line, y_line, color="green")
ax1.plot(x_line, y_adj, color="red")
ax1.set_title(
f'Entire test dataset with {int(hyper_params["TEST_NOISE"] * 100)}% of noise'
)
ax1.set_xticklabels([])
ax2.scatter(numpy.arange(0, zoom, 1),
pval[pval_order][0:zoom],
c=index[pval_order].reshape(-1)[0:zoom])
ax2.plot(x_line[0:zoom], y_line[0:zoom], color="green")
ax2.plot(x_line[0:zoom], y_adj[0:zoom], color="red")
ax2.set_title('Zoomed in')
ax2.set_xticklabels([])
experiment.log_figure(figure_name="empirical_test_hypothesis",
figure=fig,
overwrite=True)
plt.savefig(os.path.join(folder, "pvalues_" + file + ".png"))
plt.show()
# Compute some stats
precision, recall, f1_score, average_precision, roc_auc = test_performances(
pval, index, hyper_params["ALPHA"])
print(f"Precision: {precision}")
print(f"Recall: {recall}")
print(f"F1 Score: {f1_score}")
print(f"AUC: {roc_auc}")
print(f"Average Precison: {average_precision}")
experiment.log_metric("precision", precision)
experiment.log_metric("recall", recall)
experiment.log_metric("f1_score", f1_score)
experiment.log_metric("auc", roc_auc)
experiment.log_metric("average_precision", average_precision)
# Show some examples
fig, axs = plt.subplots(5, 5)
fig.tight_layout()
axs = axs.ravel()
for i in range(25):
image = test_data.data[pval_order[i]]
axs[i].imshow(image, cmap='gray')
axs[i].axis('off')
experiment.log_figure(figure_name="rejetcted_observations",
figure=fig,
overwrite=True)
plt.show()
fig, axs = plt.subplots(5, 5)
fig.tight_layout()
axs = axs.ravel()
for i in range(25):
image = test_data.data[pval_order[int(len(pval) - 1) - i]]
axs[i].imshow(image, cmap='gray')
axs[i].axis('off')
experiment.log_figure(figure_name="better_observations",
figure=fig,
overwrite=True)
plt.show()
# Save the results in the output file
col_names = ["timestamp", "precision", "recall", "f1_score",
"average_precision", "auc"]
results_file = os.path.join(folder, "results_" + file + ".csv")
if os.path.exists(results_file):
df_results = pandas.read_csv(results_file, names=col_names, header=0)
else:
df_results = pandas.DataFrame(columns=col_names)
df_results = df_results.append(
pandas.DataFrame(
numpy.concatenate(
(numpy.array(
datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')).reshape(1),
precision.reshape(1), recall.reshape(1),
f1_score.reshape(1), average_precision.reshape(1),
roc_auc.reshape(1))).reshape(1, -1), columns=col_names), ignore_index=True)
df_results.to_csv(results_file)
def train(folder, file, p_train, p_test):
# Create an experiment
experiment = Experiment(project_name="deep-stats-thesis",
workspace="stecaron",
disabled=True)
experiment.add_tag("cars_dogs")
# General parameters
PATH_DATA_CARS = os.path.join(os.path.expanduser("~"),
'data/stanford_cars')
PATH_DATA_DOGS = os.path.join(os.path.expanduser("~"),
'data/stanford_dogs2')
MEAN = [0.485, 0.456, 0.406]
STD = [0.229, 0.224, 0.225]
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# Define training parameters
hyper_params = {
"IMAGE_SIZE": (128, 128),
"NUM_WORKERS": 10,
"EPOCH": 20,
"BATCH_SIZE": 130,
"LR": 0.001,
"TRAIN_SIZE": 10000,
"TRAIN_NOISE": p_train,
"TEST_SIZE": 1000,
"TEST_NOISE": p_test,
"LATENT_DIM": 25, # latent distribution dimensions
"ALPHA": p_test, # level of significance for the test
"BETA_epoch": [5, 10, 15],
"BETA": [0, 100, 10], # hyperparameter to weight KLD vs RCL
"MODEL_NAME": "vae_model_cars",
"LOAD_MODEL": False,
"LOAD_MODEL_NAME": "vae_model_carsscenario_cars_plus"
}
# Log experiment parameters
experiment.log_parameters(hyper_params)
# Define some transformations
transform = transforms.Compose([
transforms.Resize((128, 128)),
#transforms.CenterCrop((128, 128)),
transforms.ToTensor(),
transforms.Normalize(mean=MEAN, std=STD)
])
# Load data
train_x_files, test_x_files, train_y, test_y = define_filenames(
PATH_DATA_DOGS, PATH_DATA_CARS, hyper_params["TRAIN_SIZE"],
hyper_params["TEST_SIZE"], hyper_params["TRAIN_NOISE"],
hyper_params["TEST_NOISE"])
train_data = DataGenerator(train_x_files,
train_y,
transform=transform,
image_size=hyper_params["IMAGE_SIZE"])
test_data = DataGenerator(test_x_files,
test_y,
transform=transform,
image_size=hyper_params["IMAGE_SIZE"])
train_loader = Data.DataLoader(dataset=train_data,
batch_size=hyper_params["BATCH_SIZE"],
shuffle=True,
num_workers=hyper_params["NUM_WORKERS"])
test_loader = Data.DataLoader(dataset=test_data,
batch_size=1,
shuffle=False,
num_workers=hyper_params["NUM_WORKERS"])
# Load model
model = SmallCarsConvVAE128(z_dim=hyper_params["LATENT_DIM"])
optimizer = torch.optim.Adam(model.parameters(), lr=hyper_params["LR"])
model.to(device)
model_save = os.path.join(folder, hyper_params["MODEL_NAME"] + file)
# Train the model
if hyper_params["LOAD_MODEL"]:
model.load_state_dict(
torch.load(f'{hyper_params["LOAD_MODEL_NAME"]}.h5'))
else:
train_mnist_vae(
train_loader,
# test_loader,
model,
criterion=optimizer,
n_epoch=hyper_params["EPOCH"],
experiment=experiment,
beta_list=hyper_params["BETA"],
beta_epoch=hyper_params["BETA_epoch"],
model_name=model_save,
device=device,
loss_type="perceptual",
flatten=False)
# Compute p-values
model.to(device)
pval, _ = compute_pval_loaders(train_loader,
test_loader,
model,
device=device,
experiment=experiment,
file=file,
folder=folder)
pval = 1 - pval # we test on the tail
pval_order = numpy.argsort(pval)
# Plot p-values
x_line = numpy.arange(0, len(test_data), step=1)
y_line = numpy.linspace(0, 1, len(test_data))
y_adj = numpy.arange(0, len(test_data),
step=1) / len(test_data) * hyper_params["ALPHA"]
zoom = int(0.2 * len(test_data)) # nb of points to zoom
index = test_data.labels
fig, (ax1, ax2) = plt.subplots(2, 1)
ax1.scatter(numpy.arange(0, len(pval), 1),
pval[pval_order],
c=index[pval_order].reshape(-1))
ax1.plot(x_line, y_line, color="green")
ax1.axhline(hyper_params["ALPHA"], color="red")
#ax1.plot(x_line, y_adj, color="red")
ax1.set_ylabel(r"Score $(1 - \gamma)$")
ax1.set_title(
f'Jeu de données test avec {int(hyper_params["TEST_NOISE"] * 100)}% de contamination'
)
ax1.set_xticklabels([])
ax2.scatter(numpy.arange(0, zoom, 1),
pval[pval_order][0:zoom],
c=index[pval_order].reshape(-1)[0:zoom])
ax2.plot(x_line[0:zoom], y_line[0:zoom], color="green")
ax2.axhline(hyper_params["ALPHA"], color="red")
#ax2.plot(x_line[0:zoom], y_adj[0:zoom], color="red")
ax2.set_ylabel(r"Score $(1 - \gamma)$")
ax2.set_title('Vue rapprochée')
ax2.set_xticklabels([])
experiment.log_figure(figure_name="empirical_test_hypothesis",
figure=fig,
overwrite=True)
plt.savefig(os.path.join(folder, "pvalues_" + file + ".pdf"))
plt.show()
# Compute some stats
precision, recall, f1_score, average_precision, roc_auc = test_performances(
pval, index, hyper_params["ALPHA"])
print(f"Precision: {precision}")
print(f"Recall: {recall}")
print(f"F1-Score: {f1_score}")
print(f"Average precision: {average_precision}")
print(f"AUC: {roc_auc}")
experiment.log_metric("precision", precision)
experiment.log_metric("recall", recall)
experiment.log_metric("F1-Score", f1_score)
experiment.log_metric("average_precision", average_precision)
experiment.log_metric("AUC", roc_auc)
# Show some examples
plt.rcParams['figure.figsize'] = [10, 10]
fig, axs = plt.subplots(4, 4)
fig.tight_layout()
axs = axs.ravel()
for i in range(16):
image = test_data[pval_order[i]][0].transpose_(0, 2)
image = denormalize(image, MEAN, STD, device=device).numpy()
axs[i].imshow(image)
axs[i].axis('off')
experiment.log_figure(figure_name="rejetcted_observations",
figure=fig,
overwrite=True)
plt.savefig(os.path.join(folder, "rejected_observations_" + file + ".pdf"))
plt.show()
fig, axs = plt.subplots(4, 4)
fig.tight_layout()
axs = axs.ravel()
for i in range(16):
image = test_data[pval_order[int(len(pval) - 1) - i]][0].transpose_(
0, 2)
image = denormalize(image, MEAN, STD, device=device).numpy()
axs[i].imshow(image)
axs[i].axis('off')
experiment.log_figure(figure_name="better_observations",
figure=fig,
overwrite=True)
plt.savefig(os.path.join(folder, "better_observations_" + file + ".pdf"))
plt.show()
# Plot some errors
preds = numpy.zeros(index.shape[0])
preds[numpy.argwhere(pval <= hyper_params["ALPHA"])] = 1
false_positive = numpy.where((index != preds) & (index == 1))[0]
nb_errors = numpy.min([16, false_positive.shape[0]])
sample_errors = numpy.random.choice(false_positive,
nb_errors,
replace=False)
fig, axs = plt.subplots(4, 4)
fig.tight_layout()
axs = axs.ravel()
for i in range(nb_errors):
image = test_data[sample_errors[i]][0].transpose_(0, 2)
image = denormalize(image, MEAN, STD, device=device).numpy()
axs[i].imshow(image)
axs[i].axis('off')
plt.savefig(os.path.join(folder, "false_positive_sample_" + file + ".pdf"))
plt.show()
false_negative = numpy.where((index != preds) & (index == 0))[0]
nb_errors = numpy.min([16, false_negative.shape[0]])
sample_errors = numpy.random.choice(false_negative,
nb_errors,
replace=False)
fig, axs = plt.subplots(4, 4)
fig.tight_layout()
axs = axs.ravel()
for i in range(nb_errors):
image = test_data[sample_errors[i]][0].transpose_(0, 2)
image = denormalize(image, MEAN, STD, device=device).numpy()
axs[i].imshow(image)
axs[i].axis('off')
plt.savefig(os.path.join(folder, "false_negative_sample_" + file + ".pdf"))
plt.show()
# Save the results in the output file
col_names = [
"timestamp", "precision", "recall", "f1_score", "average_precision",
"auc"
]
results_file = os.path.join(folder, "results_" + file + ".csv")
if os.path.exists(results_file):
df_results = pandas.read_csv(results_file, names=col_names, header=0)
else:
df_results = pandas.DataFrame(columns=col_names)
df_results = df_results.append(pandas.DataFrame(numpy.concatenate(
(numpy.array(
datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')).reshape(1),
precision.reshape(1), recall.reshape(1), f1_score.reshape(1),
average_precision.reshape(1), roc_auc.reshape(1))).reshape(1, -1),
columns=col_names),
ignore_index=True)
df_results.to_csv(results_file)
def train(rank, defparams, hyper):
params = {}
for param in defparams.keys():
params[param] = defparams[param]
hyperp = {}
for hp in hyper.keys():
hyperp[hp] = hyper[hp]
experiment = Experiment(api_key="keGmeIz4GfKlQZlOP6cit4QOi",
project_name="hadron-shower",
workspace="engineren")
experiment.add_tag(params['exp'])
experiment.log_parameters(hyperp)
device = torch.device("cuda")
torch.manual_seed(params["seed"])
world_size = int(os.environ["SLURM_NNODES"])
rank = int(os.environ["SLURM_PROCID"])
dist.init_process_group(backend='nccl',
world_size=world_size,
rank=rank,
init_method=params["DDP_init_file"])
aD = DCGAN_D(hyperp["ndf"]).to(device)
aG = DCGAN_G(hyperp["ngf"], hyperp["z"]).to(device)
aE = energyRegressor().to(device)
aP = PostProcess_Size1Conv_EcondV2(48,
13,
3,
128,
bias=True,
out_funct='none').to(device)
optimizer_g = torch.optim.Adam(aG.parameters(),
lr=hyperp["L_gen"],
betas=(0.5, 0.9))
optimizer_d = torch.optim.Adam(aD.parameters(),
lr=hyperp["L_crit"],
betas=(0.5, 0.9))
optimizer_e = torch.optim.SGD(aE.parameters(), lr=hyperp["L_calib"])
optimizer_p = torch.optim.Adam(aP.parameters(),
lr=hyperp["L_post"],
betas=(0.5, 0.9))
assert torch.backends.cudnn.enabled, "NVIDIA/Apex:Amp requires cudnn backend to be enabled."
torch.backends.cudnn.benchmark = True
# Initialize Amp
models, optimizers = amp.initialize([aG, aD], [optimizer_g, optimizer_d],
opt_level="O1",
num_losses=2)
#aD = nn.DataParallel(aD)
#aG = nn.DataParallel(aG)
#aE = nn.DataParallel(aE)
aG, aD = models
optimizer_g, optimizer_d = optimizers
aG = nn.parallel.DistributedDataParallel(aG, device_ids=[0])
aD = nn.parallel.DistributedDataParallel(aD, device_ids=[0])
aE = nn.parallel.DistributedDataParallel(aE, device_ids=[0])
aP = nn.parallel.DistributedDataParallel(aP, device_ids=[0])
experiment.set_model_graph(str(aG), overwrite=False)
experiment.set_model_graph(str(aD), overwrite=False)
if params["restore_pp"]:
aP.load_state_dict(
torch.load(params["restore_path_PP"] + params["post_saved"],
map_location=torch.device(device)))
if params["restore"]:
checkpoint = torch.load(params["restore_path"])
aG.load_state_dict(checkpoint['Generator'])
aD.load_state_dict(checkpoint['Critic'])
optimizer_g.load_state_dict(checkpoint['G_optimizer'])
optimizer_d.load_state_dict(checkpoint['D_optimizer'])
itr = checkpoint['iteration']
else:
aG.apply(weights_init)
aD.apply(weights_init)
itr = 0
if params["c0"]:
aE.apply(weights_init)
elif params["c1"]:
aE.load_state_dict(
torch.load(params["calib_saved"],
map_location=torch.device(device)))
one = torch.tensor(1.0).to(device)
mone = (one * -1).to(device)
print('loading data...')
paths_list = [
'/beegfs/desy/user/eren/data_generator/pion/hcal_only/pion40part1.hdf5',
'/beegfs/desy/user/eren/data_generator/pion/hcal_only/pion40part2.hdf5',
'/beegfs/desy/user/eren/data_generator/pion/hcal_only/pion40part3.hdf5',
'/beegfs/desy/user/eren/data_generator/pion/hcal_only/pion40part4.hdf5',
'/beegfs/desy/user/eren/data_generator/pion/hcal_only/pion40part5.hdf5',
'/beegfs/desy/user/eren/data_generator/pion/hcal_only/pion40part6.hdf5',
'/beegfs/desy/user/eren/data_generator/pion/hcal_only/pion40part7.hdf5'
]
train_data = PionsDataset(paths_list, core=True)
train_sampler = torch.utils.data.distributed.DistributedSampler(
train_data, num_replicas=world_size, rank=rank)
dataloader = DataLoader(train_data,
batch_size=hyperp["batch_size"],
num_workers=0,
shuffle=False,
drop_last=True,
pin_memory=True,
sampler=train_sampler)
print('done')
#scheduler_g = optim.lr_scheduler.StepLR(optimizer_g, step_size=1, gamma=params["gamma_g"])
#scheduler_d = optim.lr_scheduler.StepLR(optimizer_d, step_size=1, gamma=params["gamma_crit"])
#scheduler_e = optim.lr_scheduler.StepLR(optimizer_e, step_size=1, gamma=params["gamma_calib"])
#writer = SummaryWriter()
e_criterion = nn.L1Loss() # for energy regressor training
dataiter = iter(dataloader)
BATCH_SIZE = hyperp["batch_size"]
LATENT = hyperp["z"]
EXP = params["exp"]
KAPPA = hyperp["kappa"]
LAMBD = hyperp["lambda"]
## Post-Processing
LDP = hyperp["LDP"]
wMMD = hyperp["wMMD"]
wMSE = hyperp["wMSE"]
## IO paths
OUTP = params['output_path']
for iteration in range(50000):
iteration += itr + 1
#---------------------TRAIN D------------------------
for p in aD.parameters(): # reset requires_grad
p.requires_grad_(True) # they are set to False below in training G
for e in aE.parameters(): # reset requires_grad (constrainer)
e.requires_grad_(True) # they are set to False below in training G
for i in range(hyperp["ncrit"]):
aD.zero_grad()
aE.zero_grad()
noise = np.random.uniform(-1, 1, (BATCH_SIZE, LATENT))
noise = torch.from_numpy(noise).float()
noise = noise.view(
-1, LATENT, 1, 1,
1) #[BS, nz] --> [Bs,nz,1,1,1] Needed for Generator
noise = noise.to(device)
batch = next(dataiter, None)
if batch is None:
dataiter = iter(dataloader)
batch = dataiter.next()
real_label = batch['energy'] ## energy label
real_label = real_label.to(device)
with torch.no_grad():
noisev = noise # totally freeze G, training D
fake_data = aG(noisev, real_label).detach()
real_data = batch['shower'] # 48x48x48 calo image
real_data = real_data.to(device)
real_data.requires_grad_(True)
#### supervised-training for energy regressor!
if params["train_calib"]:
output = aE(real_data.float())
e_loss = e_criterion(output, real_label.view(BATCH_SIZE, 1))
e_loss.backward()
optimizer_e.step()
######
# train with real data
disc_real = aD(real_data.float(), real_label.float())
# train with fake data
fake_data = fake_data.unsqueeze(
1) ## transform to [BS, 1, 48, 48, 48]
disc_fake = aD(fake_data, real_label.float())
# train with interpolated data
gradient_penalty = calc_gradient_penalty(aD,
real_data.float(),
fake_data,
real_label,
BATCH_SIZE,
device,
DIM=13)
## wasserstein-1 distace
w_dist = torch.mean(disc_fake) - torch.mean(disc_real)
# final disc cost
disc_cost = torch.mean(disc_fake) - torch.mean(
disc_real) + LAMBD * gradient_penalty
with amp.scale_loss(disc_cost, optimizer_d) as scaled_loss:
scaled_loss.backward()
optimizer_d.step()
#--------------Log to COMET ML ----------
if i == hyperp["ncrit"] - 1:
experiment.log_metric("L_crit", disc_cost, step=iteration)
experiment.log_metric("gradient_pen",
gradient_penalty,
step=iteration)
experiment.log_metric("Wasserstein Dist",
w_dist,
step=iteration)
if params["train_calib"]:
experiment.log_metric("L_const", e_loss, step=iteration)
#---------------------TRAIN G------------------------
for p in aD.parameters():
p.requires_grad_(False) # freeze D
for c in aE.parameters():
c.requires_grad_(False) # freeze C
gen_cost = None
for i in range(hyperp["ngen"]):
aG.zero_grad()
noise = np.random.uniform(-1, 1, (BATCH_SIZE, LATENT))
noise = torch.from_numpy(noise).float()
noise = noise.view(
-1, LATENT, 1, 1,
1) #[BS, nz] --> [Bs,nz,1,1,1] Needed for Generator
noise = noise.to(device)
batch = next(dataiter, None)
if batch is None:
dataiter = iter(dataloader)
batch = dataiter.next()
real_label = batch['energy'] ## energy label
real_label = real_label.to(device)
noise.requires_grad_(True)
real_data = batch['shower'] # 48x48x48 calo image
real_data = real_data.to(device)
fake_data = aG(noise, real_label.float())
fake_data = fake_data.unsqueeze(
1) ## transform to [BS, 1, 48, 48, 48]
## calculate loss function
gen_cost = aD(fake_data.float(), real_label.float())
## label conditioning
#output_g = aE(fake_data)
#output_r = aE(real_data.float())
output_g = 0.0 #for now
output_r = 0.0 #for now
aux_fake = (output_g - real_label)**2
aux_real = (output_r - real_label)**2
aux_errG = torch.abs(aux_fake - aux_real)
## Total loss function for generator
g_cost = -torch.mean(gen_cost) + KAPPA * torch.mean(aux_errG)
with amp.scale_loss(g_cost, optimizer_g) as scaled_loss_G:
scaled_loss_G.backward()
optimizer_g.step()
#--------------Log to COMET ML ----------
experiment.log_metric("L_Gen", g_cost, step=iteration)
## plot example image
if iteration % 100 == 0.0 or iteration == 1:
image = fake_data.view(-1, 48, 13, 13).cpu().detach().numpy()
cmap = mpl.cm.viridis
cmap.set_bad('white', 1.)
figExIm = plt.figure(figsize=(6, 6))
axExIm1 = figExIm.add_subplot(1, 1, 1)
image1 = np.sum(image[0], axis=0)
masked_array1 = np.ma.array(image1, mask=(image1 == 0.0))
im1 = axExIm1.imshow(masked_array1,
filternorm=False,
interpolation='none',
cmap=cmap,
vmin=0.01,
vmax=100,
norm=mpl.colors.LogNorm(),
origin='lower')
figExIm.patch.set_facecolor('white')
axExIm1.set_xlabel('y [cells]', family='serif')
axExIm1.set_ylabel('x [cells]', family='serif')
figExIm.colorbar(im1)
experiment.log_figure(figure=plt, figure_name="x-y")
figExIm = plt.figure(figsize=(6, 6))
axExIm2 = figExIm.add_subplot(1, 1, 1)
image2 = np.sum(image[0], axis=1)
masked_array2 = np.ma.array(image2, mask=(image2 == 0.0))
im2 = axExIm2.imshow(masked_array2,
filternorm=False,
interpolation='none',
cmap=cmap,
vmin=0.01,
vmax=100,
norm=mpl.colors.LogNorm(),
origin='lower')
figExIm.patch.set_facecolor('white')
axExIm2.set_xlabel('y [cells]', family='serif')
axExIm2.set_ylabel('z [layers]', family='serif')
figExIm.colorbar(im2)
experiment.log_figure(figure=plt, figure_name="y-z")
figExIm = plt.figure(figsize=(6, 6))
axExIm3 = figExIm.add_subplot(1, 1, 1)
image3 = np.sum(image[0], axis=2)
masked_array3 = np.ma.array(image3, mask=(image3 == 0.0))
im3 = axExIm3.imshow(masked_array3,
filternorm=False,
interpolation='none',
cmap=cmap,
vmin=0.01,
vmax=100,
norm=mpl.colors.LogNorm(),
origin='lower')
figExIm.patch.set_facecolor('white')
axExIm3.set_xlabel('x [cells]', family='serif')
axExIm3.set_ylabel('z [layers]', family='serif')
figExIm.colorbar(im3)
#experiment.log_metric("L_aux", aux_errG, step=iteration)
experiment.log_figure(figure=plt, figure_name="x-z")
## E-sum monitoring
figEsum = plt.figure(figsize=(6, 6 * 0.77 / 0.67))
axEsum = figEsum.add_subplot(1, 1, 1)
etot_real = getTotE(real_data.cpu().detach().numpy(),
xbins=13,
ybins=13)
etot_fake = getTotE(image, xbins=13, ybins=13)
axEsumReal = axEsum.hist(etot_real,
bins=25,
range=[0, 1500],
weights=np.ones_like(etot_real) /
(float(len(etot_real))),
label="orig",
color='blue',
histtype='stepfilled')
axEsumFake = axEsum.hist(etot_fake,
bins=25,
range=[0, 1500],
weights=np.ones_like(etot_fake) /
(float(len(etot_fake))),
label="generated",
color='red',
histtype='stepfilled')
axEsum.text(0.25,
0.81,
"WGAN",
horizontalalignment='left',
verticalalignment='top',
transform=axEsum.transAxes,
color='red')
axEsum.text(0.25,
0.87,
'GEANT 4',
horizontalalignment='left',
verticalalignment='top',
transform=axEsum.transAxes,
color='blue')
experiment.log_figure(figure=plt, figure_name="E-sum")
#end = timer()
#print(f'---train G elapsed time: {end - start}')
if params["train_postP"]:
#---------------------TRAIN P------------------------
for p in aD.parameters():
p.requires_grad_(False) # freeze D
for c in aG.parameters():
c.requires_grad_(False) # freeze G
lossP = None
for i in range(1):
noise = np.random.uniform(-1, 1, (BATCH_SIZE, LATENT))
noise = torch.from_numpy(noise).float()
noise = noise.view(
-1, LATENT, 1, 1,
1) #[BS, nz] --> [Bs,nz,1,1,1] Needed for Generator
noise = noise.to(device)
batch = next(dataiter, None)
if batch is None:
dataiter = iter(dataloader)
batch = dataiter.next()
real_label = batch['energy'] ## energy label
real_label = real_label.to(device)
noise.requires_grad_(True)
real_data = batch['shower'] # calo image
real_data = real_data.to(device)
## forward pass to generator
fake_data = aG(noise, real_label.float())
fake_data = fake_data.unsqueeze(
1) ## transform to [BS, 1, layer, size, size]
### first LossD_P
fake_dataP = aP(fake_data.float(), real_label.float())
lossD_P = aD(fake_dataP.float(), real_label.float())
lossD_P = lossD_P.mean()
## lossFixP
real_sorted = real_data.view(BATCH_SIZE, -1)
fake_sorted = fake_dataP.view(BATCH_SIZE, -1)
real_sorted, _ = torch.sort(real_sorted,
dim=1,
descending=True) #.view(900,1)
fake_sorted, _ = torch.sort(fake_sorted,
dim=1,
descending=True) #.view(900,1)
lossFixPp1 = mmd_hit_sortKernel(real_sorted.float(),
fake_sorted,
kernel_size=100,
stride=50,
cutoff=2000,
alpha=200)
lossFixPp2 = F.mse_loss(fake_dataP.view(BATCH_SIZE, -1),
fake_data.detach().view(
BATCH_SIZE, -1),
reduction='mean')
lossFixP = wMMD * lossFixPp1 + wMSE * lossFixPp2
lossP = LDP * lossD_P - lossFixP
lossP.backward(mone)
optimizer_p.step()
if iteration % 100 == 0 or iteration == 1:
print('iteration: {}, critic loss: {}'.format(
iteration,
disc_cost.cpu().data.numpy()))
if rank == 0:
torch.save(
{
'Generator': aG.state_dict(),
'Critic': aD.state_dict(),
'G_optimizer': optimizer_g.state_dict(),
'D_optimizer': optimizer_d.state_dict(),
'iteration': iteration
}, OUTP + '{0}/wgan_itrs_{1}.pth'.format(EXP, iteration))
if params["train_calib"]:
torch.save(
aE.state_dict(),
OUTP + '/{0}/netE_itrs_{1}.pth'.format(EXP, iteration))
if params["train_postP"]:
torch.save(
aP.state_dict(),
OUTP + '{0}/netP_itrs_{1}.pth'.format(EXP, iteration))
ap, f1_max, precision, recall, f1_max_th, fig_pre_rec, fig_th_pre_rec = precision_recall(y_test,
score_test,
limit=1000,
label_anomaly=labels[0])
#fig_score_train = plot_score(score_train, 'train')
#fig_score_val = plot_score(score_val, 'validation')
fig_score_test = plot_score(score_test, 'test', 10, labels=y_test, th=f1_max_th)
fig_cumsum = pca.plot_cumsum()
y_pred, conf_matrix = predict(score_test, f1_max_th, y_test, labels)
experiment.add_tags([data, metric])
parameters = {'var': var, 'pc': pca.pcs_, 'metric': metric}
experiment.log_parameters(parameters)
experiment.log_metric('ap', ap)
experiment.log_metric('f1', f1_max)
experiment.log_metric('precision', precision)
experiment.log_metric('recall', recall)
experiment.log_metric('train_time', pca.time_)
experiment.log_parameter('th_f1', f1_max_th)
experiment.log_figure('cumsum', fig_cumsum)
experiment.log_figure('score_test',fig_score_test)
experiment.log_figure('precision_recall',fig_pre_rec)
experiment.log_figure('th_pre_rec_f1', fig_th_pre_rec)
experiment.log_confusion_matrix(matrix=conf_matrix, labels=labels)
experiment.end()
D_G_z2 = output.mean().item()
# Update G
optimizerG.step()
"""
Loss_D - discriminator loss calculated as the sum of losses for the all real and all fake batches (log(D(x))+log(D(G(z)))).
Loss_G - generator loss calculated as log(D(G(z)))
D(x) - the average output (across the batch) of the discriminator for the all real batch.
This should start close to 1 then theoretically converge to 0.5 when G gets better. Think about why this is.
D(G(z)) - average discriminator outputs for the all fake batch. The first number is before D is updated and the second number
is after D is updated. These numbers should start near 0 and converge to 0.5 as G gets better. Think about why this is.
"""
# Output training stats
if i % 10 == 0:
print('step:', steps, ' epoch:', epoch)
experiment.log_metrics({'Loss_D': errD.item(), 'Loss_G': errG.item(), 'D(x)': D_x, 'D(G(z1))': D_G_z1, 'D(G(z2))': D_G_z2})
if (steps % 100 == 0) or ((epoch == num_epochs-1) and (i == len(dataloader)-1)):
fixed_noise = torch.randn(3, nz, 1, 1, device=device)
with torch.no_grad():
fake = netG(fixed_noise).detach().cpu().numpy()
plot = plt.figure(figsize=(20, 10))
for m in range(3):
plt.subplot(1, 3, m + 1)
plt.imshow((fake[m].transpose((1, 2, 0))+1)/2)
experiment.log_figure(figure_name='epoch_{}_{}'.format(epoch, steps))
plt.close()
steps += 1
class ModelTrainer:
def __init__(self, model, dataloader, args):
self.model = model
self.args = args
self.data = dataloader
self.metric = args.metric
if (dataloader is not None):
self.frq_log = len(dataloader['train']) // args.frq_log
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
model.to(self.device)
if args.optimizer == 'sgd':
self.optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
elif args.optimizer == 'adam':
self.optimizer = optim.Adam(model.parameters(),
lr=args.lr,
betas=(args.beta1, 0.999),
weight_decay=args.weight_decay)
else:
raise Exception('--optimizer should be one of {sgd, adam}')
if args.scheduler == 'set':
self.scheduler = optim.lr_scheduler.LambdaLR(
self.optimizer,
lambda epoch: 10**(epoch / args.scheduler_factor))
elif args.scheduler == 'auto':
self.scheduler = optim.lr_scheduler.ReduceLROnPlateau(
self.optimizer,
mode='min',
factor=args.scheduler_factor,
patience=5,
verbose=True,
threshold=0.0001,
threshold_mode='rel',
cooldown=0,
min_lr=0,
eps=1e-08)
self.experiment = Experiment(api_key=args.comet_key,
project_name=args.comet_project,
workspace=args.comet_workspace,
auto_weight_logging=True,
auto_metric_logging=False,
auto_param_logging=False)
self.experiment.set_name(args.name)
self.experiment.log_parameters(vars(args))
self.experiment.set_model_graph(str(self.model))
def train_one_epoch(self, epoch):
self.model.train()
train_loader = self.data['train']
train_loss = 0
correct = 0
comet_offset = epoch * len(train_loader)
for batch_idx, (data, target) in tqdm(enumerate(train_loader),
leave=True,
total=len(train_loader)):
data, target = data.to(self.device), target.to(self.device)
self.optimizer.zero_grad()
output = self.model(data)
loss = F.cross_entropy(output, target, reduction='sum')
loss.backward()
self.optimizer.step()
pred = output.argmax(dim=1, keepdim=True)
acc = pred.eq(target.view_as(pred)).sum().item()
train_loss += loss.item()
correct += acc
loss = loss.item() / len(data)
acc = 100. * acc / len(data)
comet_step = comet_offset + batch_idx
self.experiment.log_metric('batch_loss', loss, comet_step, epoch)
self.experiment.log_metric('batch_acc', acc, comet_step, epoch)
if (batch_idx + 1) % self.frq_log == 0:
self.experiment.log_metric('log_loss', loss, comet_step, epoch)
self.experiment.log_metric('log_acc', acc, comet_step, epoch)
print('Epoch: {} [{}/{}]\tLoss: {:.6f}\tAcc: {:.2f}%'.format(
epoch + 1, (batch_idx + 1) * len(data),
len(train_loader.dataset), loss, acc))
train_loss /= len(train_loader.dataset)
acc = 100. * correct / len(train_loader.dataset)
comet_step = comet_offset + len(train_loader) - 1
self.experiment.log_metric('loss', train_loss, comet_step, epoch)
self.experiment.log_metric('acc', acc, comet_step, epoch)
print(
'Epoch: {} [Done]\tLoss: {:.4f}\tAccuracy: {}/{} ({:.2f}%)'.format(
epoch + 1, train_loss, correct, len(train_loader.dataset),
acc))
return {'loss': train_loss, 'acc': acc}
def train(self):
self.log_cmd()
best = -1
history = {'lr': [], 'train_loss': []}
try:
print(">> Training %s" % self.model.name)
for epoch in range(self.args.nepoch):
with self.experiment.train():
train_res = self.train_one_epoch(epoch)
with self.experiment.validate():
print("\nvalidation...")
comet_offset = (epoch + 1) * len(self.data['train']) - 1
res = self.val(self.data['val'], comet_offset, epoch)
if res[self.metric] > best:
best = res[self.metric]
self.save_weights(epoch)
if self.args.scheduler == 'set':
lr = self.optimizer.param_groups[0]['lr']
history['lr'].append(lr)
history['train_loss'].append(train_res['loss'])
self.scheduler.step(epoch + 1)
lr = self.optimizer.param_groups[0]['lr']
print('learning rate changed to: %.10f' % lr)
elif self.args.scheduler == 'auto':
self.scheduler.step(train_res['loss'])
finally:
print(">> Training model %s. [Stopped]" % self.model.name)
self.experiment.log_asset_folder(os.path.join(
self.args.outf, self.args.name, 'weights'),
step=None,
log_file_name=False,
recursive=False)
if self.args.scheduler == 'set':
plt.semilogx(history['lr'], history['train_loss'])
plt.grid(True)
self.experiment.log_figure(figure=plt)
plt.show()
def val(self, val_loader, comet_offset=-1, epoch=-1):
self.model.eval()
test_loss = 0
correct = 0
labels = list(range(self.args.nclass))
cm = np.zeros((len(labels), len(labels)))
with torch.no_grad():
for data, target in tqdm(val_loader,
leave=True,
total=len(val_loader)):
data, target = data.to(self.device), target.to(self.device)
output = self.model(data)
test_loss += F.cross_entropy(output, target,
reduction='sum').item()
pred = output.argmax(dim=1, keepdim=True)
correct += pred.eq(target.view_as(pred)).sum().item()
pred = pred.view_as(target).data.cpu().numpy()
target = target.data.cpu().numpy()
cm += confusion_matrix(target, pred, labels=labels)
test_loss /= len(val_loader.dataset)
accuracy = 100. * correct / len(val_loader.dataset)
print('Evaluation: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)'.
format(test_loss, correct, len(val_loader.dataset), accuracy))
res = {'loss': test_loss, 'acc': accuracy}
self.experiment.log_metrics(res, step=comet_offset, epoch=epoch)
self.experiment.log_confusion_matrix(
matrix=cm,
labels=[ClassDict.getName(x) for x in labels],
title='confusion matrix after epoch %03d' % epoch,
file_name="confusion_matrix_%03d.json" % epoch)
return res
def test(self):
self.load_weights()
with self.experiment.test():
print('\ntesting....')
res = self.val(self.data['test'])
def log_cmd(self):
d = vars(self.args)
cmd = '!python main.py \\\n'
tab = ' '
for k, v in d.items():
if v is None or v == '' or (isinstance(v, bool) and v is False):
continue
if isinstance(v, bool):
arg = '--{} \\\n'.format(k)
else:
arg = '--{} {} \\\n'.format(k, v)
cmd = cmd + tab + arg
# print(cmd);
self.experiment.log_text(cmd)
def save_weights(self, epoch: int):
weight_dir = os.path.join(self.args.outf, self.args.name, 'weights')
if not os.path.exists(weight_dir):
os.makedirs(weight_dir)
torch.save({
'epoch': epoch,
'state_dict': self.model.state_dict()
}, os.path.join(weight_dir, 'model.pth'))
def load_weights(self):
path_g = self.args.weights_path
if path_g is None:
weight_dir = os.path.join(self.args.outf, self.args.name,
'weights')
path_g = os.path.join(weight_dir, 'model.pth')
print('>> Loading weights...')
weights_g = torch.load(path_g, map_location=self.device)['state_dict']
self.model.load_state_dict(weights_g)
print(' Done.')
def predict(self, x):
x = x / 2**15
self.model.eval()
with torch.no_grad():
x = torch.from_numpy(x).float()
x = self.transform(x)
x = x.unsqueeze(0)
x = self.model(x)
x = F.softmax(x, dim=1)
x = x.numpy()
return x
