def main(arguments):
with open(arguments.filepath, 'r') as fp:
json_exp = json.load(fp)
neptune.init(api_token=arguments.neptune_api_token,
project_qualified_name=arguments.project_name)
with neptune.create_experiment(
name=json_exp['name'],
description=json_exp['description'],
params=json_exp['params'],
properties=json_exp['properties'],
tags=json_exp['tags'],
upload_source_files=json_exp['upload_source_files']):
for name, channel_xy in json_exp['send_metric'].items():
for x, y in zip(channel_xy['x'], channel_xy['y']):
neptune.send_metric(name, x=x, y=y)
for name, channel_xy in json_exp['send_text'].items():
for x, y in zip(channel_xy['x'], channel_xy['y']):
neptune.send_text(name, x=x, y=y)
for name, channel_xy in json_exp['send_image'].items():
for x, y in zip(channel_xy['x'], channel_xy['y']):
neptune.send_image(name, x=x, y=y)
def plot_latent_2d(self, test_encode, test_targets, test_loss):
if self.trainer.distribution == 'flower':
test_encode = test_encode * 2 - 1.
# save encoded samples plot
plt.figure(figsize=(10, 10))
#colordict = {0 : 'black', 1 : 'brown', 2 : 'b', 3 : 'cyan', 4 : 'g', 5 : 'lime', 6 : 'm', 7 : 'r', 8 : 'orange', 9 : 'y'}
#for k in range(len(test_encode)):
# plt.scatter(test_encode[k, 0], test_encode[k, 1], c=colordict[test_targets[k]])
#plt.scatter(test_encode[:, 0], test_encode[:, 1], c=(10 * test_targets), cmap=plt.cm.Spectral)
plt.scatter(test_encode[:, self.projone],
test_encode[:, self.projtwo],
c=(10 * test_targets),
cmap=plt.cm.Spectral)
plt.colorbar()
if self.trainer.distribution == 'gaussflower':
plt.xlim([-6, 6])
plt.ylim([-6, 6])
else:
plt.xlim([-1.5, 1.5])
plt.ylim([-1.5, 1.5])
#plt.title('Test Latent Space\nLoss: {:.5f}'.format(test_loss))
filename = '{}/test_latent_epoch_{}.pdf'.format(
self.imagesdir, self.epoch + 1)
plt.savefig(filename)
filename = '{}/test_latent_epoch_{}.png'.format(
self.imagesdir, self.epoch + 1)
plt.savefig(filename)
plt.close()
neptune.send_image('plot_latent_2d', x=self.global_iters, y=filename)
def _send_image_channels(self):
self.model.eval()
pred_masks = self.get_prediction_masks()
self.model.train()
for name, pred_mask in pred_masks.items():
for i, image_duplet in enumerate(pred_mask):
h, w = image_duplet.shape[1:]
image_glued = np.zeros((h, 2 * w + 10))
image_glued[:, :w] = image_duplet[0, :, :]
image_glued[:, (w + 10):] = image_duplet[1, :, :]
pill_image = Image.fromarray(
(image_glued * 255.).astype(np.uint8))
h_, w_ = image_glued.shape
pill_image = pill_image.resize(
(int(self.image_resize * w_), int(self.image_resize * h_)),
Image.ANTIALIAS)
neptune.send_image('{} {}'.format(self.model_name, name),
pill_image)
if i == self.image_nr:
break
def save_scatter(x, y, filedir, name, it, other_extensions = ['pdf']):
filename = '{}/{}_epoch_{}.'.format(filedir, name, it+1)
extensions = ['png'] + other_extensions
plt.scatter(x, y, s=1)
for ext in extensions:
plt.savefig(filename+'{}'.format(ext))
plt.close()
neptune.send_image(name, x=it, y=filename+'png')
def prepare_plots3(target_vals,epoch, METRICSPATH):
cfm_file = f"{METRICSPATH}/confusion-{epoch}.png"
bok_file = f"{METRICSPATH}/ranking_{epoch}.html"
classes, rankings, preds, pred_rank = [],[],[],[]
for t_ in target_vals:
cl_,ra_,pr_,em_ = t_
classes.append(cl_)
rankings.append(ra_)
preds.append(pr_)
pred_rank.append(em_)
classes = np.squeeze(np.concatenate(classes))
rankings = np.squeeze(np.concatenate(rankings))
predictions = np.concatenate([softmax(p,axis=0) for p in preds])
pred_rank = np.concatenate(pred_rank)
activations = np.argmax(predictions,axis=1)
conf_mat = confusion_matrix(classes,activations)
fig = plt.figure(figsize=[10,8])
plot_confusion_matrix(conf_mat, classes=class_names, normalize=False,
title=f'Confusion matrix epoch {epoch}')
plt.savefig(cfm_file,format="png")
pil_image = fig2pil(fig)
neptune.send_image('conf_mat', pil_image)
df = pd.DataFrame(data={ 'tar': rankings, 'pred': pred_rank, 'class': classes})
palette = magma(num_of_classes + 1)
p = figure(plot_width=600, plot_height=800, title=f"Ranking by exercise, epoch {epoch}")
p.xgrid.grid_line_color = None
p.xaxis.axis_label = 'Target ranking'
p.yaxis.axis_label = 'Predicted ranking'
for cl in range(num_of_classes):
if cl == 6:
continue
df2 = df.loc[df['class']==cl]
p.circle(x=jitter('tar',0.3), y='pred', size=8, alpha=0.1, color=palette[cl], legend=class_names[cl], source=df2 )
p.line(x='tar', y='pred', line_width=2, alpha=0.5, color=palette[cl],legend=class_names[cl], source=df2.groupby(by="tar").mean())
p.legend.location = "top_left"
p.legend.click_policy="hide"
output_file(bok_file, title="Ranking by exercise")
save(p)
pil_image2 = get_screenshot_as_png(p)
neptune.send_image('rank_distances', pil_image2)
def bias_benchmark(task, feature, fraction_range, plot=True):
if task == 'classification':
get_datasets = get_classification_datasets
else:
get_datasets = get_regression_datasets
SEED = 42
for d in get_datasets():
X_train, X_test, y_train, y_test, dataset_name = d
print(dataset_name)
X = np.concatenate([X_train, X_test], axis=0)
y = np.concatenate([y_train, y_test], axis=0)
features = [f'f{i + 1}' for i in range(X.shape[1])]
df = pd.DataFrame(X, columns=features)
image_path = f'./logs/{task}_{dataset_name}_{feature}'
correlations, rf_scores, sf_scores, permutation_importances = bias_experiment(
df, y, task, feature, fraction_range, SEED)
if plot:
plot_bias(fraction_range, correlations, rf_scores, sf_scores,
permutation_importances, dataset_name,
image_path + '.png')
# Log chart and raw results into Neptune
neptune.send_image(f'{dataset_name}', image_path + '.png')
results_dict = {
'correlations': correlations.tolist(),
'rf_scores': rf_scores.tolist(),
'sf_scores': sf_scores.tolist(),
'permutation_importances': permutation_importances
}
results_json = json.dumps(results_dict)
result_file_path = f'./logs/{task}_{dataset_name}_{feature}_results.json'
with open(result_file_path, 'w+') as f:
json.dump(results_json, f)
print(f'Saved results to: {result_file_path}')
neptune.log_artifact(result_file_path)
def _send_image_channels(self):
self.model.eval()
image_triplets = self._get_image_triplets()
if self.image_nr is not None:
image_triplets = image_triplets[:self.image_nr]
self.model.train()
for i, image_triplet in enumerate(image_triplets):
h, w = image_triplet.shape[1:]
image_glued = np.zeros((h, 3 * w + 20))
image_glued[:, :w] = image_triplet[0, :, :]
image_glued[:, (w + 10):(2 * w + 10)] = image_triplet[1, :, :]
image_glued[:, (2 * w + 20):] = image_triplet[2, :, :]
pill_image = Image.fromarray((image_glued * 255.).astype(np.uint8))
h_, w_ = image_glued.shape
pill_image = pill_image.resize(
(int(self.image_resize * w_), int(self.image_resize * h_)),
Image.ANTIALIAS)
neptune.send_image('{} predictions'.format(self.model_name),
pill_image)
def handle_files_and_images(self):
# image
# `image_name` and `description` will be lost (`send_image` the same as `log_image`)
neptune.send_image("image",
self.img_path,
name="name",
description="desc")
# artifact with default dest
neptune.send_artifact(self.text_file_path)
exp = neptune.get_experiment()
with self.with_check_if_file_appears("text.txt"):
exp.download_artifact("text.txt")
with self.with_check_if_file_appears("custom_dest/text.txt"):
exp.download_artifact("text.txt", "custom_dest")
# artifact with custom dest
neptune.send_artifact(self.text_file_path, destination="something.txt")
exp = neptune.get_experiment()
with self.with_check_if_file_appears("something.txt"):
exp.download_artifact("something.txt")
with self.with_check_if_file_appears("custom_dest/something.txt"):
exp.download_artifact("something.txt", "custom_dest")
# destination dirs
neptune.log_artifact(self.text_file_path,
destination="dir/text file artifact")
neptune.log_artifact(self.text_file_path,
destination="dir/artifact_to_delete")
# deleting
neptune.delete_artifacts("dir/artifact_to_delete")
# streams
with open(self.text_file_path, mode="r") as f:
neptune.send_artifact(f, destination="file stream.txt")
def box_plot(x,y,cl_ar,bok_file,epoch):
class_names = ['squat', 'deadlift', 'pushups', 'pullups', 'wallpushups', 'lunges', 'other', 'cleanandjerk']
num_of_classes = len(class_names)
df = pd.DataFrame(data={ 'tar': x, 'pred': y, 'class': cl_ar})
palette = magma(num_of_classes + 1)
p = figure(plot_width=600, plot_height=800, title=f"Ranking by exercise, epoch {epoch}")
p.xgrid.grid_line_color = None
p.xaxis.axis_label = 'Target ranking'
p.yaxis.axis_label = 'Predicted ranking'
for cl in range(num_of_classes):
if cl == 6:
continue
df2 = df.loc[df['class']==cl]
p.circle(x=jitter('tar', 0.5), y='pred', size=8, alpha=0.1, color=palette[cl], legend=class_names[cl], source=df2 )
p.line(x='tar', y='pred', line_width=2, alpha=0.5, color=palette[cl], source=df2.groupby(by="tar").mean())
p.legend.location = "top_left"
p.legend.click_policy="hide"
output_file(bok_file, title="Ranking by exercise")
save(p)
pil_image2 = get_screenshot_as_png(p)
neptune.send_image('rank_distances', pil_image2)
def run_training_games_multi_process(
self,
opponent_to_train,
baselines,
epochs,
n_test_games,
mcts_passes,
exploration_ceofficient,
experiment_name: str = 'MCTS value training',
weights_path=None,
confidence_threshold: float = 0.1,
confidence_limit: int = 2,
count_ratio: float = 6,
replay_buffer_n_games: int = 10,
neural_network_train_epochs: int = 2,
reset_network: bool = True,
create_visualizer: bool = True,
use_neptune: bool = True,
tags=['experiment'],
source_files=None):
count_threshold = int(count_ratio * mcts_passes)
if main_process:
self.params['mcts passes'] = mcts_passes
self.params['exploration coefficient'] = exploration_ceofficient
self.params['n test games'] = n_test_games
self.params['n proc'] = comm_size
self.params['replay buffer games'] = replay_buffer_n_games
self.params[
'opponent name'] = opponent_to_train.name if opponent_to_train != 'self' else 'self-play'
self.params['train_epochs'] = neural_network_train_epochs
self.params['count threshold'] = count_threshold
self.parse_params_files()
self.mcts_agent = SingleMCTSAgent(mcts_passes,
self.value_policy,
exploration_ceofficient,
create_visualizer=create_visualizer,
show_unvisited_nodes=False,
log_to_neptune=(main_process
and use_neptune))
if opponent_to_train == 'self':
self.opponent = SingleMCTSAgent(mcts_passes,
self.opponent_value_policy,
exploration_ceofficient,
create_visualizer=False,
show_unvisited_nodes=False,
log_to_neptune=False)
self.opponent.name = 'MCTS - opponent'
else:
self.opponent = opponent_to_train
if main_process and use_neptune:
self.create_neptune_experiment(experiment_name=experiment_name,
source_files=source_files)
if opponent_to_train == 'self':
tags.append('self-play')
neptune.append_tag(tags)
for epoch_idx in range(epochs):
if n_test_games > 0:
for baseline in baselines:
results_with_baseline = self.arena.run_many_duels(
'deterministic', [self.mcts_agent, baseline],
n_games=n_test_games,
n_proc_per_agent=1,
shuffle=False)
if main_process:
print(results_with_baseline)
_, _, baseline_win_rate, baseline_victory_points = results_with_baseline.return_stats(
)
neptune.send_metric(f'Win rate vs {baseline.name}',
x=epoch_idx + 1,
y=baseline_win_rate / n_test_games)
neptune.send_metric(f'Win points vs {baseline.name}',
x=epoch_idx + 1,
y=baseline_victory_points /
n_test_games)
if main_process:
print('============ \n Running MCTS games \n============')
results = self.arena.run_many_duels(
'deterministic', [self.mcts_agent, self.opponent],
n_games=comm_size,
n_proc_per_agent=1,
shuffle=False)
if main_process:
print(results)
self.data_collector.setup_root(
self.mcts_agent.mcts_algorithm.original_root)
local_data_for_training = self.data_collector.generate_all_tree_data_as_list(
confidence_threshold, count_threshold, confidence_limit)
combined_data = mpi_communicator.gather(local_data_for_training,
root=0)
if main_process:
data_from_this_epoch = self.flatten_data(combined_data)
self.replay_buffer.add_game(data_from_this_epoch)
data_for_training = self.replay_buffer.data_from_last_games(
replay_buffer_n_games)
_, _, mcts_win_rate, mcts_victory_points = results.return_stats(
)
if use_neptune:
neptune.log_metric('MCTS train win rate',
x=epoch_idx,
y=mcts_win_rate / comm_size)
neptune.log_metric('MCTS train victory points',
x=epoch_idx,
y=mcts_victory_points / comm_size)
plt.hist(data_for_training['mcts_value'], bins=100)
plt.savefig('epoch_histogram.png')
plt.clf()
img_histogram = Image.open('epoch_histogram.png')
if use_neptune:
neptune.send_image(
f'Train set histogram epoch = {epoch_idx}',
img_histogram)
self.data_collector.clean_memory()
if reset_network:
self.reset_weights()
fit_history = self.model.train_on_mcts_data(
data_for_training,
train_epochs=neural_network_train_epochs)
if use_neptune:
neptune.send_metric('training set size',
x=epoch_idx,
y=len(data_for_training['mcts_value']))
neptune.send_metric('loss',
x=epoch_idx,
y=fit_history.history['loss'][0])
self.mcts_agent.dump_weights(weights_file=weights_path +
f'epoch_{epoch_idx}.h5')
saved = main_process
weights_saved = mpi_communicator.bcast(saved, root=0)
if not main_process:
self.mcts_agent.load_weights(weights_file=weights_path +
f'epoch_{epoch_idx}.h5')
self.opponent.load_weights(weights_file=weights_path +
f'epoch_{epoch_idx}.h5')
if main_process and use_neptune:
neptune.stop()
def log_histograms(self, file1, file2):
img1 = Image.open(file1)
neptune.send_image('train set histogram', img1)
img2 = Image.open(file2)
neptune.send_image('val set histogram', img2)
print('Best Params: {}'.format(best_params))
neptune.send_metric('validation auc', best_auc)
neptune.set_property('best_params', str(to_named_params(best_params)))
# log results
skopt.dump(results, 'artifacts/gp_results.pkl')
joblib.dump(SPACE, 'artifacts/gp_space.pkl')
neptune.send_artifact('artifacts/gp_results.pkl')
neptune.send_artifact('artifacts/gp_space.pkl')
# log diagnostic plots
fig, ax = plt.subplots(figsize=(16, 12))
skopt.plots.plot_convergence(results, ax=ax)
fig.savefig('plots/gp_convergence.png')
neptune.send_image('diagnostics', 'plots/gp_convergence.png')
axes = skopt.plots.plot_evaluations(results)
fig = axes2fig(axes, figsize=(16, 12))
fig.savefig('plots/gp_evaluations.png')
neptune.send_image('diagnostics', 'plots/gp_evaluations.png')
axes = skopt.plots.plot_objective(results)
fig = axes2fig(axes, figsize=(16, 12))
fig.savefig('plots/gp_objective.png')
neptune.send_image('diagnostics', 'plots/gp_objective.png')
def prepare_plots2(pairings,target_vals,epoch, METRICSPATH):
cfm_file = f"{METRICSPATH}/confusion-{epoch}.png"
bok_file = f"{METRICSPATH}/ranking_{epoch}.html"
pairing_file = f"{METRICSPATH}/pairings_{epoch}.csv"
classes, rankings, preds, embeds = [],[],[],[]
rank1, rank2, dists, pair_class = [], [], [], []
for t_ , p_ in zip(target_vals, pairings):
cl_,ra_,pr_,em_ = t_
rk1_, rk2_, dist_, pcl_ = zip(*p_)
classes.append(cl_)
rankings.append(ra_)
preds.append(pr_)
embeds.append(em_)
rank1.append(rk1_)
rank2.append(rk2_)
dists.append(dist_)
pair_class.append(pcl_)
classes = np.squeeze(np.concatenate(classes))
rankings = np.squeeze(np.concatenate(rankings))
predictions = np.concatenate([softmax(p,axis=0) for p in preds])
embeds = np.concatenate(embeds)
dists = np.concatenate(dists)
rank1 = np.concatenate(rank1)
rank2 = np.concatenate(rank2)
pair_class = np.concatenate(pair_class)
activations = np.argmax(predictions,axis=1)
conf_mat = confusion_matrix(classes,activations)
fig = plt.figure(figsize=[10,8])
plot_confusion_matrix(conf_mat, classes=class_names, normalize=False,
title=f'Confusion matrix epoch {epoch}')
plt.savefig(cfm_file,format="png")
pil_image = fig2pil(fig)
neptune.send_image('conf_mat', pil_image)
max_rank = np.max(rankings)
true_dist = rank1 - rank2
tar_d = max_rank - true_dist
pred_d = max_rank - dists
pair_class = pair_class.astype(int)
df = pd.DataFrame(data={ 'tar': tar_d, 'pred': pred_d, 'class': pair_class})
palette = magma(num_of_classes + 1)
p = figure(plot_width=600, plot_height=800, title=f"Ranking by exercise, epoch {epoch}")
p.xgrid.grid_line_color = None
p.xaxis.axis_label = 'Target ranking'
p.yaxis.axis_label = 'Predicted ranking'
for cl in range(num_of_classes):
if cl == 6:
continue
df2 = df.loc[df['class']==cl]
p.circle(x=jitter('tar', 0.5), y='pred', size=8, alpha=0.1, color=palette[cl], legend=class_names[cl], source=df2 )
p.line(x='tar', y='pred', line_width=2, alpha=0.5, color=palette[cl], legend=class_names[cl], source=df2.groupby(by="tar").mean())
p.legend.location = "top_left"
p.legend.click_policy="hide"
output_file(bok_file, title="Ranking by exercise")
save(p)
pil_image2 = get_screenshot_as_png(p)
neptune.send_image('rank_distances', pil_image2)
pair_df = pd.DataFrame(data={ 'class': pair_class, 'rank1': rank1, 'rank2': rank2, 'dist':dists, 'true_dist':true_dist}).to_csv(pairing_file, index=False)
def main(args):
utils.makedirs(args.save_dir)
with open(f'{args.save_dir}/params.txt', 'w') as f:
json.dump(args.__dict__, f)
if args.print_to_log:
sys.stdout = open(f'{args.save_dir}/log.txt', 'w')
t.manual_seed(seed)
if t.cuda.is_available():
t.cuda.manual_seed_all(seed)
# datasets
dload_train, dload_train_labeled, dload_valid, dload_test, dload_train_splits, dload_train_labeled_splits, labels_in_splits = get_data(
args)
device = t.device('cuda' if t.cuda.is_available() else 'cpu')
sample_q = get_sample_q(args, device)
f, replay_buffer, past_buffer = get_model_and_buffer(
args, device, sample_q)
sqrt = lambda x: int(t.sqrt(t.Tensor([x])))
plot = lambda p, x: tv.utils.save_image(
t.clamp(x, -1, 1), p, normalize=True, nrow=sqrt(x.size(0)))
# optimizer
params = f.class_output.parameters() if args.clf_only else f.parameters()
if args.optimizer == "adam":
optim = t.optim.Adam(params,
lr=args.lr,
betas=[.9, .999],
weight_decay=args.weight_decay)
else:
optim = t.optim.SGD(params,
lr=args.lr,
momentum=.9,
weight_decay=args.weight_decay)
best_valid_acc = 0.0
global_iter = 0
for sp in range(args.num_splits):
print('\nFIT ON {}. SPLIT\n'.format(sp))
cur_iter = 0
past_labels = list(np.array(labels_in_splits[:sp]).flat)
print(past_labels)
curr_labels = list(np.array(labels_in_splits[sp]).flat)
print(curr_labels)
dload_train = dload_train_splits[sp]
dload_train_labeled = dload_train_labeled_splits[sp]
if sp > 0:
dload_train_prev_labeled = dload_train_labeled_splits[sp - 1]
if args.p_y_given_x_past_weight > 0 and len(past_labels) > 0:
print("Wokring on the past buffer a little bit..")
for j in range(500):
y_past = t.from_numpy(
np.random.choice(past_labels, args.batch_size,
p=None)).to(device)
x_past = sample_q(f, past_buffer, y=y_past, n_steps=100)
for epoch in range(args.n_epochs):
if epoch in args.decay_epochs:
for param_group in optim.param_groups:
new_lr = param_group['lr'] * args.decay_rate
param_group['lr'] = new_lr
print("Decaying lr to {}".format(new_lr))
for i, (x_p_d, _) in tqdm(enumerate(dload_train)):
if cur_iter <= args.warmup_iters:
lr = args.lr * cur_iter / float(args.warmup_iters)
for param_group in optim.param_groups:
param_group['lr'] = lr
x_p_d = x_p_d.to(device)
x_lab, y_lab = dload_train_labeled.__next__()
x_lab, y_lab = x_lab.to(device), y_lab.to(device)
L = 0.
if args.p_x_weight > 0: # maximize log p(x)
if args.class_cond_p_x_sample:
assert not args.uncond, "can only draw class-conditional samples if EBM is class-cond"
y_q = t.from_numpy(
np.random.choice(curr_labels,
args.batch_size,
p=None)).to(device)
#y_q = t.randint(0, args.n_classes, (args.batch_size,)).to(device)
x_q = sample_q(f, replay_buffer, y=y_q)
else:
x_q = sample_q(f,
replay_buffer) # sample from log-sumexp
fp_all = f(x_p_d)
fq_all = f(x_q)
fp = fp_all.mean()
fq = fq_all.mean()
l_p_x = -(fp - fq)
if global_iter % args.print_every == 0:
print(
'P(x) | {}:{:>d} f(x_p_d)={:>14.9f} f(x_q)={:>14.9f} d={:>14.9f}'
.format(epoch, i, fp, fq, fp - fq))
neptune.send_metric('f(x_p_d)', x=global_iter, y=fp)
neptune.send_metric('f(x_q)', x=global_iter, y=fq)
neptune.send_metric('f(x_p_d)-f(x_q)',
x=global_iter,
y=fp - fq)
L += args.p_x_weight * l_p_x
if args.p_y_given_x_weight > 0: # maximize log p(y | x)
logits = f.classify(x_lab)
l_p_y_given_x = nn.CrossEntropyLoss()(logits, y_lab)
if global_iter % args.print_every == 0:
acc = (logits.max(1)[1] == y_lab).float().mean()
print('P(y|x) {}:{:>d} loss={:>14.9f}, acc={:>14.9f}'.
format(epoch, global_iter, l_p_y_given_x.item(),
acc.item()))
neptune.send_metric('acc', x=global_iter, y=acc.item())
neptune.send_metric('loss_p_y_given_x',
x=global_iter,
y=l_p_y_given_x.item())
L += args.p_y_given_x_weight * l_p_y_given_x
if args.p_x_y_weight > 0: # maximize log p(x, y)
assert not args.uncond, "this objective can only be trained for class-conditional EBM DUUUUUUUUHHHH!!!"
x_q_lab = sample_q(f, replay_buffer, y=y_lab)
fp, fq = f(x_lab, y_lab).mean(), f(x_q_lab, y_lab).mean()
l_p_x_y = -(fp - fq)
if global_iter % args.print_every == 0:
print(
'P(x, y) | {}:{:>d} f(x_p_d)={:>14.9f} f(x_q)={:>14.9f} d={:>14.9f}'
.format(epoch, i, fp, fq, fp - fq))
neptune.send_metric('f(x_p_d)', x=global_iter, y=fp)
neptune.send_metric('f(x_q)', x=global_iter, y=fq)
neptune.send_metric('l_p_x_y',
x=global_iter,
y=-(fp - fq))
L += args.p_x_y_weight * l_p_x_y
if args.p_y_given_x_past_weight > 0 and len(past_labels) > 0:
y_past = t.from_numpy(
np.random.choice(past_labels, args.batch_size,
p=None)).to(device)
##y_past = t.randint(0, args.n_classes, (args.batch_size,)).to(device)
x_past = sample_q(f, past_buffer, y=y_past)
#x_past, y_past = dload_train_prev_labeled.__next__()
x_past, y_past = x_past.to(device), y_past.to(device)
logits_past = f.classify(x_past)
l_p_y_given_x_past = nn.CrossEntropyLoss()(logits_past,
y_past)
if global_iter % args.print_every == 0:
acc_past = (logits.max(1)[1] == y_past).float().mean()
print(
'P(y|x_past) {}:{:>d} loss={:>14.9f}, acc={:>14.9f}'
.format(epoch, global_iter,
l_p_y_given_x_past.item(),
acc_past.item()))
neptune.send_metric('acc_past',
x=global_iter,
y=acc_past.item())
neptune.send_metric('loss_p_y_given_x_past',
x=global_iter,
y=l_p_y_given_x_past.item())
L += args.p_y_given_x_past_weight * l_p_y_given_x_past
# break if the loss diverged...easier for poppa to run experiments this way
if L.abs().item() > 1e8:
print("BAD BOIIIIIIIIII")
1 / 0
optim.zero_grad()
L.backward()
optim.step()
cur_iter += 1
global_iter += 1
if global_iter % 100 == 0:
if args.plot_uncond:
if args.class_cond_p_x_sample:
assert not args.uncond, "can only draw class-conditional samples if EBM is class-cond"
y_q = t.randint(0, args.n_classes,
(args.batch_size, )).to(device)
x_q = sample_q(f, replay_buffer, y=y_q)
else:
x_q = sample_q(f, replay_buffer)
filename_uncond = '{}/x_q_{}_{:>06d}_{:>06d}.png'.format(
args.save_dir, epoch, i, global_iter)
plot(filename_uncond, x_q)
neptune.send_image('samples_uncond',
x=global_iter,
y=filename_uncond)
if args.plot_cond: # generate class-conditional samples
y = t.arange(0,
args.n_classes).repeat(10, 1).transpose(
1, 0).contiguous().view(-1).to(device)
x_q_y = sample_q(f, replay_buffer, y=y)
filename_cond = '{}/x_q_y{}_{:>06d}_{:>06d}.png'.format(
args.save_dir, epoch, i, global_iter)
plot(filename_cond, x_q_y)
neptune.send_image('samples_cond',
x=global_iter,
y=filename_cond)
if epoch % args.ckpt_every == 0:
checkpoint(f, replay_buffer, f'ckpt_{epoch}.pt', args, device)
if epoch % args.eval_every == 0 and (args.p_y_given_x_weight > 0
or args.p_x_y_weight > 0):
f.eval()
with t.no_grad():
# validation set
correct, loss = eval_classification(f, dload_valid, device)
print("Epoch {}: Valid Loss {}, Valid Acc {}".format(
epoch, loss, correct))
neptune.send_metric('valid_loss', x=global_iter, y=loss)
neptune.send_metric('valid_acc', x=global_iter, y=correct)
if correct > best_valid_acc:
best_valid_acc = correct
print("Best Valid!: {}".format(correct))
checkpoint(f, replay_buffer, "best_valid_ckpt.pt",
args, device)
neptune.send_metric('best_valid',
x=global_iter,
y=best_valid_acc)
# test set
correct, loss = eval_classification(f, dload_test, device)
print("Epoch {}: Test Loss {}, Test Acc {}".format(
epoch, loss, correct))
neptune.send_metric('test_loss', x=global_iter, y=loss)
neptune.send_metric('test_acc', x=global_iter, y=correct)
f.train()
checkpoint(f, replay_buffer, "last_ckpt.pt", args, device)
**HPO_PARAMS)
best_auc = -1.0 * results.fun
best_params = results.x
# log metrics
print('Best Validation AUC: {}'.format(best_auc))
print('Best Params: {}'.format(best_params))
neptune.send_metric('validation auc', best_auc)
neptune.set_property('best_params', str(to_named_params(best_params)))
# log results
skopt.dump(results, 'artifacts/random_results.pkl')
joblib.dump(SPACE, 'artifacts/random_space.pkl')
neptune.send_artifact('artifacts/random_results.pkl')
neptune.send_artifact('artifacts/random_space.pkl')
# log diagnostic plots
fig, ax = plt.subplots(figsize=(16, 12))
skopt.plots.plot_convergence(results, ax=ax)
fig.savefig('plots/random_convergence.png')
neptune.send_image('diagnostics', 'plots/random_convergence.png')
axes = skopt.plots.plot_evaluations(results)
fig = axes2fig(axes, figsize=(16, 12))
fig.savefig('plots/random_evaluations.png')
neptune.send_image('diagnostics', 'plots/random_evaluations.png')
def save_image(x, name, it, filename, normalize=True, x_range=(0., 1.)):
vutils.save_image(x, filename, nrow = 10, normalize=normalize, range=x_range)
neptune.send_image(name, x=it, y=filename)
def main():
neptune.init(api_token=os.getenv('NEPTUNE_API_TOKEN'),
project_qualified_name=os.getenv('NEPTUNE_PROJECT'))
train_idx = pd.read_csv(TRAIN_IDX_PATH, nrows=NROWS)
valid_idx = pd.read_csv(VALID_IDX_PATH, nrows=NROWS)
features = pd.read_csv(FEATURES_PATH, nrows=NROWS)
train = pd.merge(train_idx, features, on='SK_ID_CURR')
valid = pd.merge(valid_idx, features, on='SK_ID_CURR')
all_params = {
'num_boost_round': NUM_BOOST_ROUND,
'early_stopping_rounds': EARLY_STOPPING_ROUNDS,
**LGBM_PARAMS
}
with neptune.create_experiment(name='model training',
params=all_params,
tags=['lgbm'],
upload_source_files=get_filepaths(),
properties={
'features_path':
FEATURES_PATH,
'features_version':
md5_hash(FEATURES_PATH),
'train_split_version':
md5_hash(TRAIN_IDX_PATH),
'valid_split_version':
md5_hash(VALID_IDX_PATH),
}):
results = train_evaluate(train,
valid,
LGBM_PARAMS,
callbacks=[neptune_monitor()])
train_score, valid_score = results['train_score'], results[
'valid_score']
train_preds, valid_preds = results['train_preds'], results[
'valid_preds']
neptune.send_metric('train_auc', train_score)
neptune.send_metric('valid_auc', valid_score)
train_pred_path = os.path.join(PREDICTION_DIRPATH, 'train_preds.csv')
train_preds.to_csv(train_pred_path, index=None)
neptune.send_artifact(train_pred_path)
valid_pred_path = os.path.join(PREDICTION_DIRPATH, 'valid_preds.csv')
valid_preds.to_csv(valid_pred_path, index=None)
neptune.send_artifact(valid_pred_path)
model_path = os.path.join(MODEL_DIRPATH, 'model.pkl')
joblib.dump(results['model'], model_path)
neptune.set_property('model_path', model_path)
neptune.set_property('model_version', md5_hash(model_path))
neptune.send_artifact(model_path)
if PACKAGE_TO_PROD:
saved_path = CreditDefaultClassifier.pack(
model=results['model']).save(PRODUCTION_DIRPATH)
neptune.set_property('production_model_path', saved_path)
fig, ax = plt.subplots(figsize=(16, 12))
sk_metrics.plot_confusion_matrix(valid_preds['TARGET'],
valid_preds['preds_pos'] > 0.5,
ax=ax)
plot_path = os.path.join(REPORTS_DIRPATH, 'conf_matrix.png')
fig.savefig(plot_path)
neptune.send_image('diagnostics', plot_path)
fig, ax = plt.subplots(figsize=(16, 12))
sk_metrics.plot_roc(valid_preds['TARGET'],
valid_preds[['preds_neg', 'preds_pos']],
ax=ax)
plot_path = os.path.join(REPORTS_DIRPATH, 'roc_auc.png')
fig.savefig(plot_path)
neptune.send_image('diagnostics', plot_path)
fig, ax = plt.subplots(figsize=(16, 12))
sk_metrics.plot_precision_recall(
valid_preds['TARGET'],
valid_preds[['preds_neg', 'preds_pos']],
ax=ax)
plot_path = os.path.join(REPORTS_DIRPATH, 'prec_recall.png')
fig.savefig(plot_path)
neptune.send_image('diagnostics', plot_path)
fig, ax = plt.subplots(figsize=(16, 12))
plot_prediction_distribution(valid_preds['TARGET'],
valid_preds['preds_pos'],
ax=ax)
plot_path = os.path.join(REPORTS_DIRPATH, 'preds_dist.png')
fig.savefig(plot_path)
neptune.send_image('diagnostics', plot_path)
best_auc = -1.0 * results.fun
best_params = results.x
neptune.send_metric('valid_auc', best_auc)
neptune.set_property('best_params', str(to_named_params(best_params)))
# log results
skopt.dump(results, os.path.join(REPORTS_DIRPATH, 'skopt_results.pkl'))
neptune.send_artifact(
os.path.join(REPORTS_DIRPATH, 'skopt_results.pkl'))
# log diagnostic plots
fig, ax = plt.subplots(figsize=(16, 12))
skopt.plots.plot_convergence(results, ax=ax)
fig.savefig(os.path.join(REPORTS_DIRPATH, 'convergence.png'))
neptune.send_image('diagnostics',
os.path.join(REPORTS_DIRPATH, 'convergence.png'))
axes = skopt.plots.plot_evaluations(results)
fig = plt.figure(figsize=(16, 12))
fig = axes2fig(axes, fig)
fig.savefig(os.path.join(REPORTS_DIRPATH, 'evaluations.png'))
neptune.send_image('diagnostics',
os.path.join(REPORTS_DIRPATH, 'evaluations.png'))
axes = skopt.plots.plot_objective(results)
fig = plt.figure(figsize=(16, 12))
fig = axes2fig(axes, fig)
fig.savefig(os.path.join(REPORTS_DIRPATH, 'objective.png'))
neptune.send_image('diagnostics',
os.path.join(REPORTS_DIRPATH, 'objective.png'))