def setup_and_run(experiment_name, config, device):
datasets = [load_dataset(task_config) for task_config in config['tasks']]
for dataset in datasets:
dataset['train_iter'] = iter(dataset['train_loader'])
nets = [create_net(task_config, config['projectors']) for task_config in config['tasks']]
num_projectors = sum([net.num_projectors for net in nets])
if num_projectors != 0:
context = aggregate_context(nets)
bias = config['projectors'].get('bias', False)
alpha = config['optimization'].get('alpha', None)
ignore_context = config['projectors'].get('ignore_context', False)
frozen_context = config['projectors'].get('frozen_context', False)
omni_projector = OmniProjector(config['projectors']['context_size'],
config['projectors']['block_in'],
config['projectors']['block_out'],
num_projectors,
config['optimization']['alternatives_per_generation'] + 1,
context,
bias=bias,
alpha=alpha,
ignore_context=ignore_context,
frozen_context=frozen_context)
else:
omni_projector = None
optimize(experiment_name, datasets, nets, omni_projector, config, device)
if options.run_deep_learning_using_keras:
run_deep_learning(options)
else:
dataset_list = []
if options.dataset == Dataset.TWENTY_NEWS_GROUPS.name:
dataset_list.append(Dataset.TWENTY_NEWS_GROUPS.name)
elif options.dataset == Dataset.IMDB_REVIEWS.name:
dataset_list.append(Dataset.IMDB_REVIEWS.name)
else:
dataset_list.append(Dataset.TWENTY_NEWS_GROUPS.name)
dataset_list.append(Dataset.IMDB_REVIEWS.name)
for dataset in dataset_list:
X_train, y_train, X_test, y_test, target_names, data_train_size_mb, data_test_size_mb = load_dataset(
dataset, options)
vectorizer, X_train, X_test = extract_text_features(
X_train, X_test, options, data_train_size_mb,
data_test_size_mb)
if options.use_hashing:
feature_names = None
else:
feature_names = vectorizer.get_feature_names()
if options.chi2_select:
select_k_best_using_chi2(X_train, y_train, X_test,
feature_names, options)
results = []
def run_magnet_loss():
'''
Test function for the magnet loss
'''
m = 8
d = 8
k = 8
alpha = 1.0
batch_size = m * d
global plotter
plotter = VisdomLinePlotter(env_name=args.name)
trainloader, testloader, trainset, testset, n_train = load_dataset(args)
emb_dim = 2
n_epochs = 15
epoch_steps = len(trainloader)
n_steps = epoch_steps * 15
cluster_refresh_interval = epoch_steps
if args.mnist:
model = torch.nn.DataParallel(LeNet(emb_dim)).cuda()
if args.cifar10:
model = torch.nn.DataParallel(VGG(depth=16, num_classes=emb_dim))
print(model)
optimizer = optim.Adam(model.parameters(), lr=args.lr)
minibatch_magnet_loss = MagnetLoss()
images = getattr(trainset, 'train_data')
labels = getattr(trainset, 'train_labels')
# Get initial embedding
initial_reps = compute_reps(model, trainset, 400)
if args.cifar10:
labels = np.array(labels, dtype=np.float32)
# Create batcher
batch_builder = ClusterBatchBuilder(labels, k, m, d)
batch_builder.update_clusters(initial_reps)
batch_losses = []
batch_example_inds, batch_class_inds = batch_builder.gen_batch()
trainloader.sampler.batch_indices = batch_example_inds
_ = model.train()
losses = AverageMeter()
for i in tqdm(range(n_steps)):
for batch_idx, (img, target) in enumerate(trainloader):
img = Variable(img).cuda()
target = Variable(target).cuda()
optimizer.zero_grad()
output, features = model(img)
batch_loss, batch_example_losses = minibatch_magnet_loss(
output, batch_class_inds, m, d, alpha)
batch_loss.backward()
optimizer.step()
# Update loss index
batch_builder.update_losses(batch_example_inds, batch_example_losses)
batch_losses.append(batch_loss.data[0])
if not i % 1000:
print(i, batch_loss)
if not i % cluster_refresh_interval:
print("Refreshing clusters")
reps = compute_reps(model, trainset, 400)
batch_builder.update_clusters(reps)
if not i % 2000:
n_plot = 10000
plot_embedding(compute_reps(model, trainset, 400)[:n_plot],
labels[:n_plot],
name=i)
batch_example_inds, batch_class_inds = batch_builder.gen_batch()
trainloader.sampler.batch_indices = batch_example_inds
losses.update(batch_loss, 1)
# Log the training loss
if args.visdom:
plotter.plot('loss', 'train', i, losses.avg.data[0])
# Plot loss curve
plot_smooth(batch_losses, "batch-losses")
def main():
# Decide device to use
device = 'cuda' if torch.cuda.is_available() else 'cpu'
# Log path: verify existence of checkpoint dir, or create it
if not osp.exists(args.checkpoint): ## what does actuallty does?? ****
os.makedirs(args.checkpoint)
print_to_log('\t '.join(LOG_HEADERS), LOG_PATH)
# Reproducibility
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed) ## what does a seed function actually does??
if device == 'cuda':
torch.cuda.manual_seed_all(args.seed)
# txt file with all params
print_training_params(args, osp.join(args.checkpoint, 'params.txt'))
# Get datasets and dataloaders
trainloader, testloader, trainset, testset, train_labels, test_labels, \
distrib_params, num_classes = load_dataset(args, magnet_training=True)
# Params for each epoch
epoch_steps = len(trainloader)
print_freq = int(np.ceil(0.01 *
epoch_steps)) ## what does np.ceil() does here??
cluster_refresh_interval = epoch_steps - 1
# Create model
model = MODEL_INIT(num_classes=num_classes).to(device)
print(model)
if args.pretrained_path is not None:
model = copy_pretrained_model(
model,
args.pretrained_path) ## from where is this function from ??
# Optimizer and scheduler
optimizer = optim.Adam(model.parameters(), lr=args.lr)
scheduler = MultiStepLR(
optimizer,
milestones=args.milestones) ## i have to learn what this thing does
# Define criterion
criterion = MagnetLoss(args.alpha)
# Get batch builders and magnet_data dict
magnet_assertions(train_labels, args.k, L=args.L, m=args.m)
# Get batch builders
batch_builder = get_batch_builders(model,
trainset,
train_labels,
args.k,
args.m,
args.d,
device,
dataset_name=args.dataset)
# Get magnet_data dict
magnet_data = get_magnet_data(batch_builder, device, args, model,
criterion, trainset, train_labels)
if args.evaluate_ckpt is not None:
final_attack_eval(
model,
testloader,
testset,
test_labels,
checkpoint=args.checkpoint,
distrib_params=distrib_params,
device=device,
evaluate_ckpt=args.evaluate_ckpt,
alpha_step=ALPHA_STEP,
L=args.L,
seed=args.seed,
normalize_probs=not args.not_normalize,
restarts=args.restarts,
attack_iters=args.iterations,
)
return
best_acc = -np.inf ## what does this thing do??
# Iterate through epochs
for epoch in range(args.epochs):
model, batch_builder, magnet_data, train_acc, test_acc, train_loss, \
test_loss = train_epoch(
model, epoch, optimizer, trainloader, device, trainset,
train_labels, testset, test_labels, batch_builder, print_freq,
cluster_refresh_interval, criterion, magnet_data,
distrib_params, hardcoded_eps=HARDCODED_EPS,
minibatch_replays=args.minibatch_replays,
actual_trades=args.actual_trades
)
best_acc = max(best_acc, test_acc)
# Report epoch and save current model
report_epoch_and_save(args.checkpoint, epoch, model, train_acc,
test_acc, train_loss, test_loss, magnet_data,
args.save_all)
# Update log with results
update_log(optimizer, epoch, train_loss, train_acc, test_loss,
test_acc, LOG_PATH)
# Update scheduler
scheduler.step() ## what is the scheduler step here ??
# Report best accuracy of all training
print(f"Best accuracy: {best_acc:4.3f}")
def run_deep_learning_KerasDL1(options):
manage_logs(options)
logging.info("Program started...")
dataset_list = get_dataset_list(options)
results = {}
for dataset in dataset_list:
X_train, y_train, X_test, y_test, target_names, data_train_size_mb, data_test_size_mb = load_dataset(dataset, options)
y_test, y_train = one_hot_enconder(dataset, options, y_test, y_train)
vectorizer, X_train, X_test = extract_text_features(X_train, X_test, options, data_train_size_mb, data_test_size_mb)
input_dim = X_train.shape[1] # Number of features
model = Sequential()
model.add(layers.Dense(10, input_dim=input_dim, activation='relu'))
add_output_layer(dataset, model, options)
compile_model(model)
if not options.epochs:
if dataset == Dataset.TWENTY_NEWS_GROUPS.name:
epochs = 10
elif dataset == Dataset.IMDB_REVIEWS.name and options.use_imdb_multi_class_labels:
epochs = 2
else: # IMDB_REVIEWS using binary classification
epochs = 1
start = time()
if not options.epochs:
print('\n\nNUMBER OF EPOCHS USED: {}\n'.format(epochs))
history = model.fit(X_train, y_train, epochs=epochs, verbose=False, validation_data=(X_test, y_test), batch_size=10)
else:
print('\n\nNUMBER OF EPOCHS USED: {}\n'.format(options.epochs))
history = model.fit(X_train, y_train, epochs=options.epochs, verbose=False, validation_data=(X_test, y_test), batch_size=10)
training_time = time() - start
training_loss, training_accuracy = model.evaluate(X_train, y_train, verbose=False)
start = time()
test_loss, test_accuracy = model.evaluate(X_test, y_test, verbose=False)
test_time = time() - start
algorithm_name = "Deep Learning using Keras 1 (KERAS_DL1)"
print_results(dataset, algorithm_name, training_loss, training_accuracy, test_loss, test_accuracy, training_time, test_time)
plt.style.use('ggplot')
if dataset == Dataset.TWENTY_NEWS_GROUPS.name:
plot_history(history, 'KERAS_DL1', '20 NEWS')
elif dataset == Dataset.IMDB_REVIEWS.name:
plot_history(history, 'KERAS_DL1', 'IMDB')
print('\n')
results[dataset] = dataset, algorithm_name, training_loss, training_accuracy, test_accuracy, training_time, test_time
return results
def run_deep_learning_KerasDL2(options):
logging.info("Program started...")
dataset_list = get_dataset_list(options)
results = {}
for dataset in dataset_list:
X_train, y_train, X_test, y_test, target_names, data_train_size_mb, data_test_size_mb = load_dataset(dataset, options)
X_train = apply_nltk_feature_extraction(X_train, options, label='X_train')
X_test = apply_nltk_feature_extraction(X_test, options, label='X_test')
y_test, y_train = one_hot_enconder(dataset, options, y_test, y_train)
print('\t===> Tokenizer: fit_on_texts(X_train)')
max_features = 6000
tokenizer = Tokenizer(num_words=max_features)
tokenizer.fit_on_texts(X_train)
list_tokenized_train = tokenizer.texts_to_sequences(X_train)
maxlen = 130
print('\t===> X_train = pad_sequences(list_tokenized_train, maxlen={})'.format(max_features))
X_t = pad_sequences(list_tokenized_train, maxlen=maxlen)
y = y_train
embed_size = 128
print('\t===> Create Keras model')
model = Sequential()
model.add(Embedding(max_features, embed_size))
model.add(Bidirectional(LSTM(32, return_sequences=True)))
model.add(GlobalMaxPool1D())
model.add(Dense(20, activation="relu"))
model.add(Dropout(0.05))
add_output_layer(dataset, model, options)
compile_model(model)
batch_size = 100
if not options.epochs:
if dataset == Dataset.TWENTY_NEWS_GROUPS.name:
epochs = 15
elif dataset == Dataset.IMDB_REVIEWS.name and options.use_imdb_multi_class_labels:
epochs = 2
else: # IMDB_REVIEWS using binary classification
epochs = 3
# Test the model
print('\t===> Tokenizer: fit_on_texts(X_test)')
list_sentences_test = X_test
list_tokenized_test = tokenizer.texts_to_sequences(list_sentences_test)
print('\t===> X_test = pad_sequences(list_sentences_test, maxlen={})'.format(max_features))
X_te = pad_sequences(list_tokenized_test, maxlen=maxlen)
# Train the model
start = time()
if not options.epochs:
print('\n\nNUMBER OF EPOCHS USED: {}\n'.format(epochs))
history = model.fit(X_t, y, batch_size=batch_size, epochs=epochs, validation_data=(X_te, y_test))
else:
print('\n\nNUMBER OF EPOCHS USED: {}\n'.format(options.epochs))
history = model.fit(X_t, y, batch_size=batch_size, epochs=options.epochs, validation_data=(X_te, y_test))
training_time = time() - start
print('\t=====> Test the model: model.predict()')
prediction = model.predict(X_te)
y_pred = (prediction > 0.5)
print_classification_report(options, y_pred, y_test, target_names)
print_ml_metrics(options, y_pred, y_test)
print_confusion_matrix(options, y_pred, y_test)
training_loss, training_accuracy = model.evaluate(X_t, y, verbose=False)
start = time()
test_loss, test_accuracy = model.evaluate(X_te, y_test, verbose=False)
test_time = time() - start
algorithm_name = "Deep Learning using Keras 2 (KERAS_DL2)"
print_results(dataset, algorithm_name, training_loss, training_accuracy, test_loss, test_accuracy, training_time, test_time)
plt.style.use('ggplot')
if dataset == Dataset.TWENTY_NEWS_GROUPS.name:
plot_history(history, 'KERAS_DL2', '20 NEWS')
elif dataset == Dataset.IMDB_REVIEWS.name:
plot_history(history, 'KERAS_DL2', 'IMDB')
print('\n')
results[dataset] = dataset, algorithm_name, training_loss, training_accuracy, test_accuracy, training_time, test_time
return results
def load_task(task_config):
dataset = load_dataset(tasks_config['dataset'])
model = create_model(tasks_config['model'])
return (dataset, model)
def main():
########################################################################
# Set-up
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
print(sys.executable)
torch.manual_seed(TORCH_SEED)
torch.cuda.manual_seed(TORCH_SEED)
np.random.seed(NUMPY_SEED)
use_gpu = torch.cuda.is_available()
# use_gpu = False
device = torch.device("cuda" if use_gpu else "cpu")
print('GPU id: {}, name: {}'.format(
GPU_ID, torch.cuda.get_device_name(torch.cuda.current_device())))
trainloader, testloader, trainset, testset, num_classes = load_dataset(
dataset_choice, batch_size_train, batch_size_test)
classes = np.arange(0, 10)
# classes = [1,2,3,4,5,6,7,8,9,0]
if train:
since = time.time()
# Define a Convolution Neural Network
net = MnistModel(embedding_dim)
if dataset_choice == 'ONLINE_PRODUCTS':
net = Net(embedding_dim)
net = net.to(device)
# Define a Loss function and optimizer
# cross_entropy = nn.CrossEntropyLoss()
cross_entropy = nn.NLLLoss()
center_loss_weight = cl_weight
center_loss_module = CenterLoss(num_classes, embedding_dim,
center_loss_weight)
center_loss_module = center_loss_module.to(device)
if use_gpu:
center_loss_module = center_loss_module.cuda()
repulsive_loss_weight = rl_weight
repulsive_loss_margin = rl_margin
repulsive_loss_module = RepulsiveLoss(num_classes, embedding_dim,
repulsive_loss_margin,
repulsive_loss_weight)
repulsive_loss_module = repulsive_loss_module.to(device)
if use_gpu:
repulsive_loss_module = repulsive_loss_module.cuda()
criterion = [cross_entropy, center_loss_module, repulsive_loss_module]
optimizer_net = optim.Adam(net.parameters(), lr=cross_entropy_lr)
optimizer_center = optim.SGD(center_loss_module.parameters(),
lr=center_loss_lr)
optimizer = [optimizer_net, optimizer_center]
for epoch in range(num_epochs): # loop over the dataset multiple times
_, centers = train_epoch(net, trainloader, criterion, optimizer,
epoch, num_classes, batch_size_train,
device, use_gpu, show_plots,
embedding_dim)
print('Finished Training')
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
torch.save(net.state_dict(), model_save_path)
else:
net = MnistModel()
if use_gpu:
net = net.cuda()
net.load_state_dict(torch.load(model_save_path))
########################################################################
# Run the network on the test data
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# Test for one batch:
embeddings_one_batch, labels_one_batch = test_one_batch(
net, testloader, classes, use_gpu, Show=show_misclassified)
# Test on the whole dataset:
accuracy = test(net, testloader, device, use_gpu)
# Classes that performed well, and the classes that did not:
test_classwise(net, testloader, classes, device, use_gpu)
# Test for retrieval
k = 3
test_retrieval(net, testloader, device, k, use_gpu)
########################################################################
# Show embeddings
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# load up data
x_data = embeddings_one_batch.data.cpu().numpy()
y_data = labels_one_batch.cpu().numpy()
# convert image data to float64 matrix. float64 is need for bh_sne
x_data = np.asarray(x_data).astype('float64')
x_data = x_data.reshape((x_data.shape[0], -1))
# perform t-SNE embedding
# vis_data = tsne(x_data)
vis_data = pca(x_data, 2)
# plot the result
if show_plots:
visualize_better(vis_data, y_data)
# logging
if log:
from utils.my_logging import save_run_info, prepare_log_dir
log_dir = prepare_log_dir('logs')
log_string = 'Dataset: {}\tEpochs: {}\tBatch size: {}\tEmbedding dim: {}\tCenter loss weigth: {:.3f}' \
'\tRepulsive loss weigth: {:.3f}\tCross entropy learning rate: {:.5f}\t' \
'Center loss learning rate: {:.4f}\tRepulsive loss margin: {:.2f}\tAccuracy: {:.3f}'. \
format(dataset_choice, num_epochs, batch_size_train, embedding_dim, cl_weight, rl_weight, cross_entropy_lr, center_loss_lr,
rl_margin, accuracy)
save_run_info(log_string, log_dir)
plot_data_better(vis_data, y_data, log_dir=log_dir)
def heatmap_plotting(print_correlation_matrix=True, plot_heatmap_values=True, show_plotting=True, save_plotting=True,
plotting_path='heatmap.png', load_dataset_with_extra_pre_processing=True, save_csv_correlation_matrix=False):
# Dataset list
datasets = [Datasets.IONOSPHERE, Datasets.ADULT, Datasets.WINE_QUALITY, Datasets.BREAST_CANCER_DIAGNOSIS]
for dataset_name in datasets:
if dataset_name == Datasets.IONOSPHERE:
path = os.path.join(os.getcwd(), 'datasets/data/ionosphere/ionosphere.data')
if load_dataset_with_extra_pre_processing:
X_np, y_np = load_ionosphere()
else: # load all dataset columns
X_np, y_np = load_dataset(path, header=None)
X = pd.DataFrame(data=X_np)
if dataset_name == Datasets.ADULT:
if load_dataset_with_extra_pre_processing:
X_np, y_np = load_adult(run_one_hot_encoder=False)
else:
X_np, y_np = load_adult(run_one_hot_encoder=True)
X = pd.DataFrame(data=X_np.astype(float))
if load_dataset_with_extra_pre_processing:
if dataset_name == Datasets.ADULT:
X.columns = ['age', 'workclass', 'fnlwgt', 'education', 'education-num', 'marital-status',
'occupation', 'relationship', 'race', 'sex',
'hours-per-week', 'native-country']
if dataset_name == Datasets.WINE_QUALITY:
X_np, y_np = load_wine_quality()
X = pd.DataFrame(data=X_np)
X.columns = ['fixed acidity', 'volatile acidity', 'citric acid', 'residual sugar', 'chlorides',
'free sulfur dioxide', 'total sulfur dioxide', 'density', 'pH', 'sulphates', 'alcohol']
if dataset_name == Datasets.BREAST_CANCER_DIAGNOSIS:
path = os.path.join(os.getcwd(), 'datasets/data/breast-cancer-wisconsin/breast-cancer-wisconsin.data')
if load_dataset_with_extra_pre_processing:
X_np, y_np = load_breast_cancer_diagnosis()
else: # load all dataset columns
X_np, y_np = load_dataset(path, header=None, remove_question_mark=True)
X = pd.DataFrame(data=X_np.astype(float))
if load_dataset_with_extra_pre_processing:
X.columns = ['Clump Thickness', 'Uniformity of Cell Size', 'Uniformity of Cell Shape', 'Marginal Adhesion', 'Single Epithelial Cell Size', 'Bare Nuclei', 'Bland Chromatin', 'Normal Nucleoli', 'Mitoses']
else: # load all dataset columns
X.columns = ['Sample code number', 'Clump Thickness', 'Uniformity of Cell Size', 'Uniformity of Cell Shape', 'Marginal Adhesion', 'Single Epithelial Cell Size', 'Bare Nuclei', 'Bland Chromatin', 'Normal Nucleoli', 'Mitoses']
sns.set(style="white")
# Compute the correlation matrix
corr = X.corr()
if print_correlation_matrix:
print('\nCorrelation matrix:\n', corr)
# Generate a custom diverging colormap
cmap = sns.diverging_palette(220, 10, as_cmap=True)
heatmap_plot = sns.heatmap(corr, xticklabels=True, yticklabels=True, annot=plot_heatmap_values, cmap=cmap,
vmax=.3, center=0, square=True, linewidths=.5, cbar_kws={"shrink": .5})
ax = plt.axes()
file_name = dataset_name.name
if plot_heatmap_values:
file_name = file_name + ' with values, '
else:
file_name = file_name + ' without values, '
if load_dataset_with_extra_pre_processing:
file_name = file_name + ' with extra pre-processing'
else:
file_name = file_name + ' without extra pre-processing'
if save_csv_correlation_matrix:
corr.to_csv(os.path.join(plotting_path, file_name) + '.csv')
if dataset_name == Datasets.ADULT and not load_dataset_with_extra_pre_processing:
ax.set_title(dataset_name.name + ' using One-Hot-Enconder')
else:
ax.set_title(dataset_name.name)
if show_plotting:
plt.show()
if save_plotting:
fig = heatmap_plot.get_figure()
fig.savefig(os.path.join(plotting_path, file_name))
