def detect(vae, dataset, sample_size, device):
"""
Compute the anomaly score for a single review,
see algorithm 4 in https://pdfs.semanticscholar.org/0611/46b1d7938d7a8dae70e3531a00fceb3c78e8.pdf
Args
vae: trained vae model
data: tensor of size (1, )
sample_size: number of z~N(\mu,\sigma) to sample
device:
Returns
Anomaly score: List[List[str]] where reviews are sorted by reconstruction loss
"""
data_loader = utils.batch_loader(dataset=dataset, batch_size=1, shuffle=True)
sents_recon_loss = []
for batch_idx, batch_data in data_loader:
# sents is tensor of size (sents_len, batch_size)
sent = vae.vocab.vocab_entry.to_input_tensor(batch_data, device=device)
wun = torch.ones([1, sample_size], device=device)
sents = torch.mm(sent.float(), wun).long()
sents_len = torch.tensor([len(sent)]*sample_size, device=device)
sents_hat, mu, log_sd = vae.forward(sents, sents_len)
loss = loss_function(sents_hat, sents, mu, log_sd, vae.padding_idx, 1)
sents_recon_loss.append((batch_data, loss.item()/len(sent)))
if batch_idx % 100 == 0:
print('Progress: [{}/{} ({:.0f}%)]'.format(batch_idx, len(dataset), 100*batch_idx/len(dataset)))
sents_recon_loss.sort(key=lambda x:x[1], reverse=True)
return sents_recon_loss
def train(self, num_epochs: int):
"""
Training loop for model.
Implements stochastic gradient descent with num_epochs passes over the train dataset.
Returns:
train_history: a dictionary containing loss and accuracy over all training steps
val_history: a dictionary containing loss and accuracy over a selected set of steps
"""
# Utility variables
num_batches_per_epoch = self.X_train.shape[0] // self.batch_size
num_steps_per_val = num_batches_per_epoch // 5
# A tracking value of loss over all training steps
train_history = dict(loss={}, accuracy={})
val_history = dict(loss={}, accuracy={})
global_step = 0
dummy_counter = 0
lowest_loss = np.inf
# value to track the progress
last_epoch = -1
for epoch in range(num_epochs):
train_loader = utils.batch_loader(self.X_train,
self.Y_train,
self.batch_size,
shuffle=self.shuffle_dataset)
for X_batch, Y_batch in iter(train_loader):
loss = self.train_step(X_batch, Y_batch)
# Track training loss continuously
train_history["loss"][global_step] = loss
# Track validation loss / accuracy every time we progress 20% through the dataset
if global_step % num_steps_per_val == 0:
val_loss, accuracy_train, accuracy_val = self.validation_step(
)
train_history["accuracy"][global_step] = accuracy_train
val_history["loss"][global_step] = val_loss
val_history["accuracy"][global_step] = accuracy_val
# Keeping track of the progress of the training
if (last_epoch != epoch):
print(epoch)
last_epoch = epoch
# You can access the validation loss in val_history["loss"]
if (self.early_stopping and
(lowest_loss < val_history["loss"][global_step])):
dummy_counter += 1
if (dummy_counter >= 10):
print(
f'Early stopping kicked in at:{epoch}, with loss value: {val_history["loss"][global_step]}, lowest value = {lowest_loss}'
)
return train_history, val_history
else:
dummy_counter = 0
lowest_loss = val_history["loss"][global_step]
global_step += 1
return train_history, val_history
def train(
self,
num_epochs: int):
"""
Training loop for model.
Implements stochastic gradient descent with num_epochs passes over the train dataset.
Returns:
train_history: a dictionary containing loss and accuracy over all training steps
val_history: a dictionary containing loss and accuracy over a selected set of steps
"""
# Utility variables
num_batches_per_epoch = self.X_train.shape[0] // self.batch_size
num_steps_per_val = num_batches_per_epoch // 5
# A tracking value of loss over all training steps
train_history = dict(
loss={},
accuracy={}
)
val_history = dict(
loss={},
accuracy={}
)
global_step = 0
counter = 0
lowest_val_loss = 1
for epoch in range(num_epochs):
train_loader = utils.batch_loader(
self.X_train, self.Y_train, self.batch_size, shuffle=self.shuffle_dataset)
for X_batch, Y_batch in iter(train_loader):
loss = self.train_step(X_batch, Y_batch)
# Track training loss continuously
train_history["loss"][global_step] = loss
# Track validation loss / accuracy every time we progress 20% through the dataset
if global_step % num_steps_per_val == 0:
val_loss, accuracy_train, accuracy_val = self.validation_step()
train_history["accuracy"][global_step] = accuracy_train
val_history["loss"][global_step] = val_loss
val_history["accuracy"][global_step] = accuracy_val
# TODO (Task 2d): Implement early stopping here.
# You can access the validation loss in val_history["loss"]
if (val_loss>lowest_val_loss):
counter = counter + 1
else:
lowest_val_loss = val_loss
counter = 0
if (counter == 10):
break
global_step += 1
if (counter == 10):
print("Stopped at epoch:", epoch)
break
return train_history, val_history
def train(self, num_epochs: int):
"""
Training loop for model.
Implements stochastic gradient descent with num_epochs passes over the train dataset.
Returns:
train_history: a dictionary containing loss and accuracy over all training steps
val_history: a dictionary containing loss and accuracy over a selected set of steps
"""
# Utility variables
num_batches_per_epoch = self.X_train.shape[0] // self.batch_size
num_steps_per_val = num_batches_per_epoch // 5
# A tracking value of loss over all training steps
train_history = dict(loss={}, accuracy={})
val_history = dict(loss={}, accuracy={})
global_step = 0
early_stopping = False
previous_validation_loss = float('inf')
validation_steps = 0
for epoch in range(num_epochs):
if (early_stopping is True):
break
train_loader = utils.batch_loader(self.X_train,
self.Y_train,
self.batch_size,
shuffle=self.shuffle_dataset)
for X_batch, Y_batch in iter(train_loader):
loss = self.train_step(X_batch, Y_batch)
# Track training loss continuously
train_history["loss"][global_step] = loss
# Track validation loss / accuracy every time we progress 20% through the dataset
if global_step % num_steps_per_val == 0:
val_loss, accuracy_train, accuracy_val = self.validation_step(
)
train_history["accuracy"][global_step] = accuracy_train
val_history["loss"][global_step] = val_loss
val_history["accuracy"][global_step] = accuracy_val
validation_steps += 1
if (previous_validation_loss >
val_history["loss"][global_step]):
previous_validation_loss = val_history["loss"][
global_step]
validation_steps = 0
if (validation_steps == 10):
print("Early stopping on epoch: ", epoch)
early_stopping = True
break
global_step += 1
return train_history, val_history
def train(self, num_epochs: int):
"""
Training loop for model.
Implements stochastic gradient descent with num_epochs passes over the train dataset.
Returns:
train_history: a dictionary containing loss and accuracy over all training steps
val_history: a dictionary containing loss and accuracy over a selected set of steps
"""
# Utility variables
num_batches_per_epoch = self.X_train.shape[0] // self.batch_size
num_steps_per_val = num_batches_per_epoch // 5
# A tracking value of loss over all training steps
train_history = dict(loss={}, accuracy={})
val_history = dict(loss={}, accuracy={})
global_step = 0
# Dummy variable, used for Early Stopping.
local_step = 0
val_min = 1000000
for epoch in range(num_epochs):
train_loader = utils.batch_loader(self.X_train,
self.Y_train,
self.batch_size,
shuffle=self.shuffle_dataset)
for X_batch, Y_batch in iter(train_loader):
loss = self.train_step(X_batch, Y_batch)
# Track training loss continuously
train_history["loss"][global_step] = loss
# Track validation loss / accuracy every time we progress 20% through the dataset
if global_step % num_steps_per_val == 0:
val_loss, accuracy_train, accuracy_val = self.validation_step(
)
train_history["accuracy"][global_step] = accuracy_train
val_history["loss"][global_step] = val_loss
val_history["accuracy"][global_step] = accuracy_val
# TODO (Task 2d): Implement early stopping here.
# You can access the validation loss in val_history["loss"]
if local_step < 10:
if val_history["loss"][global_step] > val_min:
local_step += 1
else:
val_min = val_history["loss"][global_step]
local_step = 0
else:
# val_history["loss"][global_step] = val_min
print(f"Early stopping after {epoch} epochs")
return train_history, val_history
global_step += 1
return train_history, val_history
def train(
self,
num_epochs: int):
"""
Training loop for model.
Implements stochastic gradient descent with num_epochs passes over the train dataset.
Returns:
train_history: a dictionary containing loss and accuracy over all training steps
val_history: a dictionary containing loss and accuracy over a selected set of steps
"""
# Utility variables
num_batches_per_epoch = self.X_train.shape[0] // self.batch_size
num_steps_per_val = num_batches_per_epoch // 5
# A tracking value of loss over all training steps
train_history = dict(
loss={},
accuracy={}
)
val_history = dict(
loss={},
accuracy={}
)
global_step = 0
best_loss = 1
count = 0
for epoch in tqdm(range(num_epochs)):
train_loader = utils.batch_loader(
self.X_train, self.Y_train, self.batch_size, shuffle=self.shuffle_dataset)
for X_batch, Y_batch in iter(train_loader):
loss = self.train_step(X_batch, Y_batch)
# Track training loss continuously
train_history["loss"][global_step] = loss
# Track validation loss / accuracy every time we progress 20% through the dataset
if global_step % num_steps_per_val == 0:
val_loss, accuracy_train, accuracy_val = self.validation_step()
train_history["accuracy"][global_step] = accuracy_train
val_history["loss"][global_step] = val_loss
val_history["accuracy"][global_step] = accuracy_val
# Early stopping
if val_loss < best_loss:
best_loss = val_loss
count = 0
else:
count +=1
if count >= 50 and self.early_stop:
print("Early stopping at ", global_step, "epoch ", epoch)
return train_history, val_history
global_step += 1
return train_history, val_history
def train(self, num_epochs: int):
"""
Training loop for model.
Implements stochastic gradient descent with num_epochs passes over the train dataset.
Returns:
train_history: a dictionary containing loss and accuracy over all training steps
val_history: a dictionary containing loss and accuracy over a selected set of steps
"""
# Utility variables
num_batches_per_epoch = self.X_train.shape[0] // self.batch_size
num_steps_per_val = num_batches_per_epoch // 5
# A tracking value of loss over all training steps
train_history = dict(loss={}, accuracy={})
val_history = dict(loss={}, accuracy={})
global_step = 0
counter = 0
prev_best_loss = np.inf
for epoch in range(num_epochs):
train_loader = utils.batch_loader(self.X_train,
self.Y_train,
self.batch_size,
shuffle=self.shuffle_dataset)
for X_batch, Y_batch in iter(train_loader):
loss = self.train_step(X_batch, Y_batch)
# Track training loss continuously
train_history["loss"][global_step] = loss
# Track validation loss / accuracy every time we progress 20% through the dataset
if global_step % num_steps_per_val == 0:
val_loss, accuracy_train, accuracy_val = self.validation_step(
)
train_history["accuracy"][global_step] = accuracy_train
val_history["loss"][global_step] = val_loss
val_history["accuracy"][global_step] = accuracy_val
if self.stop_at_count:
# No improvement
if prev_best_loss < val_loss:
counter += 1
else:
counter = 0
prev_best_loss = val_loss
# We have reached max number of passes trough dataset without improvement
if counter == self.stop_at_count:
print(
f"We went trough {epoch} of {num_epochs} before stopping"
)
return train_history, val_history
global_step += 1
return train_history, val_history
def train(self, num_epochs: int):
"""
Training loop for model.
Implements stochastic gradient descent with num_epochs passes over the train dataset.
Returns:
train_history: a dictionary containing loss and accuracy over all training steps
val_history: a dictionary containing loss and accuracy over a selected set of steps
"""
# Utility variables
num_batches_per_epoch = self.X_train.shape[0] // self.batch_size
num_steps_per_val = num_batches_per_epoch // 5
# A tracking value of loss over all training steps
train_history = dict(loss={}, accuracy={})
val_history = dict(loss={}, accuracy={})
global_step = 0
lowest_val_loss = np.inf
early_stop_counter = 0
for epoch in range(num_epochs):
train_loader = utils.batch_loader(self.X_train,
self.Y_train,
self.batch_size,
shuffle=self.shuffle_dataset)
for X_batch, Y_batch in iter(train_loader):
loss = self.train_step(X_batch, Y_batch)
# Track training loss continuously
train_history["loss"][global_step] = loss
# Track validation loss / accuracy every time we progress 20% through the dataset
if global_step % num_steps_per_val == 0:
val_loss, accuracy_train, accuracy_val = self.validation_step(
)
train_history["accuracy"][global_step] = accuracy_train
val_history["loss"][global_step] = val_loss
val_history["accuracy"][global_step] = accuracy_val
# Task 2d - Early Stopping implemented here with
if val_loss < lowest_val_loss:
lowest_val_loss = val_loss
early_stop_counter = 0
else:
early_stop_counter += 1
if early_stop_counter >= 10:
print("early stopping implemented in epoch: ", epoch)
#return train_history, val_history #remove comment to have early_stopping
# You can access the validation loss in val_history["loss"]
global_step += 1
return train_history, val_history
def train(self, num_epochs: int):
"""
Training loop for model.
Implements stochastic gradient descent with num_epochs passes over the train dataset.
Returns:
train_history: a dictionary containing loss and accuracy over all training steps
val_history: a dictionary containing loss and accuracy over a selected set of steps
"""
# Utility variables
num_batches_per_epoch = self.X_train.shape[0] // self.batch_size
num_steps_per_val = num_batches_per_epoch // 5
# A tracking value of loss over all training steps
train_history = dict(loss={}, accuracy={})
val_history = dict(loss={}, accuracy={})
min_val_loss = 1
es_tracker = 0
global_step = 0
for epoch in range(num_epochs):
train_loader = utils.batch_loader(self.X_train,
self.Y_train,
self.batch_size,
shuffle=self.shuffle_dataset)
for X_batch, Y_batch in iter(train_loader):
loss = self.train_step(X_batch, Y_batch)
# Track training loss continuously
train_history["loss"][global_step] = loss
# Track validation loss / accuracy every time we progress 20% through the dataset
if global_step % num_steps_per_val == 0:
val_loss, accuracy_train, accuracy_val = self.validation_step(
)
train_history["accuracy"][global_step] = accuracy_train
val_history["loss"][global_step] = val_loss
val_history["accuracy"][global_step] = accuracy_val
# TODO (Task 2d): Implement early stopping here.
# You can access the validation loss in val_history["loss"]
# need to find average of 10 previous
if val_loss < min_val_loss:
min_val_loss = val_loss
es_tracker = 0
else:
es_tracker += 1
if es_tracker > 9:
# Early stop triggered
print('ES Triggered, Step: ', global_step, epoch)
break
global_step += 1
if es_tracker > 9:
break
return train_history, val_history
def train(self, num_epochs: int):
"""
Training loop for model.
Implements stochastic gradient descent with num_epochs passes over the train dataset.
Returns:
train_history: a dictionary containing loss and accuracy over all training steps
val_history: a dictionary containing loss and accuracy over a selected set of steps
"""
# Utility variables
num_batches_per_epoch = self.X_train.shape[0] // self.batch_size
num_steps_per_val = num_batches_per_epoch // 5
# A tracking value of loss over all training steps
train_history = dict(loss={}, accuracy={})
val_history = dict(loss={}, accuracy={})
do_early_stopping = True
global_step = 0
early_stop_counter = 0
lowest_val_loss = sys.maxsize
for epoch in range(num_epochs):
train_loader = utils.batch_loader(self.X_train,
self.Y_train,
self.batch_size,
shuffle=self.shuffle_dataset)
for X_batch, Y_batch in iter(train_loader):
loss = self.train_step(X_batch, Y_batch)
# Track training loss continuously
train_history["loss"][global_step] = loss
# Track validation loss / accuracy every time we progress 20% through the dataset
if global_step % num_steps_per_val == 0:
val_loss, accuracy_train, accuracy_val = self.validation_step(
)
train_history["accuracy"][global_step] = accuracy_train
val_history["loss"][global_step] = val_loss
val_history["accuracy"][global_step] = accuracy_val
# TODO: Implement early stopping (copy from last assignment)
if do_early_stopping:
if val_loss < lowest_val_loss:
lowest_val_loss = val_loss
early_stop_counter = 0
else:
early_stop_counter += 1
if early_stop_counter >= 50:
print("Early stop at epoch ", epoch,
" and global step ", global_step)
return train_history, val_history
global_step += 1
return train_history, val_history
def train(self, num_epochs: int):
"""
Training loop for model.
Implements stochastic gradient descent with num_epochs passes over the train dataset.
Returns:
train_history: a dictionary containing loss and accuracy over all training steps
val_history: a dictionary containing loss and accuracy over a selected set of steps
"""
# Utility variables
num_batches_per_epoch = self.X_train.shape[0] // self.batch_size
num_steps_per_val = num_batches_per_epoch // 5
# A tracking value of loss over all training steps
train_history = dict(loss={}, accuracy={})
val_history = dict(loss={}, accuracy={})
global_step = 0
stop = False
best = 1000000000 #Some great number representing infinity
best_arr = [
] #This will hold the best values for each epoch (validation loss)
for epoch in tqdm(range(num_epochs)):
train_loader = utils.batch_loader(self.X_train,
self.Y_train,
self.batch_size,
shuffle=self.shuffle_dataset)
for X_batch, Y_batch in iter(train_loader):
loss = self.train_step(X_batch, Y_batch)
# Track training loss continuously
train_history["loss"][global_step] = loss
# Track validation loss / accuracy every time we progress 20% through the dataset
if global_step % num_steps_per_val == 0:
val_loss, accuracy_train, accuracy_val = self.validation_step(
)
train_history["accuracy"][global_step] = accuracy_train
val_history["loss"][global_step] = val_loss
val_history["accuracy"][global_step] = accuracy_val
best_this_epoch = np.min(val_history["loss"][global_step])
if best_this_epoch < best:
best = best_this_epoch
best_arr.append(best_this_epoch)
# TODO (Task 2d): Implement early stopping here.
# You can access the validation loss in val_history["loss"]
if (best not in best_arr[-10:] and len(best_arr) >= 10):
return train_history, val_history
global_step += 1
return train_history, val_history
def train(self, num_epochs: int):
"""
Training loop for model.
Implements stochastic gradient descent with num_epochs passes over the train dataset.
Returns:
train_history: a dictionary containing loss and accuracy over all training steps
val_history: a dictionary containing loss and accuracy over a selected set of steps
"""
# Utility variables
num_batches_per_epoch = self.X_train.shape[0] // self.batch_size
num_steps_per_val = num_batches_per_epoch // 5
# A tracking value of loss over all training steps
train_history = dict(loss={}, accuracy={})
val_history = dict(loss={}, accuracy={})
# variables for early stopping
counter = 0
times = 50
lowest_val_loss = float('inf')
global_step = 0
for epoch in range(num_epochs):
train_loader = utils.batch_loader(self.X_train,
self.Y_train,
self.batch_size,
shuffle=self.shuffle_dataset)
for X_batch, Y_batch in iter(train_loader):
loss = self.train_step(X_batch, Y_batch)
# Track training loss continuously
train_history["loss"][global_step] = loss
# Track validation loss / accuracy every time we progress 20% through the dataset
if global_step % num_steps_per_val == 0:
val_loss, accuracy_train, accuracy_val = self.validation_step(
)
train_history["accuracy"][global_step] = accuracy_train
val_history["loss"][global_step] = val_loss
val_history["accuracy"][global_step] = accuracy_val
# TODO: Implement early stopping (copy from last assignment)
counter += 1
if (val_history["loss"][global_step] < lowest_val_loss):
lowest_val_loss = val_history["loss"][global_step]
counter = 0
if (counter > times):
print(f'Training stopped after {epoch} epochs')
return train_history, val_history
global_step += 1
return train_history, val_history
def train(self, num_epochs: int):
"""
Training loop for model.
Implements stochastic gradient descent with num_epochs passes over the train dataset.
Returns:
train_history: a dictionary containing loss and accuracy over all training steps
val_history: a dictionary containing loss and accuracy over a selected set of steps
"""
# Utility variables
num_batches_per_epoch = self.X_train.shape[0] // self.batch_size
num_steps_per_val = num_batches_per_epoch // 5
# A tracking value of loss over all training steps
train_history = dict(loss={}, accuracy={})
val_history = dict(loss={}, accuracy={})
global_step = 0
for epoch in range(num_epochs):
train_loader = utils.batch_loader(self.X_train,
self.Y_train,
self.batch_size,
shuffle=self.shuffle_dataset)
for X_batch, Y_batch in iter(train_loader):
loss = self.train_step(X_batch, Y_batch)
# Track training loss continuously
train_history["loss"][global_step] = loss
# Track validation loss / accuracy every time we progress 20% through the dataset
if global_step % num_steps_per_val == 0:
val_loss, accuracy_train, accuracy_val = self.validation_step(
)
train_history["accuracy"][global_step] = accuracy_train
val_history["loss"][global_step] = val_loss
val_history["accuracy"][global_step] = accuracy_val
# TODO: Implement early stopping (copy from last assignment)
# You can access the validation loss in val_history["loss"]
# min_step: the training step when the loss was minimal
if len(val_history["loss"]) > 50:
min_step = min(val_history["loss"].items(),
key=lambda x: x[1])[0]
# if min_step is not in the last 10 loss result, then stop the training
if min_step not in sorted(val_history['loss'].keys(),
reverse=True)[:50]:
print(
"Early stopping after {} epoch".format(epoch))
return train_history, val_history
global_step += 1
return train_history, val_history
def train(
self,
num_epochs: int):
"""
Training loop for model.
Implements stochastic gradient descent with num_epochs passes over the train dataset.
Returns:
train_history: a dictionary containing loss and accuracy over all training steps
val_history: a dictionary containing loss and accuracy over a selected set of steps
"""
# Utility variables
num_batches_per_epoch = self.X_train.shape[0] // self.batch_size
num_steps_per_val = num_batches_per_epoch // 5
# A tracking value of loss over all training steps
train_history = dict(
loss={},
accuracy={}
)
val_history = dict(
loss={},
accuracy={}
)
global_step = 0
for epoch in range(num_epochs):
train_loader = utils.batch_loader(
self.X_train, self.Y_train, self.batch_size, shuffle=self.shuffle_dataset)
for X_batch, Y_batch in iter(train_loader):
loss = self.train_step(X_batch, Y_batch)
# Track training loss continuously
train_history["loss"][global_step] = loss
# Track validation loss / accuracy every time we progress 20% through the dataset
if global_step % num_steps_per_val == 0:
val_loss, accuracy_train, accuracy_val = self.validation_step()
train_history["accuracy"][global_step] = accuracy_train
val_history["loss"][global_step] = val_loss
val_history["accuracy"][global_step] = accuracy_val
# early stopping: stop training when loss didn't improved in last 60 steps
if global_step > 60 and np.argmin(list(val_history["loss"].values())[-61:]) == 0:
print(f'Early stopping after {epoch} epochs')
print(f'Final Accuracy: {val_history["accuracy"][global_step]}')
print(f'Final Training Loss: {train_history["loss"][global_step]}')
return train_history, val_history
global_step += 1
return train_history, val_history
def train(epoch, vae, optimizer, dataset, batch_size, device, keep_rate, a, b):
KLweight = utils.KLweight(a, b)
vae.train()
total_loss = 0
data_loader = utils.batch_loader(dataset=dataset, batch_size=batch_size, shuffle=True)
dataset_size = len(dataset)
#batch_data is List[List[str]]
for batch_idx, batch_data in data_loader:
# sents is tensor of size (sents_len, batch_size)
sents = vae.vocab.vocab_entry.to_input_tensor(batch_data, device=device)
batch_data_dropout = utils.word_dropout(batch_data, keep_rate, '<unk>')
sents_dropout = vae.vocab.vocab_entry.to_input_tensor(batch_data_dropout, device=device)
sents_len = torch.tensor([len(sent) for sent in batch_data], device=device)
optimizer.zero_grad()
sents_hat, mu, log_sd = vae.forward(sents_dropout, sents_len)
x = epoch+min(dataset_size, (batch_idx+1)*batch_size)/dataset_size
loss = loss_function(sents_hat, sents, mu, log_sd, vae.padding_idx, KLweight.weight(x))
loss.backward()
total_loss += loss.item()
optimizer.step()
print('Train Epoch: {} [{}/{} ({:.0f}%)]\t Loss: {:.6f}'.format(epoch, min(dataset_size, (batch_idx+1)*batch_size), dataset_size, 100.*min(1, (batch_idx+1)*batch_size/dataset_size), loss.item()))
