def __init__(self, kick=KICK, snare=SNARE, tom1=TOM1, tom2=TOM2, hihat=HIHAT, crash=CRASH, ride=RIDE):
self.kick_file = kick
self.snare_file = snare
self.tom1_file = tom1
self.tom2_file = tom2
self.hihat_file = hihat
self.crash_file = crash
self.ride_file = ride
drumset_files = [
self.kick_file,
self.snare_file,
self.tom1_file,
self.tom2_file,
self.hihat_file,
self.crash_file,
self.ride_file,
]
Sampler.__init__(self, drumset_files)
self.kick = self[0]
self.snare = self[1]
self.tom1 = self[2]
self.tom2 = self[3]
self.hihat = self[4]
self.crash = self[5]
self.ride = self[6]
def _init_al_dataset(self):
""" Initialises dataset for active learning
"""
self._init_dataset()
train_dataset = self.datasets['train']
dataset_size = len(train_dataset)
self.budget = math.ceil(self.budget_frac * dataset_size)
Sampler.__init__(self, config,
self.budget) # TODO: Weird place to initialise this
all_indices = set(np.arange(dataset_size))
k_initial = math.ceil(len(all_indices) * self.initial_budget_frac)
initial_indices = random.sample(list(all_indices), k=k_initial)
sampler_init = data.sampler.SubsetRandomSampler(
initial_indices) # need to sample from training dataset
self.labelled_dataloader = data.DataLoader(train_dataset,
sampler=sampler_init,
batch_size=self.batch_size,
drop_last=True)
self.val_dataloader = data.DataLoader(self.datasets['valid'],
batch_size=self.batch_size,
drop_last=False)
self.test_dataloader = data.DataLoader(self.datasets['test'],
batch_size=self.batch_size,
drop_last=False)
return all_indices, initial_indices
def _init_al_data(self):
""" Initialises train, validation and test sets for active learning including partitions """
self._init_dataset()
train_dataset = self.datasets['train']
dataset_size = len(train_dataset)
self.budget = math.ceil(
self.budget_frac *
dataset_size) # currently can only have a fixed budget size
Sampler.__init__(self, self.budget)
all_indices = set(np.arange(dataset_size))
k_initial = math.ceil(len(all_indices) * self.initial_budget_frac)
initial_indices = random.sample(list(all_indices), k=k_initial)
sampler_init = data.sampler.SubsetRandomSampler(
initial_indices) # need to sample from training dataset
self.labelled_dataloader = data.DataLoader(train_dataset,
sampler=sampler_init,
batch_size=self.batch_size,
drop_last=True)
self.val_dataloader = data.DataLoader(self.datasets['valid'],
batch_size=self.batch_size,
shuffle=True,
drop_last=False)
self.test_dataloader = data.DataLoader(self.datasets['test'],
batch_size=self.batch_size,
shuffle=True,
drop_last=False)
print(
f'{datetime.now()}: Dataloaders sizes: Train {len(self.labelled_dataloader)} Valid {len(self.val_dataloader)} Test {len(self.test_dataloader)}'
)
return all_indices, initial_indices
def __init__(self, data_map, params, likelihood_constraint, no):
"""
Initializes the uniform sampler
Parameters
----------
likelihood_constraint: float
name says it all
no : int
Number of likelihood evaluations until this point
"""
self.LC = likelihood_constraint
self.number = no
Sampler.__init__(self, data_map, params)
def __init__(self, data_map, params, likelihood_constraint, no):
"""
Initializes the uniform sampler
Parameters
----------
likelihood_constraint: float
name says it all
no : int
Number of likelihood evaluations until this point
"""
self.LC = likelihood_constraint
self.number = no
Sampler.__init__(self, data_map, params)
def __init__(self, data_map, params, to_evolve, likelihood_constraint, no):
"""
Initializes the Metropolis sampler
Parameters
----------
to_evolve : object
The sample to evolve
likelihood_constraint: float
name says it all
no : int
Number of likelihood evaluations until this point
"""
Sampler.__init__(self, data_map, params)
self.source = to_evolve
self.LC = likelihood_constraint
self.step = params['dispersion']
self.number = no
def __init__(self,
target_obj,
target_region,
explr_p,
infeasible_action_value,
policy=None):
Sampler.__init__(self, policy, target_region)
self.explr_p = explr_p
self.k_nearest_neighbor_to_best_point = None
self.best_evaled_x = None
self.infeasible_action_value = infeasible_action_value
# todo there is pick_distance and place_distance function in the gtamp_utils
def distance_fn(x, y):
pick_dist = utils.pick_parameter_distance(target_obj, x[0:6],
y[0:6])
place_dist = utils.place_parameter_distance(x[6:], y[6:])
return pick_dist + place_dist
self.distance_fn = distance_fn
self.distance_fn = lambda x, y: np.linalg.norm(x - y)
def __init__(self, data_map, params, to_evolve, likelihood_constraint, no):
"""
Initializes the Metropolis sampler
Parameters
----------
to_evolve : object
The sample to evolve
likelihood_constraint: float
name says it all
no : int
Number of likelihood evaluations until this point
"""
Sampler.__init__(self, data_map, params)
self.source = to_evolve
self.LC = likelihood_constraint
self.step = params['dispersion']
self.number = no
def __init__(self, data_map, params, active_samples, likelihood_constraint,
enlargement, no):
"""
Initializes the clustered ellipsoidal sampler.
Parameters
----------
data_map : array
Data map we want to sample on
params : dict
Dictionary of various parameters
active_samples : array
The array containing the active samples for this clustered sampling phase
likelihood_constraint : float
Name says it all
enlargement : float
Enlargement factor for ellipsoid
no : int
Number of likelihood calculations until the current sampling phase
"""
Sampler.__init__(self, data_map, params)
self.eps = params['eps']
self.minPts = params['minPts']
self.params = params
#self.points = copy.deepcopy(active_samples)
self.LC = likelihood_constraint
self.enlargement = 1.5
self.clustered_point_set = None
self.number_of_clusters = None
activepoint_set = self.build_set(active_samples)
self.ellipsoid_set = self.optimal_ellipsoids(activepoint_set)
self.total_vol = None
self.number = no
def __init__(self, num_samples, num_steps):
Sampler.__init__(self, num_samples)
self.num_steps = num_steps
def _init_data(self, batch_size=None):
kfold_xval = False
self.x_y_pair_name = 'seq_label_pairs_enc' if self.data_name == 'ag_news' else 'seq_tags_pairs_enc' # Key in dataset - semantically correct for the task at hand.
if batch_size is None:
batch_size = self.config['Train']['batch_size']
# Load pre-processed data
path_data = os.path.join('/home/tyler/Desktop/Repos/s-vaal/data', self.task_type, self.data_name, 'data.json')
path_vocab = os.path.join('/home/tyler/Desktop/Repos/s-vaal/data', self.task_type, self.data_name, 'vocabs.json')
self.preprocess_data = load_json(path_data)
self.vocab = load_json(path_vocab) # Required for decoding sequences for interpretations. TODO: Find suitable location... or leave be...
self.vocab_size = len(self.vocab['words']) # word vocab is used for model dimensionality setting + includes special characters (EOS, SOS< UNK, PAD)
self.tagset_size = len(self.vocab['tags']) # this includes special characters (EOS, SOS, UNK, PAD)
self.datasets = dict()
if kfold_xval:
# Perform k-fold cross-validation
# Join all datasets and then randomly assign train/val/test
print('Performing k-fold x-val')
for split in self.data_splits:
print(self.preprocess_data[split][self.x_y_pair_name])
else:
for split in self.data_splits:
# Access data
split_data = self.preprocess_data[split][self.x_y_pair_name]
# Convert lists of encoded sequences into tensors and stack into one large tensor
split_seqs = torch.stack([torch.tensor(enc_pair[0]) for key, enc_pair in split_data.items()])
split_tags = torch.stack([torch.tensor(enc_pair[1]) for key, enc_pair in split_data.items()])
# Create torch dataset from tensors
split_dataset = RealDataset(sequences=split_seqs, tags=split_tags)
# Add to dictionary
self.datasets[split] = split_dataset #split_dataloader
# Create torch dataloader generator from dataset
if split == 'test':
self.test_dataloader = DataLoader(dataset=split_dataset, batch_size=batch_size, shuffle=True, num_workers=0)
if split == 'valid':
self.val_dataloader = DataLoader(dataset=split_dataset, batch_size=batch_size, shuffle=True, num_workers=0)
print(f'{datetime.now()}: Data loaded succesfully')
train_dataset = self.datasets['train']
dataset_size = len(train_dataset)
self.budget = math.ceil(self.budget_frac*dataset_size)
Sampler.__init__(self, self.budget)
all_indices = set(np.arange(dataset_size))
k_initial = math.ceil(len(all_indices)*self.budget_frac)
initial_indices = random.sample(list(all_indices), k=k_initial)
sampler_init = torch.utils.data.sampler.SubsetRandomSampler(initial_indices)
self.labelled_dataloader = DataLoader(train_dataset, sampler=sampler_init, batch_size=self.batch_size, drop_last=True)
self.val_dataloader = DataLoader(self.datasets['valid'], batch_size=self.batch_size, shuffle=True, drop_last=False)
self.test_dataloader = DataLoader(self.datasets['test'], batch_size=self.batch_size, shuffle=True, drop_last=False)
print(f'{datetime.now()}: Dataloaders sizes: Train {len(self.labelled_dataloader)} Valid {len(self.val_dataloader)} Test {len(self.test_dataloader)}')
return all_indices, initial_indices
def __init__(self, num_samples, num_steps, total_sz=0):
Sampler.__init__(self, num_samples)
self.num_steps = num_steps
self.total_sz = total_sz
def train_single_cycle(self, parameterisation=None):
""" Performs a single cycle of the active learning routine at a specified data split for optimisation purposes"""
print(parameterisation)
if parameterisation is None:
params = {
'tl': self.model_config['TaskLearner']['Parameters'],
'svae': self.model_config['SVAE']['Parameters'],
'disc': self.model_config['Discriminator']['Parameters']
}
params.update({
'tl_learning_rate':
self.model_config['TaskLearner']['learning_rate']
})
params.update({
'svae_learning_rate':
self.model_config['SVAE']['learning_rate']
})
params.update({
'disc_learning_rate':
self.model_config['Discriminator']['learning_rate']
})
params.update({'epochs': self.config['Train']['epochs']})
params.update({'k': self.model_config['SVAE']['k']})
params.update({'x0': self.model_config['SVAE']['x0']})
params.update({
'adv_hyperparameter':
self.model_config['SVAE']['adversarial_hyperparameter']
})
if parameterisation:
params = {
'epochs':
parameterisation['epochs'],
'batch_size':
parameterisation['batch_size'],
'tl_learning_rate':
self.model_config['TaskLearner']
['learning_rate'], #parameterisation['tl_learning_rate'],
'svae_learning_rate':
parameterisation['svae_learning_rate'],
'disc_learning_rate':
parameterisation['disc_learning_rate'],
'k':
parameterisation['svae_k'],
'x0':
parameterisation['svae_x0'],
'adv_hyperparameter':
self.model_config['SVAE']
['adversarial_hyperparameter'], #parameterisation['svae_adv_hyperparameter'],
'tl':
self.model_config['TaskLearner']['Parameters'],
#{'embedding_dim': parameterisation['tl_embedding_dim'],
# 'hidden_dim': parameterisation['tl_hidden_dim'],
# 'rnn_type': parameterisation['tl_rnn_type']},
'svae': {
'embedding_dim':
parameterisation['svae_embedding_dim'],
'hidden_dim':
parameterisation['svae_hidden_dim'],
'word_dropout':
parameterisation['svae_word_dropout'],
'embedding_dropout':
parameterisation['svae_embedding_dropout'],
'num_layers':
self.model_config['SVAE']['Parameters']
['num_layers'], #parameterisation['svae_num_layers'],
'bidirectional':
self.model_config['SVAE']['Parameters']
['bidirectional'], #parameterisation['svae_bidirectional'],
'rnn_type':
self.model_config['SVAE']['Parameters']
['rnn_type'], #parameterisation['svae_rnn_type'],
'latent_size':
parameterisation['latent_size']
},
'disc': {
'z_dim': parameterisation['latent_size'],
'fc_dim': parameterisation['disc_fc_dim']
}
}
split = self.config['Train']['cycle_frac']
print(f'\n{datetime.now()}\nSplit: {split*100:0.0f}%')
meta = f' {self.al_mode} run x data split {split*100:0.0f}'
self._init_dataset()
train_dataset = self.datasets['train']
dataset_size = len(train_dataset)
self.budget = math.ceil(
self.budget_frac *
dataset_size) # currently can only have a fixed budget size
Sampler.__init__(self, self.budget)
all_indices = set(
np.arange(dataset_size)) # indices of all samples in train
k_initial = math.ceil(
len(all_indices) *
split) # number of initial samples given split size
initial_indices = random.sample(
list(all_indices),
k=k_initial) # random sample of initial indices from train
sampler_init = data.sampler.SubsetRandomSampler(
initial_indices
) # sampler method for dataloader to randomly sample initial indices
current_indices = list(
initial_indices) # current set of labelled indices
dataloader_l = data.DataLoader(train_dataset,
sampler=sampler_init,
batch_size=params['batch_size'],
drop_last=True)
dataloader_v = data.DataLoader(self.datasets['valid'],
batch_size=params['batch_size'],
shuffle=True,
drop_last=False)
dataloader_t = data.DataLoader(self.datasets['test'],
batch_size=params['batch_size'],
shuffle=True,
drop_last=False)
unlabelled_indices = np.setdiff1d(
list(all_indices), current_indices
) # set of unlabelled indices (all - initial/current)
unlabelled_sampler = data.sampler.SubsetRandomSampler(
unlabelled_indices
) # sampler method for dataloader to randomly sample unlabelled indices
dataloader_u = data.DataLoader(self.datasets['train'],
sampler=unlabelled_sampler,
batch_size=params['batch_size'],
drop_last=False)
print(
f'{datetime.now()}: Indices - Labelled {len(current_indices)} Unlabelled {len(unlabelled_indices)} Total {len(all_indices)}'
)
# Initialise models
task_learner = TaskLearner(**params['tl'],
vocab_size=self.vocab_size,
tagset_size=self.tagset_size,
task_type=self.task_type).to(self.device)
if self.task_type == 'SEQ':
tl_loss_fn = nn.NLLLoss().to(self.device)
if self.task_type == 'CLF':
tl_loss_fn = nn.CrossEntropyLoss().to(self.device)
tl_optim = optim.SGD(
task_learner.parameters(),
lr=params['tl_learning_rate']) #, momentum=0, weight_decay=0.1)
task_learner.train()
svae = SVAE(**params['svae'],
vocab_size=self.vocab_size).to(self.device)
discriminator = Discriminator(**params['disc']).to(self.device)
dsc_loss_fn = nn.BCELoss().to(self.device)
svae_optim = optim.Adam(svae.parameters(),
lr=params['svae_learning_rate'])
dsc_optim = optim.Adam(discriminator.parameters(),
lr=params['disc_learning_rate'])
# Training Modes
svae.train()
discriminator.train()
print(f'{datetime.now()}: Models initialised successfully')
# Perform AL training and sampling
early_stopping = EarlyStopping(
patience=self.config['Train']['es_patience'],
verbose=True,
path="checkpoints/checkpoint.pt")
dataset_size = len(dataloader_l) + len(
dataloader_u) if dataloader_u is not None else len(dataloader_l)
train_iterations = dataset_size * (params['epochs'] + 1)
print(
f'{datetime.now()}: Dataset size (batches) {dataset_size} Training iterations (batches) {train_iterations}'
)
step = 0
epoch = 1
for train_iter in tqdm(range(train_iterations),
desc='Training iteration'):
batch_sequences_l, batch_lengths_l, batch_tags_l = next(
iter(dataloader_l))
if torch.cuda.is_available():
batch_sequences_l = batch_sequences_l.to(self.device)
batch_lengths_l = batch_lengths_l.to(self.device)
batch_tags_l = batch_tags_l.to(self.device)
if dataloader_u is not None:
batch_sequences_u, batch_lengths_u, _ = next(
iter(dataloader_u))
batch_sequences_u = batch_sequences_u.to(self.device)
batch_length_u = batch_lengths_u.to(self.device)
# Strip off tag padding and flatten
# Don't do sequences here as its done in the forward pass of the seq2seq models
batch_tags_l = trim_padded_seqs(batch_lengths=batch_lengths_l,
batch_sequences=batch_tags_l,
pad_idx=self.pad_idx).view(-1)
# Task Learner Step
tl_optim.zero_grad()
tl_preds = task_learner(batch_sequences_l, batch_lengths_l)
tl_loss = tl_loss_fn(tl_preds, batch_tags_l)
tl_loss.backward()
tl_optim.step()
# Used in SVAE and Discriminator
batch_size_l = batch_sequences_l.size(0)
batch_size_u = batch_sequences_u.size(0)
# SVAE Step
for i in range(self.svae_iterations):
logp_l, mean_l, logv_l, z_l = svae(batch_sequences_l,
batch_lengths_l)
NLL_loss_l, KL_loss_l, KL_weight_l = svae.loss_fn(
logp=logp_l,
target=batch_sequences_l,
length=batch_lengths_l,
mean=mean_l,
logv=logv_l,
anneal_fn=self.model_config['SVAE']['anneal_function'],
step=step,
k=params['k'],
x0=params['x0'])
logp_u, mean_u, logv_u, z_u = svae(batch_sequences_u,
batch_lengths_u)
NLL_loss_u, KL_loss_u, KL_weight_u = svae.loss_fn(
logp=logp_u,
target=batch_sequences_u,
length=batch_lengths_u,
mean=mean_u,
logv=logv_u,
anneal_fn=self.model_config['SVAE']['anneal_function'],
step=step,
k=params['k'],
x0=params['x0'])
# VAE loss
svae_loss_l = (NLL_loss_l +
KL_weight_l * KL_loss_l) / batch_size_l
svae_loss_u = (NLL_loss_u +
KL_weight_u * KL_loss_u) / batch_size_u
# Adversary loss - trying to fool the discriminator!
dsc_preds_l = discriminator(z_l) # mean_l
dsc_preds_u = discriminator(z_u) # mean_u
dsc_real_l = torch.ones(batch_size_l)
dsc_real_u = torch.ones(batch_size_u)
if torch.cuda.is_available():
dsc_real_l = dsc_real_l.to(self.device)
dsc_real_u = dsc_real_u.to(self.device)
adv_dsc_loss_l = dsc_loss_fn(dsc_preds_l, dsc_real_l)
adv_dsc_loss_u = dsc_loss_fn(dsc_preds_u, dsc_real_u)
adv_dsc_loss = adv_dsc_loss_l + adv_dsc_loss_u
total_svae_loss = svae_loss_u + svae_loss_l + params[
'adv_hyperparameter'] * adv_dsc_loss
svae_optim.zero_grad()
total_svae_loss.backward()
svae_optim.step()
if i < self.svae_iterations - 1:
batch_sequences_l, batch_lengths_l, _ = next(
iter(dataloader_l))
batch_sequences_u, batch_length_u, _ = next(
iter(dataloader_u))
if torch.cuda.is_available():
batch_sequences_l = batch_sequences_l.to(self.device)
batch_lengths_l = batch_lengths_l.to(self.device)
batch_sequences_u = batch_sequences_u.to(self.device)
batch_length_u = batch_length_u.to(self.device)
step += 1
# Discriminator Step
for j in range(self.dsc_iterations):
with torch.no_grad():
_, _, _, z_l = svae(batch_sequences_l, batch_lengths_l)
_, _, _, z_u = svae(batch_sequences_u, batch_lengths_u)
dsc_preds_l = discriminator(z_l)
dsc_preds_u = discriminator(z_u)
dsc_real_l = torch.ones(batch_size_l)
dsc_real_u = torch.zeros(batch_size_u)
if torch.cuda.is_available():
dsc_real_l = dsc_real_l.to(self.device)
dsc_real_u = dsc_real_u.to(self.device)
# Discriminator wants to minimise the loss here
dsc_loss_l = dsc_loss_fn(dsc_preds_l, dsc_real_l)
dsc_loss_u = dsc_loss_fn(dsc_preds_u, dsc_real_u)
total_dsc_loss = dsc_loss_l + dsc_loss_u
dsc_optim.zero_grad()
total_dsc_loss.backward()
dsc_optim.step()
# Sample new batch of data while training adversarial network
if j < self.dsc_iterations - 1:
batch_sequences_l, batch_lengths_l, _ = next(
iter(dataloader_l))
batch_sequences_u, batch_length_u, _ = next(
iter(dataloader_u))
if torch.cuda.is_available():
batch_sequences_l = batch_sequences_l.to(self.device)
batch_lengths_l = batch_lengths_l.to(self.device)
batch_sequences_u = batch_sequences_u.to(self.device)
batch_length_u = batch_length_u.to(self.device)
if (train_iter % dataset_size == 0):
print("Initiating Early Stopping")
early_stopping(
tl_loss, task_learner
) # TODO: Review. Should this be the metric we early stop on?
if early_stopping.early_stop:
print(
f'Early stopping at {train_iter}/{train_iterations} training iterations'
)
break
if (train_iter > 0) & (epoch == 1
or train_iter % dataset_size == 0):
if train_iter % dataset_size == 0:
val_metrics = self.evaluation(task_learner=task_learner,
dataloader=dataloader_v,
task_type=self.task_type)
val_string = f'Task Learner ({self.task_type}) Validation ' + f'Scores:\nF1: Macro {val_metrics["f1 macro"]*100:0.2f}% Micro {val_metrics["f1 micro"]*100:0.2f}%\n' if self.task_type == 'SEQ' else f'Accuracy {val_metrics["accuracy"]*100:0.2f}'
print(val_string)
if (train_iter > 0) & (train_iter % dataset_size == 0):
# Completed an epoch
train_iter_str = f'Train Iter {train_iter} - Losses (TL-{self.task_type} {tl_loss:0.2f} | SVAE {total_svae_loss:0.2f} | Disc {total_dsc_loss:0.2f} | Learning rates: TL ({tl_optim.param_groups[0]["lr"]})'
print(train_iter_str)
print(f'Completed epoch: {epoch}')
epoch += 1
# Evaluation at the end of the first training cycle
test_metrics = self.evaluation(task_learner=task_learner,
dataloader=dataloader_t,
task_type='SEQ')
f1_macro_1 = test_metrics['f1 macro']
# SVAE and Discriminator need to be evaluated on the TL metric n+1 split from their current training split
# So, data needs to be sampled via SVAAL and then the TL retrained. The final metric from the retrained TL is then used to
# optimise the SVAE and Discriminator. For this optimisation problem, the TL parameters are fixed.
# Sample data via SVAE and Discriminator
sampled_indices, _, _ = self.sample_adversarial(
svae=svae,
discriminator=discriminator,
data=dataloader_u,
indices=unlabelled_indices,
cuda=True) # TODO: review usage of indices arg
# Add new samples to labelled dataset
current_indices = list(current_indices) + list(sampled_indices)
sampler = data.sampler.SubsetRandomSampler(current_indices)
self.labelled_dataloader = data.DataLoader(self.datasets['train'],
sampler=sampler,
batch_size=self.batch_size,
drop_last=True)
dataloader_l = self.labelled_dataloader # to maintain naming conventions
print(
f'{datetime.now()}: Indices - Labelled {len(current_indices)} Unlabelled {len(unlabelled_indices)} Total {len(all_indices)}'
)
task_learner = TaskLearner(**params['tl'],
vocab_size=self.vocab_size,
tagset_size=self.tagset_size,
task_type=self.task_type).to(self.device)
if self.task_type == 'SEQ':
tl_loss_fn = nn.NLLLoss().to(self.device)
if self.task_type == 'CLF':
tl_loss_fn = nn.CrossEntropyLoss().to(self.device)
tl_optim = optim.SGD(
task_learner.parameters(),
lr=params['tl_learning_rate']) #, momentum=0, weight_decay=0.1)
task_learner.train()
early_stopping = EarlyStopping(
patience=self.config['Train']['es_patience'],
verbose=True,
path="checkpoints/checkpoint.pt")
print(f'{datetime.now()}: Task Learner initialised successfully')
# Train Task Learner on adversarially selected samples
dataset_size = len(dataloader_l)
train_iterations = dataset_size * (params['epochs'] + 1)
epoch = 1
for train_iter in tqdm(range(train_iterations),
desc='Training iteration'):
batch_sequences_l, batch_lengths_l, batch_tags_l = next(
iter(dataloader_l))
if torch.cuda.is_available():
batch_sequences_l = batch_sequences_l.to(self.device)
batch_lengths_l = batch_lengths_l.to(self.device)
batch_tags_l = batch_tags_l.to(self.device)
# Strip off tag padding and flatten
# Don't do sequences here as its done in the forward pass of the seq2seq models
batch_tags_l = trim_padded_seqs(batch_lengths=batch_lengths_l,
batch_sequences=batch_tags_l,
pad_idx=self.pad_idx).view(-1)
# Task Learner Step
tl_optim.zero_grad()
tl_preds = task_learner(batch_sequences_l, batch_lengths_l)
tl_loss = tl_loss_fn(tl_preds, batch_tags_l)
tl_loss.backward()
tl_optim.step()
if (train_iter % dataset_size == 0):
print("Initiating Early Stopping")
early_stopping(
tl_loss, task_learner
) # TODO: Review. Should this be the metric we early stop on?
if early_stopping.early_stop:
print(
f'Early stopping at {train_iter}/{train_iterations} training iterations'
)
break
if (train_iter > 0) & (epoch == 1
or train_iter % dataset_size == 0):
if train_iter % dataset_size == 0:
val_metrics = self.evaluation(task_learner=task_learner,
dataloader=dataloader_v,
task_type=self.task_type)
val_string = f'Task Learner ({self.task_type}) Validation ' + f'Scores:\nF1: Macro {val_metrics["f1 macro"]*100:0.2f}% Micro {val_metrics["f1 micro"]*100:0.2f}%\n' if self.task_type == 'SEQ' else f'Accuracy {val_metrics["accuracy"]*100:0.2f}'
print(val_string)
if (train_iter > 0) & (train_iter % dataset_size == 0):
# Completed an epoch
train_iter_str = f'Train Iter {train_iter} - Losses (TL-{self.task_type} {tl_loss:0.2f} | Learning rate: TL ({tl_optim.param_groups[0]["lr"]})'
print(train_iter_str)
print(f'Completed epoch: {epoch}')
epoch += 1
# Compute test metrics
test_metrics = self.evaluation(task_learner=task_learner,
dataloader=dataloader_t,
task_type='SEQ')
print(
f'{datetime.now()}: Test Eval.: F1 Scores - Macro {test_metrics["f1 macro"]*100:0.2f}% Micro {test_metrics["f1 micro"]*100:0.2f}%'
)
# return test_metrics["f1 macro"]
# Should the output be maximum rate of the change between iter n and iter n+1 metrics? this makes more sense than just f1 macro?
f1_macro_diff = test_metrics['f1 macro'] - f1_macro_1
print(f'Macro f1 difference: {f1_macro_diff*100:0.2f}%')
return f1_macro_diff
def __init__(self, atype, target_region):
Sampler.__init__(self, atype, target_region, sampler=None)
