def __init__(self, kick=KICK, snare=SNARE, tom1=TOM1, tom2=TOM2, hihat=HIHAT, crash=CRASH, ride=RIDE): self.kick_file = kick self.snare_file = snare self.tom1_file = tom1 self.tom2_file = tom2 self.hihat_file = hihat self.crash_file = crash self.ride_file = ride drumset_files = [ self.kick_file, self.snare_file, self.tom1_file, self.tom2_file, self.hihat_file, self.crash_file, self.ride_file, ] Sampler.__init__(self, drumset_files) self.kick = self[0] self.snare = self[1] self.tom1 = self[2] self.tom2 = self[3] self.hihat = self[4] self.crash = self[5] self.ride = self[6]
def _init_al_dataset(self): """ Initialises dataset for active learning """ self._init_dataset() train_dataset = self.datasets['train'] dataset_size = len(train_dataset) self.budget = math.ceil(self.budget_frac * dataset_size) Sampler.__init__(self, config, self.budget) # TODO: Weird place to initialise this all_indices = set(np.arange(dataset_size)) k_initial = math.ceil(len(all_indices) * self.initial_budget_frac) initial_indices = random.sample(list(all_indices), k=k_initial) sampler_init = data.sampler.SubsetRandomSampler( initial_indices) # need to sample from training dataset self.labelled_dataloader = data.DataLoader(train_dataset, sampler=sampler_init, batch_size=self.batch_size, drop_last=True) self.val_dataloader = data.DataLoader(self.datasets['valid'], batch_size=self.batch_size, drop_last=False) self.test_dataloader = data.DataLoader(self.datasets['test'], batch_size=self.batch_size, drop_last=False) return all_indices, initial_indices
def _init_al_data(self): """ Initialises train, validation and test sets for active learning including partitions """ self._init_dataset() train_dataset = self.datasets['train'] dataset_size = len(train_dataset) self.budget = math.ceil( self.budget_frac * dataset_size) # currently can only have a fixed budget size Sampler.__init__(self, self.budget) all_indices = set(np.arange(dataset_size)) k_initial = math.ceil(len(all_indices) * self.initial_budget_frac) initial_indices = random.sample(list(all_indices), k=k_initial) sampler_init = data.sampler.SubsetRandomSampler( initial_indices) # need to sample from training dataset self.labelled_dataloader = data.DataLoader(train_dataset, sampler=sampler_init, batch_size=self.batch_size, drop_last=True) self.val_dataloader = data.DataLoader(self.datasets['valid'], batch_size=self.batch_size, shuffle=True, drop_last=False) self.test_dataloader = data.DataLoader(self.datasets['test'], batch_size=self.batch_size, shuffle=True, drop_last=False) print( f'{datetime.now()}: Dataloaders sizes: Train {len(self.labelled_dataloader)} Valid {len(self.val_dataloader)} Test {len(self.test_dataloader)}' ) return all_indices, initial_indices
def __init__(self, data_map, params, likelihood_constraint, no): """ Initializes the uniform sampler Parameters ---------- likelihood_constraint: float name says it all no : int Number of likelihood evaluations until this point """ self.LC = likelihood_constraint self.number = no Sampler.__init__(self, data_map, params)
def __init__(self, data_map, params, to_evolve, likelihood_constraint, no): """ Initializes the Metropolis sampler Parameters ---------- to_evolve : object The sample to evolve likelihood_constraint: float name says it all no : int Number of likelihood evaluations until this point """ Sampler.__init__(self, data_map, params) self.source = to_evolve self.LC = likelihood_constraint self.step = params['dispersion'] self.number = no
def __init__(self, target_obj, target_region, explr_p, infeasible_action_value, policy=None): Sampler.__init__(self, policy, target_region) self.explr_p = explr_p self.k_nearest_neighbor_to_best_point = None self.best_evaled_x = None self.infeasible_action_value = infeasible_action_value # todo there is pick_distance and place_distance function in the gtamp_utils def distance_fn(x, y): pick_dist = utils.pick_parameter_distance(target_obj, x[0:6], y[0:6]) place_dist = utils.place_parameter_distance(x[6:], y[6:]) return pick_dist + place_dist self.distance_fn = distance_fn self.distance_fn = lambda x, y: np.linalg.norm(x - y)
def __init__(self, data_map, params, active_samples, likelihood_constraint, enlargement, no): """ Initializes the clustered ellipsoidal sampler. Parameters ---------- data_map : array Data map we want to sample on params : dict Dictionary of various parameters active_samples : array The array containing the active samples for this clustered sampling phase likelihood_constraint : float Name says it all enlargement : float Enlargement factor for ellipsoid no : int Number of likelihood calculations until the current sampling phase """ Sampler.__init__(self, data_map, params) self.eps = params['eps'] self.minPts = params['minPts'] self.params = params #self.points = copy.deepcopy(active_samples) self.LC = likelihood_constraint self.enlargement = 1.5 self.clustered_point_set = None self.number_of_clusters = None activepoint_set = self.build_set(active_samples) self.ellipsoid_set = self.optimal_ellipsoids(activepoint_set) self.total_vol = None self.number = no
def __init__(self, num_samples, num_steps): Sampler.__init__(self, num_samples) self.num_steps = num_steps
def _init_data(self, batch_size=None): kfold_xval = False self.x_y_pair_name = 'seq_label_pairs_enc' if self.data_name == 'ag_news' else 'seq_tags_pairs_enc' # Key in dataset - semantically correct for the task at hand. if batch_size is None: batch_size = self.config['Train']['batch_size'] # Load pre-processed data path_data = os.path.join('/home/tyler/Desktop/Repos/s-vaal/data', self.task_type, self.data_name, 'data.json') path_vocab = os.path.join('/home/tyler/Desktop/Repos/s-vaal/data', self.task_type, self.data_name, 'vocabs.json') self.preprocess_data = load_json(path_data) self.vocab = load_json(path_vocab) # Required for decoding sequences for interpretations. TODO: Find suitable location... or leave be... self.vocab_size = len(self.vocab['words']) # word vocab is used for model dimensionality setting + includes special characters (EOS, SOS< UNK, PAD) self.tagset_size = len(self.vocab['tags']) # this includes special characters (EOS, SOS, UNK, PAD) self.datasets = dict() if kfold_xval: # Perform k-fold cross-validation # Join all datasets and then randomly assign train/val/test print('Performing k-fold x-val') for split in self.data_splits: print(self.preprocess_data[split][self.x_y_pair_name]) else: for split in self.data_splits: # Access data split_data = self.preprocess_data[split][self.x_y_pair_name] # Convert lists of encoded sequences into tensors and stack into one large tensor split_seqs = torch.stack([torch.tensor(enc_pair[0]) for key, enc_pair in split_data.items()]) split_tags = torch.stack([torch.tensor(enc_pair[1]) for key, enc_pair in split_data.items()]) # Create torch dataset from tensors split_dataset = RealDataset(sequences=split_seqs, tags=split_tags) # Add to dictionary self.datasets[split] = split_dataset #split_dataloader # Create torch dataloader generator from dataset if split == 'test': self.test_dataloader = DataLoader(dataset=split_dataset, batch_size=batch_size, shuffle=True, num_workers=0) if split == 'valid': self.val_dataloader = DataLoader(dataset=split_dataset, batch_size=batch_size, shuffle=True, num_workers=0) print(f'{datetime.now()}: Data loaded succesfully') train_dataset = self.datasets['train'] dataset_size = len(train_dataset) self.budget = math.ceil(self.budget_frac*dataset_size) Sampler.__init__(self, self.budget) all_indices = set(np.arange(dataset_size)) k_initial = math.ceil(len(all_indices)*self.budget_frac) initial_indices = random.sample(list(all_indices), k=k_initial) sampler_init = torch.utils.data.sampler.SubsetRandomSampler(initial_indices) self.labelled_dataloader = DataLoader(train_dataset, sampler=sampler_init, batch_size=self.batch_size, drop_last=True) self.val_dataloader = DataLoader(self.datasets['valid'], batch_size=self.batch_size, shuffle=True, drop_last=False) self.test_dataloader = DataLoader(self.datasets['test'], batch_size=self.batch_size, shuffle=True, drop_last=False) print(f'{datetime.now()}: Dataloaders sizes: Train {len(self.labelled_dataloader)} Valid {len(self.val_dataloader)} Test {len(self.test_dataloader)}') return all_indices, initial_indices
def __init__(self, num_samples, num_steps, total_sz=0): Sampler.__init__(self, num_samples) self.num_steps = num_steps self.total_sz = total_sz
def train_single_cycle(self, parameterisation=None): """ Performs a single cycle of the active learning routine at a specified data split for optimisation purposes""" print(parameterisation) if parameterisation is None: params = { 'tl': self.model_config['TaskLearner']['Parameters'], 'svae': self.model_config['SVAE']['Parameters'], 'disc': self.model_config['Discriminator']['Parameters'] } params.update({ 'tl_learning_rate': self.model_config['TaskLearner']['learning_rate'] }) params.update({ 'svae_learning_rate': self.model_config['SVAE']['learning_rate'] }) params.update({ 'disc_learning_rate': self.model_config['Discriminator']['learning_rate'] }) params.update({'epochs': self.config['Train']['epochs']}) params.update({'k': self.model_config['SVAE']['k']}) params.update({'x0': self.model_config['SVAE']['x0']}) params.update({ 'adv_hyperparameter': self.model_config['SVAE']['adversarial_hyperparameter'] }) if parameterisation: params = { 'epochs': parameterisation['epochs'], 'batch_size': parameterisation['batch_size'], 'tl_learning_rate': self.model_config['TaskLearner'] ['learning_rate'], #parameterisation['tl_learning_rate'], 'svae_learning_rate': parameterisation['svae_learning_rate'], 'disc_learning_rate': parameterisation['disc_learning_rate'], 'k': parameterisation['svae_k'], 'x0': parameterisation['svae_x0'], 'adv_hyperparameter': self.model_config['SVAE'] ['adversarial_hyperparameter'], #parameterisation['svae_adv_hyperparameter'], 'tl': self.model_config['TaskLearner']['Parameters'], #{'embedding_dim': parameterisation['tl_embedding_dim'], # 'hidden_dim': parameterisation['tl_hidden_dim'], # 'rnn_type': parameterisation['tl_rnn_type']}, 'svae': { 'embedding_dim': parameterisation['svae_embedding_dim'], 'hidden_dim': parameterisation['svae_hidden_dim'], 'word_dropout': parameterisation['svae_word_dropout'], 'embedding_dropout': parameterisation['svae_embedding_dropout'], 'num_layers': self.model_config['SVAE']['Parameters'] ['num_layers'], #parameterisation['svae_num_layers'], 'bidirectional': self.model_config['SVAE']['Parameters'] ['bidirectional'], #parameterisation['svae_bidirectional'], 'rnn_type': self.model_config['SVAE']['Parameters'] ['rnn_type'], #parameterisation['svae_rnn_type'], 'latent_size': parameterisation['latent_size'] }, 'disc': { 'z_dim': parameterisation['latent_size'], 'fc_dim': parameterisation['disc_fc_dim'] } } split = self.config['Train']['cycle_frac'] print(f'\n{datetime.now()}\nSplit: {split*100:0.0f}%') meta = f' {self.al_mode} run x data split {split*100:0.0f}' self._init_dataset() train_dataset = self.datasets['train'] dataset_size = len(train_dataset) self.budget = math.ceil( self.budget_frac * dataset_size) # currently can only have a fixed budget size Sampler.__init__(self, self.budget) all_indices = set( np.arange(dataset_size)) # indices of all samples in train k_initial = math.ceil( len(all_indices) * split) # number of initial samples given split size initial_indices = random.sample( list(all_indices), k=k_initial) # random sample of initial indices from train sampler_init = data.sampler.SubsetRandomSampler( initial_indices ) # sampler method for dataloader to randomly sample initial indices current_indices = list( initial_indices) # current set of labelled indices dataloader_l = data.DataLoader(train_dataset, sampler=sampler_init, batch_size=params['batch_size'], drop_last=True) dataloader_v = data.DataLoader(self.datasets['valid'], batch_size=params['batch_size'], shuffle=True, drop_last=False) dataloader_t = data.DataLoader(self.datasets['test'], batch_size=params['batch_size'], shuffle=True, drop_last=False) unlabelled_indices = np.setdiff1d( list(all_indices), current_indices ) # set of unlabelled indices (all - initial/current) unlabelled_sampler = data.sampler.SubsetRandomSampler( unlabelled_indices ) # sampler method for dataloader to randomly sample unlabelled indices dataloader_u = data.DataLoader(self.datasets['train'], sampler=unlabelled_sampler, batch_size=params['batch_size'], drop_last=False) print( f'{datetime.now()}: Indices - Labelled {len(current_indices)} Unlabelled {len(unlabelled_indices)} Total {len(all_indices)}' ) # Initialise models task_learner = TaskLearner(**params['tl'], vocab_size=self.vocab_size, tagset_size=self.tagset_size, task_type=self.task_type).to(self.device) if self.task_type == 'SEQ': tl_loss_fn = nn.NLLLoss().to(self.device) if self.task_type == 'CLF': tl_loss_fn = nn.CrossEntropyLoss().to(self.device) tl_optim = optim.SGD( task_learner.parameters(), lr=params['tl_learning_rate']) #, momentum=0, weight_decay=0.1) task_learner.train() svae = SVAE(**params['svae'], vocab_size=self.vocab_size).to(self.device) discriminator = Discriminator(**params['disc']).to(self.device) dsc_loss_fn = nn.BCELoss().to(self.device) svae_optim = optim.Adam(svae.parameters(), lr=params['svae_learning_rate']) dsc_optim = optim.Adam(discriminator.parameters(), lr=params['disc_learning_rate']) # Training Modes svae.train() discriminator.train() print(f'{datetime.now()}: Models initialised successfully') # Perform AL training and sampling early_stopping = EarlyStopping( patience=self.config['Train']['es_patience'], verbose=True, path="checkpoints/checkpoint.pt") dataset_size = len(dataloader_l) + len( dataloader_u) if dataloader_u is not None else len(dataloader_l) train_iterations = dataset_size * (params['epochs'] + 1) print( f'{datetime.now()}: Dataset size (batches) {dataset_size} Training iterations (batches) {train_iterations}' ) step = 0 epoch = 1 for train_iter in tqdm(range(train_iterations), desc='Training iteration'): batch_sequences_l, batch_lengths_l, batch_tags_l = next( iter(dataloader_l)) if torch.cuda.is_available(): batch_sequences_l = batch_sequences_l.to(self.device) batch_lengths_l = batch_lengths_l.to(self.device) batch_tags_l = batch_tags_l.to(self.device) if dataloader_u is not None: batch_sequences_u, batch_lengths_u, _ = next( iter(dataloader_u)) batch_sequences_u = batch_sequences_u.to(self.device) batch_length_u = batch_lengths_u.to(self.device) # Strip off tag padding and flatten # Don't do sequences here as its done in the forward pass of the seq2seq models batch_tags_l = trim_padded_seqs(batch_lengths=batch_lengths_l, batch_sequences=batch_tags_l, pad_idx=self.pad_idx).view(-1) # Task Learner Step tl_optim.zero_grad() tl_preds = task_learner(batch_sequences_l, batch_lengths_l) tl_loss = tl_loss_fn(tl_preds, batch_tags_l) tl_loss.backward() tl_optim.step() # Used in SVAE and Discriminator batch_size_l = batch_sequences_l.size(0) batch_size_u = batch_sequences_u.size(0) # SVAE Step for i in range(self.svae_iterations): logp_l, mean_l, logv_l, z_l = svae(batch_sequences_l, batch_lengths_l) NLL_loss_l, KL_loss_l, KL_weight_l = svae.loss_fn( logp=logp_l, target=batch_sequences_l, length=batch_lengths_l, mean=mean_l, logv=logv_l, anneal_fn=self.model_config['SVAE']['anneal_function'], step=step, k=params['k'], x0=params['x0']) logp_u, mean_u, logv_u, z_u = svae(batch_sequences_u, batch_lengths_u) NLL_loss_u, KL_loss_u, KL_weight_u = svae.loss_fn( logp=logp_u, target=batch_sequences_u, length=batch_lengths_u, mean=mean_u, logv=logv_u, anneal_fn=self.model_config['SVAE']['anneal_function'], step=step, k=params['k'], x0=params['x0']) # VAE loss svae_loss_l = (NLL_loss_l + KL_weight_l * KL_loss_l) / batch_size_l svae_loss_u = (NLL_loss_u + KL_weight_u * KL_loss_u) / batch_size_u # Adversary loss - trying to fool the discriminator! dsc_preds_l = discriminator(z_l) # mean_l dsc_preds_u = discriminator(z_u) # mean_u dsc_real_l = torch.ones(batch_size_l) dsc_real_u = torch.ones(batch_size_u) if torch.cuda.is_available(): dsc_real_l = dsc_real_l.to(self.device) dsc_real_u = dsc_real_u.to(self.device) adv_dsc_loss_l = dsc_loss_fn(dsc_preds_l, dsc_real_l) adv_dsc_loss_u = dsc_loss_fn(dsc_preds_u, dsc_real_u) adv_dsc_loss = adv_dsc_loss_l + adv_dsc_loss_u total_svae_loss = svae_loss_u + svae_loss_l + params[ 'adv_hyperparameter'] * adv_dsc_loss svae_optim.zero_grad() total_svae_loss.backward() svae_optim.step() if i < self.svae_iterations - 1: batch_sequences_l, batch_lengths_l, _ = next( iter(dataloader_l)) batch_sequences_u, batch_length_u, _ = next( iter(dataloader_u)) if torch.cuda.is_available(): batch_sequences_l = batch_sequences_l.to(self.device) batch_lengths_l = batch_lengths_l.to(self.device) batch_sequences_u = batch_sequences_u.to(self.device) batch_length_u = batch_length_u.to(self.device) step += 1 # Discriminator Step for j in range(self.dsc_iterations): with torch.no_grad(): _, _, _, z_l = svae(batch_sequences_l, batch_lengths_l) _, _, _, z_u = svae(batch_sequences_u, batch_lengths_u) dsc_preds_l = discriminator(z_l) dsc_preds_u = discriminator(z_u) dsc_real_l = torch.ones(batch_size_l) dsc_real_u = torch.zeros(batch_size_u) if torch.cuda.is_available(): dsc_real_l = dsc_real_l.to(self.device) dsc_real_u = dsc_real_u.to(self.device) # Discriminator wants to minimise the loss here dsc_loss_l = dsc_loss_fn(dsc_preds_l, dsc_real_l) dsc_loss_u = dsc_loss_fn(dsc_preds_u, dsc_real_u) total_dsc_loss = dsc_loss_l + dsc_loss_u dsc_optim.zero_grad() total_dsc_loss.backward() dsc_optim.step() # Sample new batch of data while training adversarial network if j < self.dsc_iterations - 1: batch_sequences_l, batch_lengths_l, _ = next( iter(dataloader_l)) batch_sequences_u, batch_length_u, _ = next( iter(dataloader_u)) if torch.cuda.is_available(): batch_sequences_l = batch_sequences_l.to(self.device) batch_lengths_l = batch_lengths_l.to(self.device) batch_sequences_u = batch_sequences_u.to(self.device) batch_length_u = batch_length_u.to(self.device) if (train_iter % dataset_size == 0): print("Initiating Early Stopping") early_stopping( tl_loss, task_learner ) # TODO: Review. Should this be the metric we early stop on? if early_stopping.early_stop: print( f'Early stopping at {train_iter}/{train_iterations} training iterations' ) break if (train_iter > 0) & (epoch == 1 or train_iter % dataset_size == 0): if train_iter % dataset_size == 0: val_metrics = self.evaluation(task_learner=task_learner, dataloader=dataloader_v, task_type=self.task_type) val_string = f'Task Learner ({self.task_type}) Validation ' + f'Scores:\nF1: Macro {val_metrics["f1 macro"]*100:0.2f}% Micro {val_metrics["f1 micro"]*100:0.2f}%\n' if self.task_type == 'SEQ' else f'Accuracy {val_metrics["accuracy"]*100:0.2f}' print(val_string) if (train_iter > 0) & (train_iter % dataset_size == 0): # Completed an epoch train_iter_str = f'Train Iter {train_iter} - Losses (TL-{self.task_type} {tl_loss:0.2f} | SVAE {total_svae_loss:0.2f} | Disc {total_dsc_loss:0.2f} | Learning rates: TL ({tl_optim.param_groups[0]["lr"]})' print(train_iter_str) print(f'Completed epoch: {epoch}') epoch += 1 # Evaluation at the end of the first training cycle test_metrics = self.evaluation(task_learner=task_learner, dataloader=dataloader_t, task_type='SEQ') f1_macro_1 = test_metrics['f1 macro'] # SVAE and Discriminator need to be evaluated on the TL metric n+1 split from their current training split # So, data needs to be sampled via SVAAL and then the TL retrained. The final metric from the retrained TL is then used to # optimise the SVAE and Discriminator. For this optimisation problem, the TL parameters are fixed. # Sample data via SVAE and Discriminator sampled_indices, _, _ = self.sample_adversarial( svae=svae, discriminator=discriminator, data=dataloader_u, indices=unlabelled_indices, cuda=True) # TODO: review usage of indices arg # Add new samples to labelled dataset current_indices = list(current_indices) + list(sampled_indices) sampler = data.sampler.SubsetRandomSampler(current_indices) self.labelled_dataloader = data.DataLoader(self.datasets['train'], sampler=sampler, batch_size=self.batch_size, drop_last=True) dataloader_l = self.labelled_dataloader # to maintain naming conventions print( f'{datetime.now()}: Indices - Labelled {len(current_indices)} Unlabelled {len(unlabelled_indices)} Total {len(all_indices)}' ) task_learner = TaskLearner(**params['tl'], vocab_size=self.vocab_size, tagset_size=self.tagset_size, task_type=self.task_type).to(self.device) if self.task_type == 'SEQ': tl_loss_fn = nn.NLLLoss().to(self.device) if self.task_type == 'CLF': tl_loss_fn = nn.CrossEntropyLoss().to(self.device) tl_optim = optim.SGD( task_learner.parameters(), lr=params['tl_learning_rate']) #, momentum=0, weight_decay=0.1) task_learner.train() early_stopping = EarlyStopping( patience=self.config['Train']['es_patience'], verbose=True, path="checkpoints/checkpoint.pt") print(f'{datetime.now()}: Task Learner initialised successfully') # Train Task Learner on adversarially selected samples dataset_size = len(dataloader_l) train_iterations = dataset_size * (params['epochs'] + 1) epoch = 1 for train_iter in tqdm(range(train_iterations), desc='Training iteration'): batch_sequences_l, batch_lengths_l, batch_tags_l = next( iter(dataloader_l)) if torch.cuda.is_available(): batch_sequences_l = batch_sequences_l.to(self.device) batch_lengths_l = batch_lengths_l.to(self.device) batch_tags_l = batch_tags_l.to(self.device) # Strip off tag padding and flatten # Don't do sequences here as its done in the forward pass of the seq2seq models batch_tags_l = trim_padded_seqs(batch_lengths=batch_lengths_l, batch_sequences=batch_tags_l, pad_idx=self.pad_idx).view(-1) # Task Learner Step tl_optim.zero_grad() tl_preds = task_learner(batch_sequences_l, batch_lengths_l) tl_loss = tl_loss_fn(tl_preds, batch_tags_l) tl_loss.backward() tl_optim.step() if (train_iter % dataset_size == 0): print("Initiating Early Stopping") early_stopping( tl_loss, task_learner ) # TODO: Review. Should this be the metric we early stop on? if early_stopping.early_stop: print( f'Early stopping at {train_iter}/{train_iterations} training iterations' ) break if (train_iter > 0) & (epoch == 1 or train_iter % dataset_size == 0): if train_iter % dataset_size == 0: val_metrics = self.evaluation(task_learner=task_learner, dataloader=dataloader_v, task_type=self.task_type) val_string = f'Task Learner ({self.task_type}) Validation ' + f'Scores:\nF1: Macro {val_metrics["f1 macro"]*100:0.2f}% Micro {val_metrics["f1 micro"]*100:0.2f}%\n' if self.task_type == 'SEQ' else f'Accuracy {val_metrics["accuracy"]*100:0.2f}' print(val_string) if (train_iter > 0) & (train_iter % dataset_size == 0): # Completed an epoch train_iter_str = f'Train Iter {train_iter} - Losses (TL-{self.task_type} {tl_loss:0.2f} | Learning rate: TL ({tl_optim.param_groups[0]["lr"]})' print(train_iter_str) print(f'Completed epoch: {epoch}') epoch += 1 # Compute test metrics test_metrics = self.evaluation(task_learner=task_learner, dataloader=dataloader_t, task_type='SEQ') print( f'{datetime.now()}: Test Eval.: F1 Scores - Macro {test_metrics["f1 macro"]*100:0.2f}% Micro {test_metrics["f1 micro"]*100:0.2f}%' ) # return test_metrics["f1 macro"] # Should the output be maximum rate of the change between iter n and iter n+1 metrics? this makes more sense than just f1 macro? f1_macro_diff = test_metrics['f1 macro'] - f1_macro_1 print(f'Macro f1 difference: {f1_macro_diff*100:0.2f}%') return f1_macro_diff
def __init__(self, atype, target_region): Sampler.__init__(self, atype, target_region, sampler=None)