def select_and_label(dataset: Dataset, model: Model, topk: int, choice_fn: Callable) -> None: # IDK what return type should be... """ Selects data points from dataset according to criterion and updates the model. Parameters ========== dataset : Dataset Dataset of predictions and observations. model : Model Bayesian assessment model. choice_fn : Callable Function used to identify the next class to be labeled. """ # Initialize outputs # Shuffle the dataset and enqueue queries dataset.shuffle() queues = dataset.enqueue() n_samples = len(dataset) mpe = np.zeros((n_samples // LOG_FREQ, dataset.num_classes)) confusion_log = np.zeros((n_samples // LOG_FREQ, dataset.num_classes, dataset.num_classes)) # Run experiment i = 0 while i < n_samples: sample = model.sample() choices = choice_fn(sample) candidates = [choice for choice in choices if len(queues[choice]) > 0] if len(candidates) < topk: topk = 1 for idx in range(topk): choice = candidates[idx] observation = queues[choice].pop() model.update(choice, observation) i += 1 if not i % LOG_FREQ: index = i // LOG_FREQ - 1 mpe[index] = model.mpe() confusion_log[index] = model.confusion_matrix() # In case we're one short mpe[-1] = model.mpe() return mpe, confusion_log