def run_demo(self): self.load() self.show_intro() meta_batch = MetaBatch(self.samples[Splits.test], self.device) task = self.analyze_task(meta_batch) self.analyze_sample(task) print("Demo Done")
def loop_outer(self): steps = self.hp.steps_outer interval = steps // 10 loader = self.loaders[Splits.train] tracker = MetricsTracker(prefix=Splits.train) stop_saver = EarlyStopSaver(self.net) for i, batch in enumerate( generate(loader, limit=steps, show_progress=True)): self.opt_outer.zero_grad() for task in MetaBatch(batch, self.device).get_tasks(): metrics = self.loop_inner(task) tracker.store(metrics) self.opt_outer.store_grad() self.opt_outer.step(i) if i % interval == 0: metrics = tracker.get_average() tracker.reset() metrics.update(self.loop_eval(Splits.val)) print({k: round(v, 3) for k, v in metrics.items()}) stop_saver.check_stop(metrics["val_loss"]) self.save() stop_saver.load_best() self.save()
def loop_val(self) -> dict: tracker = MetricsTracker(prefix=Splits.val) for task in MetaBatch(self.batch_val, torch.device("cpu")).get_tasks(): x_train, y_train, x_test, y_test = task model = self.model model.fit(x_train.numpy(), y_train.numpy()) logits_numpy = model.predict_proba(x_test.numpy()) logits = torch.from_numpy(logits_numpy) acc = ReptileSystem.get_accuracy(logits, y_test) tracker.store(dict(acc=acc)) return tracker.get_average()
def loop_eval(self) -> dict: tracker = MetricsTracker(prefix=Splits.val) for task in MetaBatch(self.batch_val, torch.device("cpu")).get_tasks(): x_train, y_train = task.train x_test, y_test = task.test # model = svm.SVC(kernel="rbf") model = linear_model.RidgeClassifier() model.fit(x_train.numpy(), y_train.numpy()) logits_numpy = model.predict(x_test.numpy()) logits = torch.from_numpy(logits_numpy) acc = accuracy_score(logits, y_test) tracker.store(dict(acc=acc)) return tracker.get_average()
def loop_val(self) -> dict: tracker = MetricsTracker(prefix=Splits.val) net_before = deepcopy(self.net.state_dict()) opt_before = deepcopy(self.opt_inner.state_dict()) def reset_state(): self.net.load_state_dict(deepcopy(net_before)) self.opt_inner.load_state_dict(deepcopy(opt_before)) for task in MetaBatch(self.batch_val, self.device).get_tasks(): metrics = self.loop_inner(task) tracker.store(metrics) reset_state() return tracker.get_average()
def loop_eval(self, data_split: str) -> dict: tracker = MetricsTracker(prefix=data_split) net_before = deepcopy(self.net.state_dict()) opt_before = deepcopy(self.opt_inner.state_dict()) def reset_state(): self.net.load_state_dict(deepcopy(net_before)) self.opt_inner.load_state_dict(deepcopy(opt_before)) for task in MetaBatch(self.samples[data_split], self.device).get_tasks(): metrics = self.loop_inner(task) tracker.store(metrics) reset_state() return tracker.get_average()
def loop_outer(self): steps = self.hparams.steps_outer interval = steps // 10 loader = self.loaders[Splits.train] tracker = MetricsTracker(prefix=Splits.train) for i, batch in enumerate( generate(loader, limit=steps, show_progress=True)): self.opt_outer.zero_grad() for task in MetaBatch(batch, self.device).get_tasks(): metrics = self.loop_inner(task, self.hparams.bs_inner, self.hparams.steps_inner) tracker.store(metrics) self.opt_outer.store_grad() self.opt_outer.step(i) if i % interval == 0: metrics = tracker.get_average() tracker.reset() metrics.update(self.loop_val()) print({k: round(v, 3) for k, v in metrics.items()})
def analyze_task(self, meta_batch: MetaBatch): st.header("Task Analysis") st.write( """ Each task or episode is created by randomly sampling a subset of classes from the original dataset, with a fixed number of examples per class. This is to simulate the few-shot learning setting across diverse tasks. The meta-learner is trained to generalize quickly to unseen tasks. """ ) st.write(f"Number of samples per class: {self.hp.num_shots}") st.write(f"Number of classes: {self.hp.num_ways}") st.write(f"Train/val/test samples: **{self.hp.num_ways * self.hp.num_shots}**") tasks_all = meta_batch.get_tasks() i = st.selectbox(label="Task Index", options=range(len(tasks_all))[::-1]) task = tasks_all[i] self.show_task(task) st.write("Overall test Set performance for this task:") st.write((self.loop_inner(task))) st.subheader("Model Output Visualization") self.analyze_model_outputs(task) return task