def setUpClass(cls) -> None: torch.manual_seed(0) np.random.seed(0) random.seed(0) n_samples_per_class = 100 dataset = make_classification(n_samples=6 * n_samples_per_class, n_classes=6, n_features=4, n_informative=4, n_redundant=0) X = torch.from_numpy(dataset[0]).float() y = torch.from_numpy(dataset[1]).long() train_X, test_X, train_y, test_y = train_test_split(X, y, train_size=0.5, shuffle=True, stratify=y) tr_d = TensorDataset(train_X, train_y) ts_d = TensorDataset(test_X, test_y) benchmark = nc_benchmark(train_dataset=tr_d, test_dataset=ts_d, n_experiences=3, task_labels=True, shuffle=False, seed=0) model = SimpleMLP(input_size=4, num_classes=benchmark.n_classes) f = open('log.txt', 'w') text_logger = TextLogger(f) eval_plugin = EvaluationPlugin( accuracy_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), loss_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), forgetting_metrics(experience=True, stream=True, task=True), confusion_matrix_metrics(num_classes=6, save_image=False, normalize='all', stream=True), bwt_metrics(experience=True, stream=True, task=True), cpu_usage_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), timing_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), ram_usage_metrics(every=0.5, minibatch=True, epoch=True, experience=True, stream=True), disk_usage_metrics(minibatch=True, epoch=True, experience=True, stream=True), MAC_metrics(minibatch=True, epoch=True, experience=True), loggers=[text_logger], collect_all=True) # collect all metrics (set to True by default) cl_strategy = BaseStrategy(model, SGD(model.parameters(), lr=0.001, momentum=0.9), CrossEntropyLoss(), train_mb_size=10, train_epochs=2, eval_mb_size=10, device=DEVICE, evaluator=eval_plugin, eval_every=1) for i, experience in enumerate(benchmark.train_stream): cl_strategy.train(experience, eval_streams=[benchmark.test_stream[i]], shuffle=False) cl_strategy.eval(benchmark.test_stream) cls.all_metrics = cl_strategy.evaluator.get_all_metrics() f.close() # with open(os.path.join(pathlib.Path(__file__).parent.absolute(), # 'target_metrics', # 'mt.pickle'), 'wb') as f: # pickle.dump(dict(cls.all_metrics), f, # protocol=pickle.HIGHEST_PROTOCOL) with open( os.path.join( pathlib.Path(__file__).parent.absolute(), 'target_metrics', 'mt.pickle'), 'rb') as f: cls.ref = pickle.load(f)
def setUpClass(cls) -> None: torch.manual_seed(0) np.random.seed(0) random.seed(0) n_samples_per_class = 100 datasets = [] for i in range(3): dataset = make_classification(n_samples=3 * n_samples_per_class, n_classes=3, n_features=3, n_informative=3, n_redundant=0) X = torch.from_numpy(dataset[0]).float() y = torch.from_numpy(dataset[1]).long() train_X, test_X, train_y, test_y = train_test_split(X, y, train_size=0.5, shuffle=True, stratify=y) datasets.append((train_X, train_y, test_X, test_y)) tr_ds = [ AvalancheTensorDataset( tr_X, tr_y, dataset_type=AvalancheDatasetType.CLASSIFICATION, task_labels=torch.randint(0, 3, (150, )).tolist()) for tr_X, tr_y, _, _ in datasets ] ts_ds = [ AvalancheTensorDataset( ts_X, ts_y, dataset_type=AvalancheDatasetType.CLASSIFICATION, task_labels=torch.randint(0, 3, (150, )).tolist()) for _, _, ts_X, ts_y in datasets ] benchmark = dataset_benchmark(train_datasets=tr_ds, test_datasets=ts_ds) model = SimpleMLP(num_classes=3, input_size=3) f = open('log.txt', 'w') text_logger = TextLogger(f) eval_plugin = EvaluationPlugin( accuracy_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, trained_experience=True), loss_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), forgetting_metrics(experience=True, stream=True), confusion_matrix_metrics(num_classes=3, save_image=False, normalize='all', stream=True), bwt_metrics(experience=True, stream=True), forward_transfer_metrics(experience=True, stream=True), cpu_usage_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), timing_metrics(minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True), ram_usage_metrics(every=0.5, minibatch=True, epoch=True, experience=True, stream=True), disk_usage_metrics(minibatch=True, epoch=True, experience=True, stream=True), MAC_metrics(minibatch=True, epoch=True, experience=True), loggers=[text_logger], collect_all=True) # collect all metrics (set to True by default) cl_strategy = BaseStrategy(model, SGD(model.parameters(), lr=0.001, momentum=0.9), CrossEntropyLoss(), train_mb_size=2, train_epochs=2, eval_mb_size=2, device=DEVICE, evaluator=eval_plugin, eval_every=1) for i, experience in enumerate(benchmark.train_stream): cl_strategy.train(experience, eval_streams=[benchmark.test_stream], shuffle=False) cl_strategy.eval(benchmark.test_stream) cls.all_metrics = cl_strategy.evaluator.get_all_metrics() f.close() # # Uncomment me to regenerate the reference metrics. Make sure # # the old tests were passing for all unchanged metrics # with open(os.path.join(pathlib.Path(__file__).parent.absolute(), # 'target_metrics', # 'tpp.pickle'), 'wb') as f: # pickle.dump(dict(cls.all_metrics), f, # protocol=4) with open( os.path.join( pathlib.Path(__file__).parent.absolute(), 'target_metrics', 'tpp.pickle'), 'rb') as f: cls.ref = pickle.load(f)
def setUpClass(cls) -> None: torch.manual_seed(0) np.random.seed(0) random.seed(0) n_samples_per_class = 100 dataset = make_classification( n_samples=6 * n_samples_per_class, n_classes=6, n_features=4, n_informative=4, n_redundant=0, ) X = torch.from_numpy(dataset[0]).float() y = torch.from_numpy(dataset[1]).long() train_X, test_X, train_y, test_y = train_test_split(X, y, train_size=0.5, shuffle=True, stratify=y) tr_d = TensorDataset(train_X, train_y) ts_d = TensorDataset(test_X, test_y) benchmark = nc_benchmark( train_dataset=tr_d, test_dataset=ts_d, n_experiences=3, task_labels=True, shuffle=False, seed=0, ) model = SimpleMLP(input_size=4, num_classes=benchmark.n_classes) f = open("log.txt", "w") text_logger = TextLogger(f) eval_plugin = EvaluationPlugin( accuracy_metrics( minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, trained_experience=True, ), loss_metrics( minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, ), forgetting_metrics(experience=True, stream=True), confusion_matrix_metrics(num_classes=6, save_image=False, normalize="all", stream=True), bwt_metrics(experience=True, stream=True), forward_transfer_metrics(experience=True, stream=True), cpu_usage_metrics( minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, ), timing_metrics( minibatch=True, epoch=True, epoch_running=True, experience=True, stream=True, ), ram_usage_metrics( every=0.5, minibatch=True, epoch=True, experience=True, stream=True, ), disk_usage_metrics(minibatch=True, epoch=True, experience=True, stream=True), MAC_metrics(minibatch=True, epoch=True, experience=True), loggers=[text_logger], collect_all=True, ) # collect all metrics (set to True by default) cl_strategy = BaseStrategy( model, SGD(model.parameters(), lr=0.001, momentum=0.9), CrossEntropyLoss(), train_mb_size=10, train_epochs=2, eval_mb_size=10, device=DEVICE, evaluator=eval_plugin, eval_every=1, ) for i, experience in enumerate(benchmark.train_stream): cl_strategy.train(experience, eval_streams=[benchmark.test_stream], shuffle=False) cl_strategy.eval(benchmark.test_stream) cls.all_metrics = cl_strategy.evaluator.get_all_metrics() f.close() # Set the environment variable UPDATE_METRICS to True to update # the pickle file with target values. # Make sure the old tests were passing for all unchanged metrics if UPDATE_METRICS: with open( os.path.join( pathlib.Path(__file__).parent.absolute(), "target_metrics", "mt.pickle", ), "wb", ) as f: pickle.dump(dict(cls.all_metrics), f, protocol=4) with open( os.path.join( pathlib.Path(__file__).parent.absolute(), "target_metrics", "mt.pickle", ), "rb", ) as f: cls.ref = pickle.load(f)