def main(args): # print args recap print(args, end="\n\n") # do not remove this line start = time.time() # Create the dataset object for example with the "ni, multi-task-nc, or nic # tracks" and assuming the core50 location in ./core50/data/ dataset = CORE50(root='core50/data/', scenario=args.scenario, preload=args.preload_data) # Get the validation set print("Recovering validation set...") full_valdidset = dataset.get_full_valid_set() # model if args.classifier == 'ResNet18': classifier = models.resnet18(pretrained=True) classifier.fc = torch.nn.Linear(512, args.n_classes) opt = torch.optim.SGD(classifier.parameters(), lr=args.lr) criterion = torch.nn.CrossEntropyLoss() # vars to update over time valid_acc = [] ext_mem_sz = [] ram_usage = [] heads = [] ext_mem = None # loop over the training incremental batches (x, y, t) for i, train_batch in enumerate(dataset): train_x, train_y, t = train_batch # adding eventual replay patterns to the current batch idxs_cur = np.random.choice(train_x.shape[0], args.replay_examples, replace=False) if i == 0: ext_mem = [train_x[idxs_cur], train_y[idxs_cur]] else: ext_mem = [ np.concatenate((train_x[idxs_cur], ext_mem[0])), np.concatenate((train_y[idxs_cur], ext_mem[1])) ] train_x = np.concatenate((train_x, ext_mem[0])) train_y = np.concatenate((train_y, ext_mem[1])) print("----------- batch {0} -------------".format(i)) print("x shape: {0}, y shape: {1}".format(train_x.shape, train_y.shape)) print("Task Label: ", t) # train the classifier on the current batch/task _, _, stats = train_net(opt, classifier, criterion, args.batch_size, train_x, train_y, t, args.epochs, preproc=preprocess_imgs) if args.scenario == "multi-task-nc": heads.append(copy.deepcopy(classifier.fc)) # collect statistics ext_mem_sz += stats['disk'] ram_usage += stats['ram'] # test on the validation set stats, _ = test_multitask(classifier, full_valdidset, args.batch_size, preproc=preprocess_imgs, multi_heads=heads, last_layer_name="fc", verbose=False) valid_acc += stats['acc'] print("------------------------------------------") print("Avg. acc: {}".format(stats['acc'])) print("------------------------------------------") # Generate submission.zip # directory with the code snapshot to generate the results sub_dir = 'submissions/' + args.sub_dir if not os.path.exists(sub_dir): os.makedirs(sub_dir) # copy code create_code_snapshot(".", sub_dir + "/code_snapshot") # generating metadata.txt: with all the data used for the CLScore elapsed = (time.time() - start) / 60 print("Training Time: {}m".format(elapsed)) with open(sub_dir + "/metadata.txt", "w") as wf: for obj in [ np.average(valid_acc), elapsed, np.average(ram_usage), np.max(ram_usage), np.average(ext_mem_sz), np.max(ext_mem_sz) ]: wf.write(str(obj) + "\n") # test_preds.txt: with a list of labels separated by "\n" print("Final inference on test set...") full_testset = dataset.get_full_test_set() stats, preds = test_multitask(classifier, full_testset, args.batch_size, preproc=preprocess_imgs, multi_heads=heads, last_layer_name="fc", verbose=False) with open(sub_dir + "/test_preds.txt", "w") as wf: for pred in preds: wf.write(str(pred) + "\n") print("Experiment completed.")
def main(args): # print args recap print(args, end="\n\n") # do not remove this line start = time.time() # Create the dataset object for example with the "ni, multi-task-nc, or nic tracks" # and assuming the core50 location in ./core50/data/ # ???review CORE50 to see if there is a way to shorten dataset and runtime for testing # Original call to dataset. Takes a long time. # dataset = CORE50(root='core50/data/', scenario=args.scenario, # preload=args.preload_data) # # custom call to create CORE50 custom object # using train=True uses training set and allows more control over batches and other stuff. dataset = CORE50(root='core50/data/', scenario=args.scenario, preload=args.preload_data) # Get the validation set print("Recovering validation set...") # default full validation set # full_valdidset = dataset.get_full_valid_set() # reduced validation set full_valdidset = dataset.get_full_valid_set(reduced=True) # model if args.classifier == 'ResNet18': classifier = models.resnet18( pretrained=True) # classifier is a pretrained model classifier.fc = torch.nn.Linear( 512, args.n_classes ) # in features: 512 # out features: set below -> args.n_classes = 50 # Applies a linear transformation to the incoming data opt = torch.optim.SGD(classifier.parameters(), lr=args.lr) # Implements stochastic gradient descent criterion = torch.nn.CrossEntropyLoss( ) # This criterion combines nn.LogSoftmax() and nn.NLLLoss() in one single class. # vars to update over time valid_acc = [] ext_mem_sz = [] ram_usage = [] heads = [] # Start Modification ewc_lambda = 4 # should this be higher? closer to 0.01 or 0.4 or 0.8? that is what was used in other examples. What does a higher penalty do? what does a lower penatly do? # fisher_max = 0.0001 # variable dictionary to hold fisher values fisher_dict = {} # variable dictionary to hold previous optimized weight values optpar_dict = {} # End Modification # loop over the training incremental batches (x, y, t) for i, train_batch in enumerate(dataset): train_x, train_y, t = train_batch # Start Modifiction # Make train_x and train_y smaller for testing here limit_size = False # make true to limit training size # make false to allow full training set if limit_size: train_size = 3200 # train_size = 11900 train_x = train_x[0:train_size] train_y = train_y[0:train_size] # End Modification # Print current batch number print("----------- batch {0} -------------".format(i)) # Print current batch shape print("x shape: {0}, y shape: {1}".format(train_x.shape, train_y.shape)) # print task label type print("Task Label: ", t) # utils.train_net: a custom function to train neural network. returns stats. _, _, stats = train_net_ewc(opt, classifier, criterion, args.batch_size, train_x, train_y, t, fisher_dict, optpar_dict, ewc_lambda, args.epochs, preproc=preprocess_imgs) # if multi-task-nc: make deep copy in list heads (aka nn brains) if args.scenario == "multi-task-nc": heads.append(copy.deepcopy(classifier.fc)) ext_mem_sz += stats['disk'] ram_usage += stats['ram'] # Start Modifiction # Calculate the Fisher matrix values given new completed task on_task_update(t, train_x, train_y, fisher_dict, optpar_dict, classifier, opt, criterion, args.batch_size, preproc=preprocess_imgs ) # training complete # compute fisher matrix values # End Modification # test all nn models in list heads for performance. return stats for each. stats, _ = test_multitask(classifier, full_valdidset, args.batch_size, preproc=preprocess_imgs, multi_heads=heads) # print new stats on performance valid_acc += stats['acc'] print("------------------------------------------") print("Avg. acc: {}".format(stats['acc'])) print("------------------------------------------") # Generate submission.zip # directory with the code snapshot to generate the results sub_dir = 'submissions/' + args.sub_dir if not os.path.exists(sub_dir): os.makedirs(sub_dir) # copy code # custom function in utils folder to deal with possible file path issues create_code_snapshot(".", sub_dir + "/code_snapshot") # generating metadata.txt: with all the data used for the CLScore elapsed = (time.time() - start) / 60 print("Training Time: {}m".format(elapsed)) with open(sub_dir + "/metadata.txt", "w") as wf: for obj in [ np.average(valid_acc), elapsed, np.average(ram_usage), np.max(ram_usage), np.average(ext_mem_sz), np.max(ext_mem_sz) ]: wf.write(str(obj) + "\n") # run final full test # test_preds.txt: with a list of labels separated by "\n" print("Final inference on test set...") full_testset = dataset.get_full_test_set() stats, preds = test_multitask(classifier, full_testset, args.batch_size, preproc=preprocess_imgs) with open(sub_dir + "/test_preds.txt", "w") as wf: for pred in preds: wf.write(str(pred) + "\n") print("Experiment completed.")
def main(args): # print args recap print(args, end="\n\n") # do not remove this line start = time.time() # Create the dataset object for example with the "ni, multi-task-nc, or nic tracks" # and assuming the core50 location in ./core50/data/ # ???review CORE50 to see if there is a way to shorten dataset and runtime for testing # Original call to dataset. Takes a long time. # dataset = CORE50(root='core50/data/', scenario=args.scenario, # preload=args.preload_data) # # custom call to create CORE50 custom object # using train=True uses training set and allows more control over batches and other stuff. dataset = CORE50(root='core50/data/', scenario=args.scenario, preload=args.preload_data) # Get the validation set print("Recovering validation set...") # default full validation set # full_valdidset = dataset.get_full_valid_set() # reduced validation set full_valdidset = dataset.get_full_valid_set(reduced=True) # model if args.classifier == 'ResNet18': classifier = models.resnet18( pretrained=True) # classifier is a pretrained model classifier.fc = torch.nn.Linear( 512, args.n_classes ) # in features: 512 # out features: set below -> args.n_classes = 50 # Applies a linear transformation to the incoming data opt = torch.optim.SGD(classifier.parameters(), lr=args.lr) # Implements stochastic gradient descent criterion = torch.nn.CrossEntropyLoss( ) # This criterion combines nn.LogSoftmax() and nn.NLLLoss() in one single class. # vars to update over time valid_acc = [] ext_mem_sz = [] ram_usage = [] heads = [] # enumerate(dataset) provides itirator over all training sets and test sets # loop over the training incremental batches (x, y, t) for i, train_batch in enumerate(dataset): train_x, train_y, t = train_batch # Start modification # run batch 0 and 1. Then break. # if i == 2: break # shuffle new data train_x, train_y = shuffle_in_unison((train_x, train_y), seed=0) if i == 0: # this is the first round # store data for later all_x = train_x[0:train_x.shape[0] // 2] all_y = train_y[0:train_y.shape[0] // 2] else: # this is not the first round # create hybrid training set old and new data # shuffle old data all_x, all_y = shuffle_in_unison((all_x, all_y), seed=0) # create temp holder temp_x = train_x temp_y = train_y # set current variables to be used for training train_x = np.append(all_x, train_x, axis=0) train_y = np.append(all_y, train_y) train_x, train_y = shuffle_in_unison((train_x, train_y), seed=0) # append half of old and all of new data temp_x, temp_y = shuffle_in_unison((temp_x, temp_y), seed=0) keep_old = (all_x.shape[0] // (i + 1)) * i keep_new = temp_x.shape[0] // (i + 1) all_x = np.append(all_x[0:keep_old], temp_x[0:keep_new], axis=0) all_y = np.append(all_y[0:keep_old], temp_y[0:keep_new]) del temp_x del temp_y # rest of code after this should be the same # End modification # Print current batch number print("----------- batch {0} -------------".format(i)) # Print current batch shape print("x shape: {0}, y shape: {1}".format(train_x.shape, train_y.shape)) # print task label type print("Task Label: ", t) # utils.train_net: a custom function to train neural network. returns stats. _, _, stats = train_net(opt, classifier, criterion, args.batch_size, train_x, train_y, t, args.epochs, preproc=preprocess_imgs) # if multi-task-nc: make deep copy in list heads (aka nn brains) if args.scenario == "multi-task-nc": heads.append(copy.deepcopy(classifier.fc)) ext_mem_sz += stats['disk'] ram_usage += stats['ram'] # test all nn models in list heads for performance. return stats for each. stats, _ = test_multitask(classifier, full_valdidset, args.batch_size, preproc=preprocess_imgs, multi_heads=heads) # print new stats on performance valid_acc += stats['acc'] print("------------------------------------------") print("Avg. acc: {}".format(stats['acc'])) print("------------------------------------------") # Generate submission.zip # directory with the code snapshot to generate the results sub_dir = 'submissions/' + args.sub_dir if not os.path.exists(sub_dir): os.makedirs(sub_dir) # copy code # custom function in utils folder to deal with possible file path issues create_code_snapshot(".", sub_dir + "/code_snapshot") # generating metadata.txt: with all the data used for the CLScore elapsed = (time.time() - start) / 60 print("Training Time: {}m".format(elapsed)) with open(sub_dir + "/metadata.txt", "w") as wf: for obj in [ np.average(valid_acc), elapsed, np.average(ram_usage), np.max(ram_usage), np.average(ext_mem_sz), np.max(ext_mem_sz) ]: wf.write(str(obj) + "\n") # run final full test # test_preds.txt: with a list of labels separated by "\n" print("Final inference on test set...") full_testset = dataset.get_full_test_set() stats, preds = test_multitask(classifier, full_testset, args.batch_size, preproc=preprocess_imgs) with open(sub_dir + "/test_preds.txt", "w") as wf: for pred in preds: wf.write(str(pred) + "\n") print("Experiment completed.")
def train_model(self, tune=True, resize=False): #GE #print('Start training') #GE start = time.time() # vars to update over time valid_acc = [] ext_mem_sz = [] ram_usage = [] heads = [] num_tasks = self.dataset.nbatch[self.dataset.scenario] # loop over the training incremental batches (x, y, t) for i, train_batch in enumerate(self.dataset): train_x_raw, train_y_raw, t = train_batch train_x_raw = train_x_raw.astype(np.uint8) train_y_raw = train_y_raw.astype(np.uint8) del train_batch if self.args.verbose: print("----------- batch {0} -------------".format(i)) for replay_epoch in range(self.args.replay_epochs): if self.args.verbose: print("replay epoch {} for batch {}".format( replay_epoch, i)) if i > 0: mem_imgs, mem_labels = self.buffer.get_mem( self.replay_used) train_x = np.concatenate((train_x_raw, mem_imgs)) train_y = np.concatenate((train_y_raw, mem_labels)) else: train_x = train_x_raw train_y = train_y_raw if self.args.verbose: print("x shape: {0}, y shape: {1}".format( train_x.shape, train_y.shape)) # train the classifier on the current batch/task if self.args.aug: _, _, stats = train_net_aug(self.optimizer, self.model, self.loss, self.args.batch_size, train_x, train_y, i, self.args.epochs, self.train_aug, use_cuda=self.use_cuda, verbose=self.args.verbose) else: _, _, stats = train_net(self.optimizer, self.model, self.loss, self.args.batch_size, train_x, train_y, i, self.args.epochs, preproc=preprocess_imgs, use_cuda=self.use_cuda, resize=resize) # collect statistics ext_mem_sz += stats['disk'] ram_usage += stats['ram'] if self.args.verbose and self.args.scenario != 'nic': print(ram_usage) if self.args.scenario == "multi-task-nc": heads.append(copy.deepcopy(self.model.fc)) # adding eventual replay patterns to the current batch self.buffer.update_buffer(train_x_raw, train_y_raw, i) # test on the validation set if self.args.aug: stats, _ = test_multitask_aug(self.model, self.full_valdidset, self.test_bz, self.test_aug, verbose=self.args.verbose, use_cuda=self.use_cuda) else: stats, _ = test_multitask(self.model, self.full_valdidset, self.args.batch_size, preproc=preprocess_imgs, multi_heads=heads, verbose=self.args.verbose, use_cuda=self.use_cuda, resize=resize) if self.args.verbose: print("------------------------------------------") print("Avg. acc:", stats['acc']) print("------------------------------------------") #GE print("------------------------------------------") print("Batch validation accuracy: {:.3f}".format(stats['acc'][0])) print("------------------------------------------") #GE valid_acc += stats['acc'] # final review if i == num_tasks - 1: for replay_epoch in range(self.args.review_epoch): for g in self.optimizer.param_groups: g['lr'] = self.args.review_lr_factor * self.args.lr train_x, train_y = self.buffer.get_mem(self.review_size) if self.args.aug: _, _, stats = train_net_aug(self.optimizer, self.model, self.loss, self.args.batch_size, train_x, train_y, i, self.args.epochs, self.train_aug, use_cuda=self.use_cuda) stats, _ = test_multitask_aug( self.model, self.full_valdidset, self.test_bz, self.test_aug, verbose=self.args.verbose, use_cuda=self.use_cuda) else: _, _, stats = train_net(self.optimizer, self.model, self.loss, self.args.batch_size, train_x, train_y, i, self.args.epochs, preproc=preprocess_imgs, use_cuda=self.use_cuda, resize=resize) stats, _ = test_multitask(self.model, self.full_valdidset, self.args.batch_size, preproc=preprocess_imgs, multi_heads=heads, verbose=self.args.verbose, use_cuda=self.use_cuda, resize=resize) if self.args.verbose: print("------------------------------------------") print("Review Avg. acc:", stats['acc']) print("------------------------------------------") final_val = stats['acc'] # clear mem del self.buffer, self.full_valdidset full_testset = self.dataset.get_full_test_set() # generating metadata.txt: with all the data used for the CLScore elapsed = (time.time() - start) / 60 print("Training Time: {:3f} min".format(elapsed)) if not tune: # test_preds.txt: with a list of labels separated by "\n" print("Generating final predictions on test set...") if self.args.aug: stats, preds = test_multitask_aug( self.model, full_testset, self.test_bz, self.test_aug, verbose=self.args.verbose, use_cuda=self.use_cuda, #GE no_output=True #GE ) else: stats, preds = test_multitask(self.model, full_testset, self.args.batch_size, preproc=preprocess_imgs, multi_heads=heads, verbose=self.args.verbose, use_cuda=self.use_cuda, resize=resize) #elapsed = (time.time() - start) / 60 #print("Total Time: {:3f} min".format(elapsed)) #GE print('Run completed') #print('Full test set accuracy: {:.3f}'.format(stats['acc'][0])) #GE return valid_acc, elapsed, ram_usage, ext_mem_sz, preds else: return final_val[0], np.average(valid_acc), elapsed, np.average( ram_usage), np.max(ram_usage)
def train_model(self, tune=True, resize=False): start = time.time() # vars to update over time valid_acc = [] ext_mem_sz = [] ram_usage = [] heads = [] # loop over the training incremental batches (x, y, t) for i, train_batch in enumerate(self.dataset): train_x, train_y, _ = train_batch print("----------- batch {0} -------------".format(i)) print("x shape: {0}, y shape: {1}".format(train_x.shape, train_y.shape)) # train the classifier on the current batch/task _, _, stats = self._train_step(optimizer=self.optimizer, model=self.model, criterion=self.loss, batch_size=self.args.batch_size, x=train_x, y=train_y, t=i, epochs=self.args.epochs, preproc=preprocess_imgs, use_cuda=self.use_cuda, resize=resize) ext_mem_sz += stats['disk'] ram_usage += stats['ram'] #print(ram_usage) if self.args.scenario == "multi-task-nc": heads.append(copy.deepcopy(self.model.fc)) # test on the validation set stats, _ = test_multitask(self.model, self.full_valdidset, mb_size=64, preproc=preprocess_imgs, multi_heads=heads, verbose=True, use_cuda=self.use_cuda, resize=resize) train_x = preprocess_imgs(train_x) self.buffer.update_buffer(train_x, train_y, i) print(self.buffer.mem[0].shape) del train_x, train_y, train_batch valid_acc += stats['acc'] print("------------------------------------------") print("Avg. acc: {}".format(stats['acc'])) print("------------------------------------------") # final review if self.args.review_epoch == -1: mem_size = self.buffer.mem[1].shape[0] review_size = 15000 if mem_size % review_size == 0: num_it = mem_size // review_size else: num_it = mem_size // review_size + 1 for it in range(num_it): start = it * review_size end = (it + 1) * review_size train_x = self.buffer.mem[0][start:end] train_y = self.buffer.mem[1][start:end] _, _, stats = train_net(self.optimizer, self.model, self.loss, self.args.batch_size, train_x, train_y, 0, 1, preproc=None, use_cuda=self.use_cuda, resize=resize) stats, _ = test_multitask(self.model, self.full_valdidset, mb_size=64, preproc=preprocess_imgs, multi_heads=heads, verbose=True, use_cuda=self.use_cuda, resize=resize) print("------------------------------------------") print("review Avg. acc: {}".format(stats['acc'])) print("------------------------------------------") else: for replay_epoch in range(self.args.review_epoch): idxs_use = np.random.choice(self.buffer.mem[1].shape[0], self.args.review_size, replace=False) train_x = self.buffer.mem[0][idxs_use] train_y = self.buffer.mem[1][idxs_use] _, _, stats = train_net(self.optimizer, self.model, self.loss, self.args.batch_size, train_x, train_y, 0, 1, preproc=None, use_cuda=self.use_cuda, resize=resize) stats, _ = test_multitask(self.model, self.full_valdidset, mb_size=64, preproc=preprocess_imgs, multi_heads=heads, verbose=True, use_cuda=self.use_cuda, resize=resize) print("------------------------------------------") print("review Avg. acc: {}".format(stats['acc'])) print("------------------------------------------") final_val = stats['acc'] del self.full_valdidset, self.buffer elapsed = (time.time() - start) / 60 print("Training Time: {:.3f} min".format(elapsed)) full_testset = self.dataset.get_full_test_set() if not tune: # test_preds.txt: with a list of labels separated by "\n" print("Final inference on test set...") stats, preds = test_multitask(self.model, full_testset, mb_size=32, preproc=preprocess_imgs, multi_heads=heads, verbose=True, use_cuda=self.use_cuda, resize=resize) return valid_acc, elapsed, ram_usage, ext_mem_sz, preds else: return final_val[0], np.average(valid_acc), elapsed, np.average( ram_usage), np.max(ram_usage)