def __init__(self, args, device): self.args = args self.device = device model = model_factory.get_network(args.network)(classes=args.n_classes, jigsaw_classes=31, rotation_classes=4, odd_classes=9) self.model = model.to(device) self.source_loader, self.val_loader = data_helper.get_train_dataloader(args) self.target_loader = data_helper.get_val_dataloader(args) self.test_loaders = {"val": self.val_loader, "test": self.target_loader} self.len_dataloader = len(self.source_loader) print("Dataset size: train %d, val %d, test %d" % (len(self.source_loader.dataset), len(self.val_loader.dataset), len(self.target_loader.dataset))) self.optimizer, self.scheduler = get_optim_and_scheduler(model, args.epochs, args.learning_rate, args.train_all) self.n_classes = args.n_classes self.nTasks = 2 if args.rotation == True: self.nTasks += 1 if args.odd_one_out == True: self.nTasks += 1 print("N of tasks: " + str(self.nTasks))
def main(config): print(config["dstpath"]) logger = config.get_logger("train") SEED = config["seed"] model = get_network(config) ORIGINAL = config["data_loader"]["path_original"] STORES_IDS = pd.read_csv(ORIGINAL + "sales_train_validation.csv")["store_id"] STORES_IDS = list(STORES_IDS.unique()) for store_id in STORES_IDS: run = wandb.init(reinit=True) with run: config["data_loader"]["store_id"] = store_id logger.info("Getting Dataset..") start = time.time() X_train, y_train, X_test, y_test = gt_dataset(config) end = time.time() - start logger.info(f"Done! (elaped:{end} sec!)") logger.info(f"Start Training") seed_everything(SEED) model.fit(X_train, y_train, eval_set=[(X_test, y_test)], eval_metric="rmse", early_stopping_rounds=5, callbacks=[wandb_callback]) model_name = "lgb_model_" + store_id + "_v" + str(1) + ".bin" pickle.dump(model, open(model_name, "wb")) # pickle.dump(estimator, open(model_name, "wb")) del X_train, y_train, X_test, y_test, model gc.collect()
def __init__(self, args, device): self.args = args self.device = device model = model_factory.get_network(args.network)( jigsaw_classes=args.jigsaw_n_classes + 1, classes=args.n_classes) self.model = model.to(device) # print(self.model) self.source_loader, self.val_loader = data_helper.get_train_dataloader( args, patches=model.is_patch_based()) self.target_loader = data_helper.get_val_dataloader( args, patches=model.is_patch_based()) self.test_loaders = { "val": self.val_loader, "test": self.target_loader } self.len_dataloader = len(self.source_loader) print("Dataset size: train %d, val %d, test %d" % (len(self.source_loader.dataset), len( self.val_loader.dataset), len(self.target_loader.dataset))) self.optimizer, self.scheduler = get_optim_and_scheduler( model, args.epochs, args.learning_rate, args.train_all, nesterov=args.nesterov) self.jig_weight = args.jig_weight self.only_non_scrambled = args.classify_only_sane self.n_classes = args.n_classes if args.target in args.source: self.target_id = args.source.index(args.target) print("Target in source: %d" % self.target_id) print(args.source) else: self.target_id = None
def __init__(self, args, device): self.args = args self.device = device # Logger self.log_frequency = 10 model = model_factory.get_network(args.network)(classes=args.n_classes) self.model = model.to(device) # The training dataset get divided into two parts (Train & Validation) self.source_loader, self.val_loader = data_helper.get_train_dataloader(args) self.target_loader = data_helper.get_target_dataloader(args) self.test_loaders = {"val": self.val_loader, "test": self.target_loader} self.init_train_dataset_size = len(self.source_loader.dataset) print("Dataset size: train %d, val %d, test %d" % (len(self.source_loader.dataset), len(self.val_loader.dataset), len(self.target_loader.dataset))) self.optimizer, self.scheduler = get_optim_and_scheduler(model, args.epochs, args.learning_rate, train_all=True, nesterov=args.nesterov, adam = args.adam) self.n_classes = args.n_classes if args.target in args.source: self.target_id = args.source.index(args.target) print("Target in source: %d" % self.target_id) print(args.source) else: self.target_id = None
def __init__(self, args, device): self.args = args self.device = device model = model_factory.get_network(args.network)( jigsaw_classes=args.jigsaw_n_classes + 1, classes=args.n_classes) self.model = model.to(device) self.exp_type = "%s/%s_to_%s/%s" % (args.exp_type, "-".join( sorted(args.source)), args.target, args.run_id) self.model_path = osp.join('logs', self.exp_type, 'best_model.pth') print("Loading best_model.pth from {}".format(self.model_path)) self.args.target = self.args.generate_for self.target_loader = data_helper.get_val_dataloader( args, patches=model.is_patch_based()) print("Dataset size: test %d" % (len(self.target_loader.dataset))) self.n_classes = args.n_classes if args.target in args.source: self.target_id = args.source.index(args.target) print("Target in source: %d" % self.target_id) print(args.source) else: self.target_id = None self.freeze_layer = args.freeze_layer
def __init__(self, args, device): self.args = args self.device = device if args.scrambled == 0: args.jig_weight_source = 0 args.jig_weight_target = 0 if args.rotated == 0: args.rot_weight_source = 0 args.rot_weight_target = 0 if args.odd == 0: args.odd_weight_source = 0 args.odd_weight_target = 0 model = model_factory.get_network(args.network)(classes=args.n_classes, odd_classes = args.grid_size**2) self.model = model.to(device) # Source Loaders self.source_loader, self.val_loader = data_helper.get_train_dataloader_JiGen(args, device, "DA") # Target Loaders self.target_train_loader, self.target_loader = data_helper.get_target_loader(args, device, "DA") self.test_loaders = {"val": self.val_loader, "test": self.target_loader} size = len(self.source_loader.dataset) + len(self.target_train_loader.dataset) print("Dataset size: train %d, val %d, test %d" % (size, len(self.val_loader.dataset), len(self.target_loader.dataset))) self.optimizer, self.scheduler = get_optim_and_scheduler(model, args.epochs, args.learning_rate, args.train_all) self.n_classes = args.n_classes # Set JiGen parameters self.jig_alpha_t = args.jig_weight_target self.jig_alpha_s = args.jig_weight_source self.permutation_number = args.permutation_number # Set Routate parameters self.rot_alpha_t = args.rot_weight_target self.rot_alpha_s = args.rot_weight_source # Set target loss weight self.target_loss_weight = args.target_entropy_weight # Set Odd parameters self.odd_alpha_t = args.odd_weight_target self.odd_alpha_s = args.odd_weight_source self.epoch_count = 0 self.tot_epoch = args.epochs
def __init__(self, args, device): self.args = args self.device = device model = model_factory.get_network(args.network)(classes=args.n_classes) model_ema = model_factory.get_network( args.network)(classes=args.n_classes) self.model = model.to(device) self.model_ema = model_ema.to(device) if args.moco: moment_update(self.model, self.model_ema, 0) self.tri_weight = self.args.tri_weight print(self.model) self.source_loader, self.val_loader = data_helper.get_train_dataloader( args, False) self.target_loader = data_helper.get_val_dataloader(args) self.test_loaders = { "val": self.val_loader, "test": self.target_loader } self.len_dataloader = len(self.source_loader) print("Dataset size: train %d, val %d, test %d" % (len(self.source_loader.dataset), len( self.val_loader.dataset), len(self.target_loader.dataset))) self.optimizer, self.scheduler = get_optim_and_scheduler( self.model, args.epochs, args.learning_rate) self.jig_weight = args.jig_weight self.n_classes = args.n_classes if args.target in args.source: self.target_id = args.source.index(args.target) print("Target in source: %d" % self.target_id) print(args.source) else: self.target_id = None self.moco_weight_init = self.args.moco_weight self.moco_weight = self.moco_weight_init self.k_triplet = args.k_triplet self.initialize_queue()
def __init__(self, args, device): self.alpha_jigsaw_weight = 0.5 self.alpha_jigsaw_weight_target = 0.5 self.alpha_rotation_weight = 0.5 self.alpha_rotation_weight_target = 0.5 self.alpha_odd_weight = 0.5 self.alpha_odd_weight_target = 0.5 self.entropi_ni = 0.1 self.args = args self.device = device self.betaJigen = args.betaJigen #if args.rotation == True: model = model_factory.get_network(args.network)(classes=args.n_classes, jigsaw_classes=31, odd_classes=10, rotation_classes=4) # elif args.oddOneOut == True: # model = model_factory.get_network(args.network)(classes=args.n_classes,jigsaw_classes=10) #else: # model = model_factory.get_network(args.network)(classes=args.n_classes,jigsaw_classes=31) self.model = model.to(device) self.source_loader, self.val_loader = data_helper.get_train_dataloader( args) self.target_loader = data_helper.get_val_dataloader(args) self.targetAsSource_loader = data_helper.get_trainTargetAsSource_dataloader( args) self.test_loaders = { "val": self.val_loader, "test": self.target_loader } self.len_dataloader = len(self.source_loader) print( "Dataset size: train %d, val %d, test %d" % (len(self.source_loader.dataset) + len(self.targetAsSource_loader), len(self.val_loader.dataset), len(self.target_loader.dataset))) self.optimizer, self.scheduler = get_optim_and_scheduler( model, args.epochs, args.learning_rate, args.train_all) self.n_classes = args.n_classes if args.oddOneOut == True and args.rotation == True: self.nTasks = 4 elif args.oddOneOut == True or args.rotation == True: self.nTasks = 3 else: self.nTasks = 2
def main(): args = get_args() model = model_factory.get_network(args.network)( jigsaw_classes=args.jigsaw_n_classes + 1, classes=args.n_classes) save_path = os.path.join("logs", args.folder_name, "m-m-m-m_to_m", "0") if not os.path.exists(save_path): os.makedirs(save_path) save_path = os.path.join(save_path, "best_model.pth") torch.save({"model_state_dict": model.state_dict()}, save_path)
def eval(model_name, dataset_dir, num_classes, gpus, batch_size=1, num_workers=2): # Setting up gpu environment os.environ["CUDA_VISIBLE_DEVICES"] = ','.join(gpus) # Setting up dataload for evaluation valdir = os.path.join(dataset_dir, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) val_loader = torch.utils.data.DataLoader(datasets.ImageFolder( valdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=True) # Loading the model model = model_factory.get_network(model_name, pretrained=True) model.cuda() model.eval() correct = 0 total = 0 # Running the evaluation with torch.no_grad(): for i_batch, (images, lables) in enumerate(val_loader): images = images.cuda() outputs = model(images) _, predicted = torch.max(outputs.data, 1) predicted = predicted.cpu() total += lables.size(0) correct += (predicted == lables).sum().item() if (total >= 1000): break print("Total: %d, Accuracy: %f " % (total, float(correct / total))) print("Finish the Evaluation")
def __init__(self, args, device): self.args = args self.device = device model = model_factory.get_network(args.network)(classes=args.n_classes) self.model = model.to(device) self.source_loader, self.val_loader = data_helper.get_train_dataloader(args) self.target_loader = data_helper.get_val_dataloader(args) self.test_loaders = {"val": self.val_loader, "test": self.target_loader} self.len_dataloader = len(self.source_loader) print("Dataset size: train %d, val %d, test %d" % (len(self.source_loader.dataset), len(self.val_loader.dataset), len(self.target_loader.dataset))) self.optimizer, self.scheduler = get_optim_and_scheduler(model, args.epochs, args.learning_rate, args.train_all) self.n_classes = args.n_classes
def __init__(self, args, device): self.args = args self.device = device model = model_factory.get_network(args.network)( pretrained=args.imagenet, jigsaw_classes=args.jigsaw_n_classes + 1, classes=args.n_classes) self.model = model.to(device) # print(self.model) self.source_loader, self.val_loader = data_helper.get_train_dataloader( args, patches=model.is_patch_based()) self.target_loader = data_helper.get_jigsaw_val_dataloader( args, patches=model.is_patch_based()) self.test_loaders = { "val": self.val_loader, "test": self.target_loader } self.len_dataloader = len(self.source_loader) print("Dataset size: train %d, val %d, test %d" % (len(self.source_loader.dataset), len( self.val_loader.dataset), len(self.target_loader.dataset))) self.optimizer, self.scheduler = get_optim_and_scheduler( model, args.epochs, args.learning_rate, args.train_all, nesterov=args.nesterov) self.jig_weight = args.jig_weight self.only_non_scrambled = args.classify_only_sane self.n_classes = args.n_classes if args.target in args.source: self.target_id = args.source.index(args.target) print("Target in source: %d" % self.target_id) print(args.source) else: self.target_id = None self.best_val_jigsaw = 0.0 self.best_jigsaw_acc = 0.0 _, logname = Logger.get_name_from_args(args) self.folder_name = "%s/%s_to_%s/%s" % (args.folder_name, "-".join( sorted(args.source)), args.target, logname)
def __init__(self, args): args.source = [d for d in dataset[args.dataset] if d != args.target] self.args = args self.device = "cuda" if torch.cuda.is_available() else "cpu" main_model, dis_model, c_model, cp_model = model_factory.get_network( args.network)(num_classes=args.num_classes, num_domains=len(args.source)) self.main_model = self._model2device(main_model) self.dis_model = self._model2device(dis_model) self.c_model = self._model2device(c_model) self.cp_model = self._model2device(cp_model) self.source_loader_list, self.val_loader, self.img_num_per_domain = get_train_dataloader( args) self.target_loader = get_val_dataloader(args) self.test_loaders = { "val": self.val_loader, "test": self.target_loader } print("Dataset size: train %d, val %d, test %d" % (sum(self.img_num_per_domain), len( self.val_loader.dataset), len(self.target_loader.dataset))) self.optimizer, self.scheduler = get_optim_and_scheduler( [self.main_model, self.dis_model, self.c_model, self.cp_model], [args.lr, args.lr_d, args.lr_c, args.lr_cp], epochs=args.epochs, lr_steps=args.lr_steps, gamma=args.lr_gamma) self.num_classes = args.num_classes self.num_domains = len(args.source) self.base_dir = os.path.join(args.exp_folder, args.network, args.dataset) self.save_dir = os.path.join(self.base_dir, args.target) if not os.path.exists(self.save_dir): os.makedirs(self.save_dir) save_options(args, self.save_dir) self.log_file = os.path.join(self.save_dir, "loss_log.txt")
def __init__(self, args, device): self.args = args self.device = device model = model_factory.get_network(args.network)( jigsaw_classes=args.jigsaw_n_classes + 1, classes=args.n_classes) self.model = model.to(device) # print(self.model) if args.target in args.source: print( "No need to include target in source, it is automatically done by this script" ) k = args.source.index(args.target) args.source = args.source[:k] + args.source[k + 1:] print("Source: %s" % args.source) self.source_loader, self.val_loader = data_helper.get_train_dataloader( args, patches=model.is_patch_based()) self.target_jig_loader = data_helper.get_target_jigsaw_loader(args) self.target_loader = data_helper.get_val_dataloader( args, patches=model.is_patch_based()) self.test_loaders = { "val": self.val_loader, "test": self.target_loader } self.len_dataloader = len(self.source_loader) print("Dataset size: train %d, target jig: %d, val %d, test %d" % (len(self.source_loader.dataset), len(self.target_jig_loader.dataset), len( self.val_loader.dataset), len(self.target_loader.dataset))) self.optimizer, self.scheduler = get_optim_and_scheduler( model, args.epochs, args.learning_rate, args.train_all, nesterov=args.nesterov) self.jig_weight = args.jig_weight self.target_weight = args.target_weight self.target_entropy = args.entropy_weight self.only_non_scrambled = args.classify_only_sane self.n_classes = args.n_classes
def bfp_quant(model_name, dataset_dir, num_classes, gpus, mantisa_bit, exp_bit, batch_size=1, num_bins=8001, eps=0.0001, num_workers=2, num_examples=10, std=None, mean=None, resize=256, crop=224, exp_act=None, bfp_act_chnl=1, bfp_weight_chnl=1, bfp_quant=1, target_module_list=None, act_bins_factor=3, fc_bins_factor=4, is_online=0): # Setting up gpu environment os.environ["CUDA_VISIBLE_DEVICES"] = ','.join(gpus) # Setting up dataload for evaluation valdir = os.path.join(dataset_dir, 'val') normalize = transforms.Normalize(mean=mean, std=std) # for collect intermediate data use collect_loader = torch.utils.data.DataLoader(datasets.ImageFolder( valdir, transforms.Compose([ transforms.Resize(resize), transforms.CenterCrop(crop), transforms.ToTensor(), normalize, ])), batch_size=num_examples, shuffle=False, num_workers=num_workers, pin_memory=True) # for validate the bfp model use val_loader = torch.utils.data.DataLoader(datasets.ImageFolder( valdir, transforms.Compose([ transforms.Resize(resize), transforms.CenterCrop(crop), transforms.ToTensor(), normalize, ])), batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=True) # Loading the model model, _ = model_factory.get_network(model_name, pretrained=True) # Insert the hook to record the intermediate result #target_module_list = [nn.BatchNorm2d,nn.Linear] # Insert hook after BN and FC model, intern_outputs = Stat_Collector.insert_hook(model, target_module_list) #model = nn.DataParallel(model) model.cuda() model.eval() # Collect the intermediate result while running number of examples logging.info("Collecting the statistics while running image examples....") images_statistc = torch.empty((1)) with torch.no_grad(): for i_batch, (images, lables) in enumerate(collect_loader): images = images.cuda() outputs = model(images) #print(lables) _, predicted = torch.max(outputs.data, 1) predicted = predicted.cpu( ) # needs to verify if this line can be deleted # Collect the input data image_shape = images.shape images_statistc = torch.reshape(images, (image_shape[0], image_shape[1], image_shape[2] * image_shape[3])) break # Deternmining the optimal exponent of activation and # Constructing the distribution for tensorboardX visualization logging.info( "Determining the optimal exponent by minimizing the KL divergence....") start = time.time() opt_exp_act_list = [] max_exp_act_list = [] # For original input opt_exp, max_exp = Utils.find_exp_act(images_statistc, mantisa_bit, exp_bit, group=3, eps=eps, bins_factor=act_bins_factor) opt_exp_act_list.append(opt_exp) max_exp_act_list.append(max_exp) sc_layer_num = [7, 10, 17, 20, 23, 30, 33, 36, 39, 42, 49, 52] ds_sc_layer_num = [14, 27, 46] mobilev2_sc_layer_num = [9, 15, 18, 24, 27, 30, 36, 39, 45, 48] for i, intern_output in enumerate(intern_outputs): #print ("No.", i, " ", intern_output.out_features.shape) #Deternmining the optimal exponent by minimizing the KL_Divergence in channel-wise manner if (isinstance(intern_output.m, nn.Conv2d) or isinstance(intern_output.m, nn.BatchNorm2d)): intern_shape = intern_output.out_features.shape #print (intern_shape, "No.", i) # assmue internal activation has shape: (batch, channel, height, width) if ((model_name == "resnet50") and (i in sc_layer_num)): #print ("Before:", intern_shape[1]) intern_features1 = intern_output.out_features intern_features2 = intern_outputs[i - 3].out_features intern_features = torch.cat( (intern_features1, intern_features2), 0) intern_features = torch.reshape( intern_features, (2 * intern_shape[0], intern_shape[1], intern_shape[2] * intern_shape[3])) #print (intern_features.shape) opt_exp, max_exp = Utils.find_exp_act( intern_features, mantisa_bit, exp_bit, group=bfp_act_chnl, eps=eps, bins_factor=act_bins_factor) opt_exp_act_list.append(opt_exp) max_exp_act_list.append(max_exp) #print ("After:", len(opt_exp)) elif ((model_name == "resnet50") and (i in ds_sc_layer_num)): intern_features1 = intern_output.out_features intern_features2 = intern_outputs[i - 1].out_features intern_features = torch.cat( (intern_features1, intern_features2), 0) intern_features = torch.reshape( intern_features, (2 * intern_shape[0], intern_shape[1], intern_shape[2] * intern_shape[3])) #print (intern_features.shape) opt_exp, max_exp = Utils.find_exp_act( intern_features, mantisa_bit, exp_bit, group=bfp_act_chnl, eps=eps, bins_factor=act_bins_factor) #print ("Current shape", np.shape(opt_exp), " No.", i) #print ("Previous shape", np.shape(opt_exp_act_list[i]), " No.", i-1) opt_exp_act_list.append(opt_exp) max_exp_act_list.append(max_exp) opt_exp_act_list[i] = (opt_exp) #Starting from 1 max_exp_act_list[i] = (max_exp) elif ((model_name == "mobilenetv2") and (i in mobilev2_sc_layer_num)): intern_features1 = intern_output.out_features intern_features2 = intern_outputs[i - 3].out_features intern_features = torch.cat( (intern_features1, intern_features2), 0) intern_features = torch.reshape( intern_features, (2 * intern_shape[0], intern_shape[1], intern_shape[2] * intern_shape[3])) #print (intern_features.shape) opt_exp, max_exp = Utils.find_exp_act( intern_features, mantisa_bit, exp_bit, group=bfp_act_chnl, eps=eps, bins_factor=act_bins_factor) opt_exp_act_list.append(opt_exp) ##changed max_exp_act_list.append(max_exp) else: intern_features = torch.reshape( intern_output.out_features, (intern_shape[0], intern_shape[1], intern_shape[2] * intern_shape[3])) opt_exp, max_exp = Utils.find_exp_act( intern_features, mantisa_bit, exp_bit, group=bfp_act_chnl, eps=eps, bins_factor=act_bins_factor) opt_exp_act_list.append(opt_exp) ##changed max_exp_act_list.append(max_exp) # ploting the distribution #writer.add_histogram("layer%d" % (i), intern_output.out_features.cpu().data.numpy(), bins='auto') quant_tensor = BFPActivation.transform_activation_offline( intern_output.out_features, exp_bit, mantisa_bit, max_exp) #writer.add_histogram("layer%d" % (i), quant_tensor.cpu().data.numpy(), bins='auto') quant_tensor = BFPActivation.transform_activation_offline( intern_output.out_features, exp_bit, mantisa_bit, opt_exp) writer.add_histogram("layer%d" % (i), quant_tensor.cpu().data.numpy(), bins='auto') #print (np.shape(opt_exp), " No.", i) elif (isinstance(intern_output.m, nn.Linear)): intern_shape = intern_output.out_features.shape opt_exp, max_exp = Utils.find_exp_fc(intern_output.out_features, mantisa_bit, exp_bit, block_size=intern_shape[1], eps=eps, bins_factor=fc_bins_factor) #print ("shape of fc exponent:", np.shape(opt_exp)) opt_exp_act_list.append(max_exp) max_exp_act_list.append(max_exp) else: intern_shape = intern_output.in_features[0].shape intern_features = torch.reshape( intern_output.in_features[0], (intern_shape[0], intern_shape[1], intern_shape[2] * intern_shape[3])) opt_exp, max_exp = Utils.find_exp_act(intern_features, mantisa_bit, exp_bit, group=bfp_act_chnl, eps=eps, bins_factor=act_bins_factor) opt_exp_act_list.append(opt_exp) max_exp_act_list.append(max_exp) #logging.info("The internal shape: %s" % ((str)(intern_output.out_features.shape))) end = time.time() logging.info( "It took %f second to determine the optimal shared exponent for each block." % ((float)(end - start))) logging.info("The shape of collect exponents: %s" % ((str)(np.shape(opt_exp_act_list)))) # Building a BFP model by insert BFPAct and BFPWeiht based on opt_exp_act_list torch.cuda.empty_cache() if (exp_act == 'kl'): exp_act_list = opt_exp_act_list else: exp_act_list = max_exp_act_list if (is_online == 1): model_name = "br_" + model_name bfp_model, weight_exp_list = model_factory.get_network( model_name, pretrained=True, bfp=(bfp_quant == 1), group=bfp_weight_chnl, mantisa_bit=mantisa_bit, exp_bit=exp_bit, opt_exp_act_list=exp_act_list) writer.close()