def build_model(): global cfg, loaded_ckpt, lg if rank == 0: lg.info('==> Building model..') net: torch.nn.Module = { 'FCNet': FCNet, }[cfg.model.name](**cfg.model.kwargs) init_params(net) if loaded_ckpt is not None: net.load_state_dict(loaded_ckpt['model']) num_para = sum(p.numel() for p in net.parameters()) / 1e6 if rank == 0: lg.info( f'==> Building model complete, type: {type(net)}, param:{num_para} * 10^6.\n' ) return net.cuda() if using_gpu else net
def __init__(self, num_classes, backbone, BatchNorm): super(Decoder, self).__init__() if backbone in {'resnet50', 'resnet101'} or backbone == 'drn': low_level_inplanes = 256 elif backbone == 'xception': low_level_inplanes = 128 elif backbone == 'mobilenet': low_level_inplanes = 24 else: raise NotImplementedError self.conv1 = nn.Conv2d(low_level_inplanes, 48, 1, bias=False) self.bn1 = BatchNorm(48) self.relu = nn.ReLU() self.last_conv = nn.Sequential( nn.Conv2d(304, 256, kernel_size=3, stride=1, padding=1, bias=False), BatchNorm(256), nn.ReLU(), nn.Dropout(0.5), nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1, bias=False), BatchNorm(256), nn.ReLU(), nn.Dropout(0.1), nn.Conv2d(256, num_classes, kernel_size=1, stride=1)) init_params(self)
def index(): url = request.args.get("url", "") if url == "": vtuber = request.args.get("vtuber", "") if vtuber != "": # print(request.args) save_data(request.args) params = init_params() return render_template("index.html", params=params) else: # URLが入力された時 _id = url.split("?v=")[-1] if exists_id(DF, _id): params = init_params() params["existsUrl"] = True params["url"] = url print(params) return render_template("index.html", params=params) data = search_youtube(_id) # print(data) params = create_params(data) return render_template("index.html", params=params)
def init_weights(self, u_emb_para, i_emb_para, rec_para): """ Initialize parameters :param u_emb_para: User embedding parameters :param i_emb_para: Item embedding parameters :param rec_para: Recommendation parameters """ # Initialize user embedding parameters init_params(self.user_embedding.parameters(), u_emb_para) # Initialize item embedding parameters init_params(self.item_embedding.parameters(), i_emb_para) # Initialize recommendation model parameters init_params(self.rec_model.parameters(), rec_para)
def defaultXMLGen_Simulator(sensor_config_file): print('\n Default Components : Version [%s]' % version) print('\n Config File %s' % sensor_config_file) #print('\n Config File %s' %sys.argv[1] #if (len(sys.argv) < 2 ) : # utils.error('Usage default XML <input_filename>') # return #sensor_config_file=sys.argv[1] """ System Parameters Related to Project """ sys_params = {} utils.init_params(sys_params) # utils.get_params(sensor_config_file, sys_params) # for wdr_mode in ['linear', 'wdr']: GEN_Params = {} init_generic_params(GEN_Params, sys_params) GEN_Params['WDR_MODE'] = wdr_mode AWB_Params = {} init_dcc_regions(sys_params, AWB_Params) generate_awb_xml(sys_params, AWB_Params, GEN_Params) RGB2RGB1_Params = {} init_dcc_regions(sys_params, RGB2RGB1_Params) generate_rgb2rgb1_xml(sys_params, RGB2RGB1_Params, GEN_Params) CFAI_Params = {} init_dcc_regions(sys_params, CFAI_Params) generate_cfai_xml(sys_params, CFAI_Params, GEN_Params) H3A_AEWB_Params = {} init_dcc_regions(sys_params, H3A_AEWB_Params) generate_h3a_aewb_xml(sys_params, H3A_AEWB_Params, GEN_Params) LDC_Params = {} init_dcc_regions(sys_params, LDC_Params) generate_ldc_xml(sys_params, LDC_Params, GEN_Params) viss_blc_Params = {} init_dcc_regions(sys_params, viss_blc_Params) generate_viss_blc_xml(sys_params, viss_blc_Params, GEN_Params) NSF4_Params = {} init_dcc_regions(sys_params, NSF4_Params) generate_nsf4_xml(sys_params, NSF4_Params, GEN_Params) H3AMUX_Params = {} init_dcc_regions(sys_params, H3AMUX_Params) generate_h3amux_xml(sys_params, H3AMUX_Params, GEN_Params) if (wdr_mode == 'wdr'): GLBCE_Params = {} init_dcc_regions(sys_params, GLBCE_Params) generate_glbce_xml(sys_params, GLBCE_Params, GEN_Params) DECMP_Params = {} init_dcc_regions(sys_params, DECMP_Params) generate_decmp_xml(sys_params, DECMP_Params, GEN_Params) generate_dcc_gen_script(sys_params, GEN_Params) return
netD.apply(weights_init) if opt.netD != '': netD.load_state_dict(torch.load(opt.netD)) logger.info(netD) ##### Classifier ##### if opt.netT != '': chk = torch.load(opt.netT) netT = chk['netT'] best_acc = chk['acc'] netT_epoch = chk['epoch'] else: # netT = ResNet18() netT = MnistClassifier(source_channels, target_channels, num_classes, opt.ngpu) init_params(netT) best_acc = 0 netT_epoch = 0 logger.info(netT) criterion_D = nn.BCEWithLogitsLoss() criterion_G = nn.BCEWithLogitsLoss() criterion_T = nn.CrossEntropyLoss() inputs = torch.FloatTensor(opt.batchSize, 3, opt.imageSize, opt.imageSize) noise = torch.FloatTensor(opt.batchSize, nz, 1, 1) fixed_noise = torch.FloatTensor(opt.batchSize, nz).uniform_(-1, 1) label = torch.FloatTensor(opt.batchSize) real_label = 1 fake_label = 0
def run(args): if not args.single_test: import pidfile resfile = pidfile.exclusive_dirfn( os.path.join(args.r_dir, args.save_dir)) if args.log_per_task: args.prec_log = args.iters args.loss_log = args.iters # -create plots- and results-directories if needed if not os.path.isdir(args.r_dir): os.mkdir(args.r_dir) if args.pdf and not os.path.isdir(args.p_dir): os.mkdir(args.p_dir) # set cuda cuda = torch.cuda.is_available() and args.cuda device = torch.device("cuda" if cuda else "cpu") # set random seeds random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if cuda: torch.cuda.manual_seed(args.seed) torch.cuda.manual_seed_all(args.seed) scenario = args.scenario #------------------------------------------------------------------------------------------------- # DATA #------------------------------------------------------------------------------------------------- (train_datasets, test_datasets), config = get_multitask_experiment( args, name=args.experiment, scenario=scenario, tasks=args.tasks, data_dir=args.d_dir, verbose=True, exception=True if args.seed == 0 else False, ) args.tasks = len(config['labels_per_task']) args.labels_per_task = config['labels_per_task'] if not args.task_boundary: args.iterations_per_virtual_epc = config['iterations_per_virtual_epc'] args.task_dict = config['task_dict'] #------------------------------------------------------------------------------------------------- # MODEL #------------------------------------------------------------------------------------------------- if args.ebm: model = EBM(args, image_size=config['size'], image_channels=config['channels'], classes=config['num_classes'], fc_units=args.fc_units).to(device) else: model = Classifier(args, image_size=config['size'], image_channels=config['channels'], classes=config['num_classes'], fc_units=args.fc_units).to(device) if args.experiment == 'cifar100': model = utils.init_params(model, args) for param in model.convE.parameters(): param.requires_grad = False if args.pretrain: checkpoint = torch.load(args.pretrain) best_acc = checkpoint['best_acc'] checkpoint_state = checkpoint['state_dict'] print( '-----------------------------------------------------------------------------' ) print('load pretrained model %s' % args.pretrain) print('best_acc', best_acc) print( '-----------------------------------------------------------------------------' ) model_dict = model.fcE.state_dict() checkpoint_state = { k[7:]: v for k, v in checkpoint_state.items() if k[7:] in model_dict } ## remove module. del checkpoint_state['classifier.weight'] del checkpoint_state['classifier.bias'] if 'y_ebm.weight' in checkpoint_state: del checkpoint_state['y_ebm.weight'] model_dict.update(checkpoint_state) model.fcE.load_state_dict(model_dict) for param in model.fcE.model.parameters(): param.requires_grad = False model.optim_list = [{ 'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.lr }] model.optim_type = args.optimizer if model.optim_type in ("adam", "adam_reset"): model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999)) elif model.optim_type == "sgd": model.optimizer = optim.SGD(model.optim_list) else: raise ValueError( "Unrecognized optimizer, '{}' is not currently a valid option". format(args.optimizer)) #------------------------------------------------------------------------------------------------- # CL-STRATEGY: ALLOCATION #------------------------------------------------------------------------------------------------- # Elastic Weight Consolidation (EWC) if isinstance(model, ContinualLearner): model.ewc_lambda = args.ewc_lambda if args.ewc else 0 if args.ewc: model.fisher_n = args.fisher_n model.gamma = args.gamma model.online = args.online model.emp_FI = args.emp_fi # Synpatic Intelligence (SI) if isinstance(model, ContinualLearner): model.si_c = args.si_c if args.si else 0 if args.si: model.epsilon = args.epsilon #------------------------------------------------------------------------------------------------- # Get parameter-stamp (and print on screen) #------------------------------------------------------------------------------------------------- param_stamp = get_param_stamp(args, model.name, verbose=True) param_stamp = param_stamp + '--' + args.model_name # -define [precision_dict] to keep track of performance during training for storing and for later plotting in pdf precision_dict = evaluate.initiate_precision_dict(args.tasks) #-------------------------------------------------------------------------------------------------# #---------------------# #----- CALLBACKS -----# #---------------------# solver_loss_cbs = [ cb._solver_loss_cb(log=args.loss_log, model=model, tasks=args.tasks, iters_per_task=args.iters) ] eval_cb = cb._eval_cb(log=args.prec_log, test_datasets=test_datasets, visdom=args.visdom, precision_dict=None, iters_per_task=args.iters, test_size=args.prec_n, labels_per_task=config['labels_per_task'], scenario=scenario) eval_cb_full = cb._eval_cb(log=args.iters, test_datasets=test_datasets, precision_dict=precision_dict, iters_per_task=args.iters, labels_per_task=config['labels_per_task'], scenario=scenario) eval_cbs = [eval_cb, eval_cb_full] #------------------------------------------------------------------------------------------------- # TRAINING #------------------------------------------------------------------------------------------------- print("--> Training:") start = time.time() if args.task_boundary: train_cl(args, model, train_datasets, scenario=scenario, labels_per_task=config['labels_per_task'], iters=args.iters, batch_size=args.batch, eval_cbs=eval_cbs, loss_cbs=solver_loss_cbs) else: train_cl_noboundary(args, model, train_datasets, scenario=scenario, labels_per_task=config['labels_per_task'], iters=args.iters, batch_size=args.batch, eval_cbs=eval_cbs, loss_cbs=solver_loss_cbs) training_time = time.time() - start #------------------------------------------------------------------------------------------------- # EVALUATION #------------------------------------------------------------------------------------------------- print("\n\n--> Evaluation ({}-incremental learning scenario):".format( args.scenario)) if args.ebm: precs = [ evaluate.validate_ebm(args, model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=False, current_task=args.tasks) for i in range(args.tasks) ] else: precs = [ evaluate.validate(args, model, test_datasets[i], verbose=False, test_size=None, task=i + 1, with_exemplars=False, current_task=args.tasks) for i in range(args.tasks) ] print("\n Precision on test-set (softmax classification):") for i in range(args.tasks): print(" - Task {}: {:.4f}".format(i + 1, precs[i])) average_precs = sum(precs) / args.tasks print('average precision over all {} tasks: {:.4f}'.format( args.tasks, average_precs)) #------------------------------------------------------------------------------------------------- # OUTPUT #------------------------------------------------------------------------------------------------- if not os.path.exists(os.path.join(args.r_dir, args.save_dir)): os.makedirs(os.path.join(args.r_dir, args.save_dir)) output_file = open( "{}/{}/{}.txt".format(args.r_dir, args.save_dir, param_stamp), 'w') output_file.write("Training time {} \n".format(training_time)) for i in range(args.tasks): output_file.write(" - Task {}: {:.4f}".format(i + 1, precs[i])) output_file.write("\n") output_file.write(' - Average {}\n'.format(average_precs)) output_file.close() file_name = "{}/{}/{}".format(args.r_dir, args.save_dir, param_stamp) utils.save_object(precision_dict, file_name) if args.pdf: pp = visual_plt.open_pdf("{}/{}/{}.pdf".format(args.r_dir, args.save_dir, param_stamp)) # -show metrics reflecting progression during training figure_list = [] #-> create list to store all figures to be plotted # -generate all figures (and store them in [figure_list]) figure = visual_plt.plot_lines( precision_dict["all_tasks"], x_axes=precision_dict["x_task"], line_names=['task {}'.format(i + 1) for i in range(args.tasks)]) figure_list.append(figure) figure = visual_plt.plot_lines([precision_dict["average"]], x_axes=precision_dict["x_task"], line_names=['average all tasks so far']) figure_list.append(figure) # -add figures to pdf (and close this pdf). for figure in figure_list: pp.savefig(figure) pp.close() if not args.single_test: resfile.done()
decode_original, result, speed, 100 * packet_loss) tkinter.messagebox.showinfo('传输结果', show_text) def init_ui(self): ''' 描述:初始化gui 参数:无 返回:无 ''' self.window = Tk() self.label1 = Label(self.window, text="未开始录音") self.label1.grid(row=0, column=0, stick=W, pady=10) self.label2 = Label(self.window, text="原先的信息(必须是ASCII码)") self.label2.grid(row=1, column=0, stick=W, pady=10) self.entry1 = Entry(self.window, width=100) self.entry1.grid(row=1, column=1, stick=W, pady=10) self.button1 = Button(self.window, text='开始录音', command=self.start_record) self.button1.grid(row=2, column=0, stick=W, pady=10) self.button2 = Button(self.window, text='结束录音', command=self.stop_record) self.button2.grid(row=2, column=1, stick=W, pady=10) self.window.mainloop() if __name__ == "__main__": args = init_params() receiver = Receiver(args) receiver.init_ui()
# 0. Postpone training for TRAINING_DELAY seconds. time.sleep(TRAINING_DELAY) # 1. Create a directory for resume (model) parameters, CSV logs, and tensor-board summary logs. log_dir = utils.mkdir_for_logs(MODEL.__name__, DATASET_FOLDER_NAME, OBJ_FUNC) # 2. Instantiate a tensorboard writer and CSV writers. tensor_board_writer = SummaryWriter(log_dir=log_dir) train_csv_file, train_csv_writer, valid_csv_file, valid_csv_writer = \ utils.csv_train_valid_writer_for_logs(log_dir=log_dir) # 3. Instantiate and initialize a CNN model. model = MODEL(num_classes=NUM_CLASSES) utils.init_params(model) # 4. Load the model to CPU or GPU. device = 'cuda' if torch.cuda.is_available() else 'cpu' model = model.to(device) # 5. (If GPU,) Turn on 'Data Parallel' mode. if device == 'cuda': model = torch.nn.DataParallel(model) cudnn.benchmark = True print('model_info:', MODEL.__name__, '/ device:', device, '/ obj.:', OBJ_FUNC, '/ dataset:', DATASET_FOLDER_NAME) # 6. Load the train and valid dataset. train_loader, valid_loader = \
def main( arch = 'resnet_weight', num_epochs = 160, batch_size = 256, load_checkpoint_dir = 'resnet56/2/model_best.pth.tar', save_checkpoint_dir = 'resnet56/2/distill/scratch/', save_prune = 'resnet56/2/', parrallel=True, scratch=True): # Data loading code transform = transforms.Compose([ transforms.ToTensor(), ]) # train_mean, train_std = get_mean_and_std(torchvision.datasets.CIFAR10('./cifar10_data', train=True, download=True, transform=transform)) # test_mean, test_std = get_mean_and_std(torchvision.datasets.CIFAR10('./cifar10_data', train=False, download=True, transform=transform)) # print(train_mean, train_std, test_mean, test_std) data_transform = { 'train' : transforms.Compose([ transforms.Pad(4), transforms.RandomCrop(32), transforms.RandomHorizontalFlip(), transforms.ToTensor(), # transforms.Normalize(train_mean, train_std), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]), 'test' : transforms.Compose([ transforms.ToTensor(), # transforms.Normalize(test_mean, test_std), transforms.Normalize((0.4942, 0.4851, 0.4504), (0.2020, 0.1991, 0.2011)), ]) } train_dataset = torchvision.datasets.CIFAR10('./cifar10_data', train=True, transform=data_transform['train'], download=False) test_dataset = torchvision.datasets.CIFAR10('./cifar10_data', train=False, transform=data_transform['test'], download=False) train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False) # Model loading code # Originmodel = OriginNetwork(models.resnet50(pretrained=False, num_classes=10), num_classes=10) # Submodel = SubNetwork(models.resnet50(pretrained=False, num_classes=10), num_classes=10) # Originmodel = models.__dict__[arch](dataset='cifar10', depth=56) # Filter pruning Originmodel = models.__dict__[arch](num_classes=10, depth=56) # Weight pruning # Submodel = models.__dict__[arch](dataset='cifar10', depth=56) # Selecting device device = torch.device('cuda:1' if torch.cuda.is_available() else 'cpu') torch.cuda.set_device(device) Originmodel.to(device) # Submodel.to(device) # resnet56_pruning 에서 device로 옮겨짐. # Loading Checkpoint load_checkpoint(Originmodel, checkpoint_dir=load_checkpoint_dir, parrallel=parrallel) unpruned_acc = test_solo(test_loader, Originmodel, device) # Submodel, num_parameters = resnet56_pruning(Originmodel, dataset='cifar10', save=save_prune, device=device, v='A') # Filter Submodel = resnet56_weight_pruning(Originmodel, percent=0.3) # Weight pruned_acc = test_solo(test_loader, Submodel, device) # Pruned before & after accuracy save with open(os.path.join(save_prune, "prune.txt"), "w") as fp: fp.write("Before pruning Test accuracy : \n"+str(unpruned_acc.item())+"\n") # fp.write("Number of parameters: \n"+str(num_parameters)+"\n") fp.write("Test accuracy: \n"+str(pruned_acc.item())+"\n") # Scratch training if scratch: Submodel.train() init_params(Submodel) test_solo(test_loader, Submodel, device) # Loss & Optimizer criterion = distill_loss(origin_model=Originmodel, subnet_model=Submodel, temperature=5) optimizer = torch.optim.SGD(Submodel.parameters(), lr=0.1, momentum=0.9, weight_decay=1e-4) # scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[15,25,30], gamma= 0.1) scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[80,120], gamma= 0.1) # Scratch # Visualization writer = SummaryWriter(os.path.join(save_checkpoint_dir, 'logs')) # Logging Metrics title = 'ResNet56' logger = Logger(os.path.join(save_checkpoint_dir, 'log.txt'), title=title) logger.set_names(['Learning Rate', 'Train Loss', 'Test Loss', 'Train Acc', 'Test Acc']) best_prec1 = 0 for epoch in range(num_epochs): print('') # Train : Distilling knowledge to Subnet from origin train_loss, train_acc = train(train_loader, Originmodel, Submodel, criterion, optimizer, epoch, device) # Test test_loss, test_acc = test(test_loader, Originmodel, Submodel, criterion, epoch, device) lr = optimizer.param_groups[0]['lr'] # append logger file logger.append([lr, train_loss, test_loss, train_acc, test_acc]) # visualize training process using tensorboardx writer.add_scalar('learning rate', lr) writer.add_scalars('loss', {'train loss':train_loss, 'test loss':test_loss}, epoch+1) writer.add_scalars('accuracy', {'train accuracy':train_acc, 'test accuracy':test_acc}, epoch+1) # Learning Rate scheduler scheduler.step() is_best = test_acc > best_prec1 best_prec1 = max(test_acc, best_prec1) save_checkpoint(state={ 'epoch':epoch+1, 'arch':arch, 'state_dict':Submodel.state_dict(), 'best_prec1':best_prec1, 'optimizer':optimizer.state_dict(), 'loss':criterion.state_dict() }, is_best=is_best, checkpoint=save_checkpoint_dir) logger.close() logger.plot() savefig(os.path.join(save_checkpoint_dir, 'log.jpg')) writer.close() print(f'Best Accuracy : {best_prec1}')
device = 'cuda' if torch.cuda.is_available() else 'cpu' net = net.to(device) if device == 'cuda': net = torch.nn.DataParallel(net) cudnn.benchmark = True if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isdir('checkpoint'), 'Error: no checkpoint directory found!' checkpoint = torch.load('./checkpoint/ckpt.t7') net.load_state_dict(checkpoint['net']) best_acc = checkpoint['acc'] start_epoch = checkpoint['epoch'] else: init_params(net) total_params = sum(p.numel() for p in net.parameters() if p.requires_grad) print('Number of trainable parameters {}'.format(total_params)) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=0.9, weight_decay=5e-4) scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min') # Training def train(epoch): print('\nEpoch: %d' % epoch)
def __init__(self, model_name, block, layers, output_stride, BatchNorm, pretrained=True, enable_dff=False, pretrained_path=None): self.model_name = model_name self.inplanes = 64 super(ResNet, self).__init__() self.enable_dff = enable_dff blocks = [1, 2, 4] if ( not enable_dff) else [1, 1, 1] # rloss: [1,2,4]; dff: [1,1,1] if output_stride == 16: strides = [1, 2, 2, 1] dilations = [1, 1, 1, 2] if (not enable_dff) else [ 2, 2, 2, 4 ] # rloss: [1,1,1,2]; dff: [2,2,2,4] Zhiwei elif output_stride == 8: strides = [1, 2, 1, 1] dilations = [1, 1, 2, 4] else: raise NotImplementedError # Modules Zhiwei if enable_dff: self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=1, padding=3, bias=False) else: self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False) self.bn1 = BatchNorm(64) self.relu = nn.ReLU(inplace=True) self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) self.pretrained_path = pretrained_path self.layer1 = self._make_layer(block, 64, layers[0], stride=strides[0], dilation=dilations[0], BatchNorm=BatchNorm) self.layer2 = self._make_layer(block, 128, layers[1], stride=strides[1], dilation=dilations[1], BatchNorm=BatchNorm) self.layer3 = self._make_layer(block, 256, layers[2], stride=strides[2], dilation=dilations[2], BatchNorm=BatchNorm) self.layer4 = self._make_MG_unit(block, 512, blocks=blocks, stride=strides[3], dilation=dilations[3], BatchNorm=BatchNorm) # self.layer4 = self._make_layer(block, 512, layers[3], stride=strides[3], dilation=dilations[3], BatchNorm=BatchNorm) init_params(self) # self._init_weight() if pretrained: self._load_pretrained_model()
def train(): if prm.optimizer.lower() == 'adam': optimizer = adam elif prm.optimizer.lower() == 'sgd': optimizer = sgd elif prm.optimizer.lower() == 'rmsprop': optimizer = rmsprop elif prm.optimizer.lower() == 'adadelta': optimizer = adadelta options = locals().copy() print 'parameters:', str(options) prm_k = vars(prm).keys() prm_d = vars(prm) prm_k.sort() for x in prm_k: if not x.startswith('__'): print x, '=', prm_d[x] print 'loading Vocabulary...' vocab = utils.load_vocab(prm.vocab_path, prm.n_words) options['vocab'] = vocab options['vocabinv'] = {} for k, v in vocab.items(): options['vocabinv'][v] = k print options print 'Loading Environment...' if prm.engine.lower() == 'lucene': import lucene_search options['engine'] = lucene_search.LuceneSearch() elif prm.engine.lower() == 'elastic': import elastic_search options['engine'] = elastic_search.ElasticSearch() print 'Loading Dataset...' dh5 = dataset_hdf5.DatasetHDF5(prm.dataset_path) qi_train = dh5.get_queries(dset='train') dt_train = dh5.get_doc_ids(dset='train') qi_valid = dh5.get_queries(dset='valid') dt_valid = dh5.get_doc_ids(dset='valid') qi_test = dh5.get_queries(dset='test') dt_test = dh5.get_doc_ids(dset='test') if prm.train_size == -1: train_size = len(qi_train) else: train_size = min(prm.train_size, len(qi_train)) if prm.valid_size == -1: valid_size = len(qi_valid) else: valid_size = min(prm.valid_size, len(qi_valid)) if prm.test_size == -1: test_size = len(qi_test) else: test_size = min(prm.test_size, len(qi_test)) print '%d train examples' % len(qi_train) print '%d valid examples' % len(qi_valid) print '%d test examples' % len(qi_test) # This create the initial parameters as np ndarrays. # Dict name (string) -> np ndarray params, exclude_params = utils.init_params(options) if prm.wordemb_path: print 'loading pre-trained word embeddings' params = utils.load_wemb(params, vocab) options['W'] = params['W'] if prm.reload_model: utils.load_params(prm.reload_model, params) print 'Building model' # This create Theano Shared Variable from the parameters. # Dict name (string) -> Theano Tensor Shared Variable # params and tparams have different copy of the weights. tparams = utils.init_tparams(params) for kk, value in tparams.iteritems(): tparams[kk] = theano.shared(value, name=kk) iin, out, updates, f_pred, consider_constant \ = build_model(tparams, options) # get only parameters that are not in the exclude_params list tparams_ = OrderedDict([(kk, vv) for kk, vv in tparams.iteritems() if kk not in exclude_params]) grads = tensor.grad(out[0], wrt=utils.itemlist(tparams_), consider_constant=consider_constant) lr = tensor.scalar(name='lr') f_grad_shared, f_update = optimizer(lr, tparams_, grads, iin, out, updates) history_errs = [] best_p = None if prm.validFreq == -1: validFreq = len(qi_train) / prm.batch_size_train else: validFreq = prm.validFreq if prm.saveFreq == -1: saveFreq = len(qi_train) / prm.batch_size_train else: saveFreq = prm.saveFreq uidx = 0 # the number of update done estop = False # early stop start_time = time.time() print 'Optimization' try: for eidx in xrange(prm.max_epochs): n_samples = 0 # Get new shuffled index for the training set. kf = utils.get_minibatches_idx(len(qi_train), prm.batch_size_train, shuffle=True) for _, train_index in kf: st = time.time() uidx += 1 qi, qi_i, qi_lst, D_gt_id, D_gt_url = get_samples( qi_train, dt_train, train_index, options) # share the current queries with the search engine. options['current_queries'] = qi_lst n_samples += len(qi) is_train = 1. out = f_grad_shared(qi_i, D_gt_id, is_train) cost = out.pop(0) cost_ent = out.pop(0) lr_t = f_update(prm.lrate) if np.isnan(cost) or np.isinf(cost): print 'NaN detected' return 1., 1., 1. print "options['reformulated_queries']", options[ 'reformulated_queries'] if np.mod(uidx, prm.dispFreq) == 0: print '\n================================================================================' print 'Epoch', eidx, 'Update', uidx, 'Cost', cost, 'LR_t', lr_t print 'Time Minibatch Update: ' + str(time.time() - st) print 'Input Query: ', qi[0].replace('\n', '\\n') print print 'Target Docs: ', str(D_gt_url[0]) print print 'Input Query Vocab: ', utils.idx2text( qi_i[0], options['vocabinv']) for ii in range(prm.n_iterations): prob = out.pop(0) ans = out.pop(0) metrics = out.pop(0) bl = out.pop(0) cost_bl = out.pop(0) D_id = out.pop(0) print "prob", prob print "ans", ans print "bl", bl print "cost_bl", cost_bl print "D_id", D_id print("current_queries", len(options['current_queries']), options['current_queries']) print print 'Iteration', ii print 'Baseline Value', bl.mean(), 'Cost', cost_bl print ' '.join(prm.metrics_map.keys()) print metrics.mean(0) print i = 7 print 'Retrieved Docs: ', str([ options['engine'].id_title_map[d_id] for d_id in D_id[i] ]) print print 'Reformulated Query:', options[ 'reformulated_queries'][ii][i] print 'Current queries:', options['current_queries'][i] print print 'Query ANS: ', for kk, word in enumerate( options['current_queries'][i][:ans.shape[1]]): print "kk, word", kk, word if word not in options['vocab'] and word != '': word += '<unk>' if ans[0, kk] == 1: word = word.upper() print str(word), print print print 'prob[:,:,0].max(1).mean(), prob[:,:,0].mean(), prob[:,:,0].min(1).mean()', prob[:, :, 0].max( 1).mean(), prob[:, :, 0].mean(), prob[:, :, 0].min(1).mean() print 'prob[:,:,1].max(1).mean(), prob[:,:,1].mean(), prob[:,:,1].min(1).mean()', prob[:, :, 1].max( 1).mean(), prob[:, :, 1].mean(), prob[:, :, 1].min(1).mean() print '==================================================================================\n' if np.mod(uidx, validFreq) == 0 or uidx == 1: kf_train = utils.get_minibatches_idx( len(qi_train), prm.batch_size_pred, shuffle=True, max_samples=train_size) kf_valid = utils.get_minibatches_idx( len(qi_valid), prm.batch_size_pred, shuffle=True, max_samples=valid_size) kf_test = utils.get_minibatches_idx(len(qi_test), prm.batch_size_pred, shuffle=True, max_samples=test_size) print '\nEvaluating - Training Set' train_metrics = pred_error(f_pred, qi_train, dt_train, options, kf_train) exit() print '\nEvaluating - Validation Set' valid_metrics = pred_error(f_pred, qi_valid, dt_valid, options, kf_valid) print '\nEvaluating - Test Set' test_metrics = pred_error(f_pred, qi_test, dt_test, options, kf_test) his = [train_metrics, valid_metrics, test_metrics] history_errs.append(his) metric_idx = prm.metrics_map[prm.reward.upper()] if (uidx == 0 or valid_metrics[-1, metric_idx] >= np.array(history_errs)[:, 1, -1, metric_idx].max()): best_p = utils.unzip(tparams) bad_counter = 0 print '=====================================================================================================' print ' '.join(prm.metrics_map.keys()) print print 'Train:' print train_metrics print print 'Valid:' print valid_metrics print print 'Test:' print test_metrics print print '=====================================================================================================' if (len(history_errs) > prm.patience and valid_metrics[-1, metric_idx] <= np.array(history_errs)[:-prm.patience, 1, -1, metric_idx].max()): bad_counter += 1 if bad_counter > prm.patience: print 'Early Stop!' estop = True break if prm.saveto and np.mod(uidx, saveFreq) == 0: print 'Saving...', if best_p is not None: params = best_p else: params = utils.unzip(tparams) np.savez(prm.saveto, history_errs=history_errs, **params) print 'Done' print 'Seen %d samples' % n_samples if estop: break except KeyboardInterrupt: print "Training interupted" return
def init_u_mem_weights(self, u_emb_para, mu, tao, i_emb_para, rec_para): init_u_mem_params(self.user_embedding.parameters(), u_emb_para, mu, tao) init_params(self.item_embedding.parameters(), i_emb_para) init_params(self.rec_model.parameters(), rec_para)