def ewc_process_without_split(train_loader, test_loader, labels, online=False, result_file='./ewc_without_split.txt'): gpu = torch.device('cuda:0') model = model_retrieval().cuda(gpu) optimizer = optim.SGD(params=model.parameters(), lr=args.first_lr) ewcs = [] for task in range(args.num_task): print('Training Task {}... Labels: {}'.format(task, labels[task])) if task == 0: for iteration in range(args.iterations): loss = normal_train(model, labels[task], optimizer, train_loader[task], gpu) print('Iteration: {}\tLoss:{}'.format(iteration, loss)) acc = test_model(model, labels[task], test_loader[task], gpu) print('Test Task: {}\tAccuracy: {}'.format(task, acc)) with open(result_file, 'a') as f: f.write('{}\t{}\t{}\t{}\t{}\t{}\n'.format(task, iteration, loss, task, task, acc)) else: for param_group in optimizer.param_groups: param_group['lr'] = args.lr for iteration in range(args.iterations): if online: loss = ewc_train(model, labels[task], optimizer, train_loader[task], ewcs[-1:], args.lam, gpu) else: loss = ewc_train(model, labels[task], optimizer, train_loader[task], ewcs, args.lam, gpu) print('Iteration: {}\tLoss:{}'.format(iteration, loss)) for sub_task in range(task + 1): acc = test_model(model, labels[sub_task], test_loader[sub_task], gpu) print('Test Task: {}\tAccuracy: {}'.format(sub_task, acc)) with open(result_file, 'a') as f: # Current server parameter (task) + device parameter (task) --> training a given task f.write('{}\t{}\t{}\t{}\t{}\n'.format(task, iteration, loss, sub_task, acc)) ewcs.append(EWC(model, train_loader[task], gpu))
def train_model(self, fname=None): time.sleep(1) try: self.load(fname) print("File load successful.") except Exception: print("File load failed.") image_dataset, dataset_size = utils.create_dataset( batch_size=batch_size) dataset_size = dataset_size - 1 print("Dataset size is", dataset_size) print("Total number of images is", dataset_size * batch_size) record_batch = utils.record_steps(int(dataset_size / 2)) start_time = time.time() print("Beginning training at", start_time) for i in range(num_epochs): print("Starting epoch {}/{}".format(i, num_epochs)) start = time.time() batch_on = 0 for source in zip(image_dataset.take(int(dataset_size / 2))): loss_identity, kl_loss = train_step(self, source, optimizer) if batch_on % record_batch == 0: print("Beginning batch #" + str(batch_on), 'out of', int(dataset_size / 2), 'of size', batch_size) self.loss_identity += [loss_identity] self.kl_loss += [kl_loss] batch_on += 1 duration = time.time() - start print(int(duration / 60), "minutes &", int(duration % 60), "seconds, for epoch", i) if i % record_epochs == 0: self.loss['Identity'] = self.loss_identity self.loss['KL'] = self.kl_loss utils.test_model(self, source, num=i, test=True, name=model_name, details='identity') for style in zip(image_dataset.take(1)): utils.test_model(self, source, style, num=i, test=True, name=model_name, details='transfer') break #act = keract.get_activations(self, source) #keract.display_activations(act) print('\n') image_dataset, _ = utils.create_dataset(batch_size=batch_size) self.save(fname) time.sleep(.5) print('Training completed in', int((time.time() - start_time) / 60), "minutes &", int(duration % 60), "seconds")
def _train(model, optimizer, scheduler, checkpointer, epochs, train_loader, test_loader, stat_tracker, log_dir, device): ''' Training loop to train classifiers on top of an encoder with fixed weights. -- e.g., use this for eval or running on new data ''' # If mixed precision is on, will add the necessary hooks into the model and # optimizer for half precision conversions model, optimizer = mixed_precision.initialize(model, optimizer) # ... time_start = time.time() total_updates = 0 next_epoch, total_updates = checkpointer.get_current_position( classifier=True) for epoch in range(next_epoch, epochs): epoch_updates = 0 epoch_stats = AverageMeterSet() for _, ((images1, images2), labels) in enumerate(train_loader): # get data and info about this minibatch images1 = images1.to(device) images2 = images2.to(device) labels = labels.to(device) # run forward pass through model and collect activations res_dict = model(x1=images1, x2=images2, class_only=True) lgt_glb_mlp, lgt_glb_lin = res_dict['class'] # compute total loss for optimization loss = (loss_xent(lgt_glb_mlp, labels) + loss_xent(lgt_glb_lin, labels)) # do optimizer step for encoder optimizer.zero_grad() mixed_precision.backward( loss, optimizer) # special mixed precision stuff optimizer.step() # record loss and accuracy on minibatch epoch_stats.update('loss', loss.item(), n=1) update_train_accuracies(epoch_stats, labels, lgt_glb_mlp, lgt_glb_lin) # shortcut diagnostics to deal with long epochs total_updates += 1 epoch_updates += 1 if (total_updates % 100) == 0: time_stop = time.time() spu = (time_stop - time_start) / 100. print( 'Epoch {0:d}, {1:d} updates -- {2:.4f} sec/update'.format( epoch, epoch_updates, spu)) time_start = time.time() # step learning rate scheduler scheduler.step(epoch) # record diagnostics test_model(model, test_loader, device, epoch_stats, max_evals=500000) epoch_str = epoch_stats.pretty_string(ignore=model.tasks) diag_str = '{0:d}: {1:s}'.format(epoch, epoch_str) print(diag_str) sys.stdout.flush() stat_tracker.record_stats(epoch_stats.averages(epoch, prefix='eval/')) checkpointer.update(epoch + 1, total_updates, classifier=True)
def train(_config, resume: bool = False, test: bool = False): print(json.dumps(config, indent=4)) device = torch.device(_config['device']) os.environ["CUDA_VISIBLE_DEVICES"] = str(device.index) device = torch.device(0) dataset = _config['data']['dataset'] model_name = _config['model']['name'] optimizer_name = _config['optimizer']['name'] scheduler_name = _config['scheduler']['name'] loss = utils.get_loss(_config['loss']['name']) loss.to(device) model = create_model(dataset, _config['model'][model_name], _config['model']['stadaptor'], device) optimizer = utils.get_optimizer(optimizer_name, model.parameters(), **_config['optimizer'][optimizer_name]) scheduler = None if scheduler_name is not None: scheduler = utils.get_scheduler(scheduler_name, optimizer, **_config['scheduler'][scheduler_name]) save_folder = os.path.join('saves', dataset, _config['name']) if not resume and not test: shutil.rmtree(save_folder, ignore_errors=True) os.makedirs(save_folder) with open(os.path.join(save_folder, 'config.yaml'), 'w+') as _f: yaml.safe_dump(_config, _f) datasets = utils.get_datasets(dataset, _config['data']['input_dim'], _config['data']['output_dim']) scaler = utils.ZScoreScaler(datasets['train'].mean, datasets['train'].std) trainer = utils.OursTrainer(model, loss, scaler, device, optimizer, **_config['trainer']) if not test: utils.train_model(datasets=datasets, batch_size=_config['data']['batch-size'], folder=save_folder, trainer=trainer, scheduler=scheduler, epochs=config['epochs'], early_stop_steps=config['early_stop_steps']) utils.test_model(datasets=datasets, batch_size=_config['data']['batch-size'], trainer=trainer, folder=save_folder)
def train_model(self, fname=None): time.sleep(1) try: self.load(fname) print("File load successful.") except Exception: print("File load failed.") model_val = AE_A(training=False) image_dataset, dataset_size = utils.create_dataset(batch_size=batch_size) dataset_size = int((dataset_size - 1)/2) print("Dataset size is", dataset_size) print("Total number of images is", dataset_size * batch_size) record_batch = utils.record_steps(dataset_size) start_time = time.time() for i in range(num_epochs): print(f'Starting epoch {i}/{num_epochs}') start = time.time() batch_on = 0 for source in zip(image_dataset.take(dataset_size)): try: loss_identity, kl_loss = self.train_step(source, optimizer) if batch_on % record_batch == 0: print(f'Beginning batch #{batch_on} out of {dataset_size} of size {batch_size}') self.loss_identity += [loss_identity] self.kl_loss += [kl_loss] except Exception: print(f'Batch #{batch_on} failed. Continuing with next batch.') batch_on += 1 self.save(fname) time.sleep(0.5) duration = time.time() - start utils.print_time_remaining(i, num_epochs, duration) if i % display_mod == 0: model_val.load(model_name, compile=False) model_val.loss['Identity'] = self.loss_identity model_val.loss['KL'] = self.kl_loss utils.test_model(model_val, source, num=i, test=True, name=model_name, details='identity') for style in zip(image_dataset.take(1)): utils.test_model(model_val, source, style, num=i, test=True, name=model_name, details='merge') break #act = keract.get_activations(self, source) #keract.display_activations(act) print('\n') image_dataset, _ = utils.create_dataset(batch_size=batch_size) print('Training completed in', int((time.time() - start_time) / 60), "minutes &", int(duration % 60), "seconds")
def build_model(data_x, data_y, split_mode='test', feature_method='f_regression', feature_num=100, pkl_dir='pkl'): # select features # feature_num: integer that >=0 # method: ['f_regression', 'mutual_info_regression', 'pca'] data_x, min_max = ut.select_feature(data_x, data_y, method=feature_method, feature_num=feature_num) #if min_max is not None: # min_max.to_csv(os.path.join(pkl_dir, 'min_max.csv'), index=False, index_label=False, encoding='gb18030') # start building model np_x = np.nan_to_num(data_x.values) np_y = np.nan_to_num(data_y.values) print('train_set.shape=%s, test_set.shape=%s' % (np_x.shape, np_y.shape)) res = {} if split_mode == 'test': x_train, x_val, y_train, y_val = train_test_split(data_x, data_y, test_size=0.2, shuffle=True) model = LinearRegression() res['lr'] = ut.test_model(model, x_train, x_val, y_train, y_val) ut.save_model(model, data_x.columns, pkl_dir) model = DecisionTreeRegressor() res['dt'] = ut.test_model(model, x_train, x_val, y_train, y_val) ut.save_model(model, data_x.columns, pkl_dir, depth=5) model = RandomForestRegressor( ) #criterion='mse', n_estimators=500, min_samples_leaf=50) res['rf'] = ut.test_model(model, x_train, x_val, y_train, y_val) ut.save_model(model, data_x.columns, pkl_dir, depth=5) model = GradientBoostingRegressor() res['gbdt'] = ut.test_model(model, x_train, x_val, y_train, y_val) ut.save_model(model, data_x.columns, pkl_dir) elif split_mode == 'cv': model = LinearRegression() res['lr'] = ut.cv_model(model, np_x, np_y) model = DecisionTreeRegressor() res['dt'] = ut.cv_model(model, np_x, np_y) model = RandomForestRegressor() res['rf'] = ut.cv_model(model, np_x, np_y) model = GradientBoostingRegressor() res['gbdt'] = ut.cv_model(model, np_x, np_y) else: print('parameter mode=%s unresolved' % (model)) return res
def train_model(self, fname=None): time.sleep(1) try: self.load(fname) print("File load successful.") except Exception: print("File load failed.") image_dataset, dataset_size = utils.create_dataset( batch_size=batch_size) dataset_size = dataset_size - 1 print("Dataset size is", dataset_size) print("Total number of images is", dataset_size * batch_size) record_batch = utils.record_steps(int(dataset_size)) start_time = time.time() print("Beginning training at", start_time) for i in range(num_epochs): print("Starting epoch {}/{}".format(i, num_epochs)) start = time.time() batch_on = 0 for source in zip(image_dataset.take(int(dataset_size))): source = utils.get_random_crop(np.array(source), 32, 32) loss_identity = train_step(self, source, optimizer) if batch_on % record_batch == 0: print("Beginning batch #" + str(batch_on), 'out of', int(dataset_size), 'of size', batch_size) self.loss_identity += [loss_identity] batch_on += 1 if i % record_epochs == 0: self.loss['Identity'] = self.loss_identity utils.test_model(self, source, num=i, test=True, name=model_name, details='identity') '''for style in zip(image_dataset.take(1)): style = utils.get_random_crop(style, 32, 32) utils.test_model(self, source, style, num=i, test=True, name=model_name, details='transfer') break''' print('\n') duration = time.time() - start utils.print_time_remaining(i, num_epochs, duration) image_dataset, _ = utils.create_dataset(batch_size=batch_size) self.save(fname) time.sleep(.5) print('Training completed in', int((time.time() - start_time) / 60), "minutes &", int(duration % 60), "seconds")
def standard_process(train_loader, test_loader, labels, result_file='./standard.txt'): gpu = torch.device('cuda:0') new_model = model_retrieval().cuda(gpu) # print_model(new_model) models = [copy.deepcopy(new_model) for _ in range(args.num_task)] # time.sleep(10) temp_model = copy.deepcopy(new_model) cut_idx = generate_cut_layer(args.split, temp_model) optimizers = [optim.SGD(params=models[idx].parameters(), lr=args.lr) for idx in range(args.num_task)] for task in range(args.num_task): print('Training Task {}... Labels: {}'.format(task, labels[task])) model, optimizer = models[task] if task == 0 else models_copy(models[task], models[task-1], cut_idx), optimizers[task] # print_model(model) if task == 0: for param_group in optimizer.param_groups: param_group['lr'] = args.first_lr for iteration in range(args.iterations): loss = normal_train(model, labels[task], optimizer, train_loader[task], gpu) # print_model(model) print('Iteration: {}\tLoss:{}'.format(iteration, loss)) for sub_task in range(task + 1): temp_model = copy.deepcopy(models[sub_task]) temp_model = models_copy(temp_model, model, cut_idx) for i in range(task + 1): acc = test_model(temp_model, labels[i], test_loader[i], gpu) print('Device Task: {}\tTest Task: {}\tAccuracy: {}'.format(sub_task, i, acc)) with open(result_file, 'a') as f: # Current server parameter (task) + device parameter (task) --> training a given task f.write('{}\t{}\t{}\t{}\t{}\t{}\n'.format(task, iteration, loss, sub_task, i, acc))
def train_model(self, fname=None): time.sleep(5) try: self.load(fname) print("File load successful.") except Exception: print("File load failed.") image_dataset, dataset_size = utils.create_dataset() dataset_size = dataset_size - 1 print("Dataset size is", dataset_size) print("Total number of images is", dataset_size * batch_size) start_time = time.time() print("Beginning training at", start_time) for i in range(num_epochs): print("Starting epoch {}/{}".format(i, num_epochs)) start = time.time() batch_on = 0 for source, style in zip(image_dataset.take(int(dataset_size / 4)), image_dataset.take(int(dataset_size / 4))): try: loss_content, loss_style, loss_identity = self.train_step( source, style, optimizer) if batch_on % 10 == 0: print("Beginning batch #" + str(batch_on), 'out of', int(dataset_size / 4), 'of size', batch_size) self.loss_content += [loss_content] self.loss_style += [loss_style] self.loss_identity += [loss_identity] except Exception: print("Batch #", batch_on, "failed. Continuing with next batch.") batch_on += 1 duration = time.time() - start print(int(duration / 60), "minutes &", int(duration % 60), "seconds, for epoch", i) if i % 20 == 0: utils.test_model(self, source, style, num=i, name=model_name) print('\n') image_dataset, _ = utils.create_dataset() self.save(fname) time.sleep(1) print('Training completed in', int((time.time() - start_time) / 60), "minutes &", int(duration % 60), "seconds")
def our_process(train_loader, test_loader, labels, online=False, result_file='./our_process.txt'): gpu = torch.device('cuda:0') new_model = model_retrieval().cuda(gpu) models = [copy.deepcopy(new_model) for _ in range(args.num_task)] temp_model = copy.deepcopy(new_model) cut_idx = generate_cut_layer(args.split, temp_model) optimizers = [optim.SGD(params=models[idx].parameters(), lr=args.lr) for idx in range(args.num_task)] ewcs = [] if_freeze = 0 for task in range(args.num_task): print('Training Task {}... Labels: {}'.format(task, labels[task])) model, optimizer = models[task] if task == 0 else models_copy(models[task], models[task-1], cut_idx), optimizers[task] #model为tmp变量循环用,model:copy了上一次task中model的param(与cut_idx有关) if task == 0: for param_group in optimizer.param_groups: param_group['lr'] = args.first_lr for iteration in range(args.iterations): loss = normal_train(model, labels[task], optimizer, train_loader[task], gpu) print('Iteration: {}\tLoss:{}'.format(iteration, loss)) acc = test_model(model, labels[task], test_loader[task], gpu) print('Device Task: {}\tTest Task: {}\tAccuracy: {}'.format(task, task, acc)) with open(result_file, 'a') as f: f.write('{}\t{}\t{}\t{}\t{}\t{}\n'.format(task, iteration, loss, task, task, acc)) else: for iteration in range(args.iterations): if online: loss = our_train(model, labels[task], optimizer, train_loader[task], ewcs[-1:], args.lam, gpu, cut_idx, if_freeze) #与normal_train相比多了ewcs[]与args.lam else: loss = our_train(model, labels[task], optimizer, train_loader[task], ewcs, args.lam, gpu, cut_idx, if_freeze) #与online区别是ewcs[-1:]为ewcs只取最后一个元素构成的列表 #判断loss,loss若小于阈值,令变量if_freeze=1,传入下次our_train #our_train相比于ewc_train多两个参数:if_freeze和cut_idx if loss < args.threshold: if_freeze = 1 else: if_freeze = 0 print('Iteration: {}\tLoss:{}\tif freeze:{}'.format(iteration, loss, if_freeze)) for sub_task in range(task + 1): #循环不同model temp_model = copy.deepcopy(models[sub_task]) temp_model = models_copy(temp_model, model, cut_idx) #temp_model 用的是当前model后半部分,models[]前半部分 for i in range(task + 1): #循环不同task acc = test_model(temp_model, labels[i], test_loader[i], gpu) print('Device Task: {}\tTest Task: {}\tAccuracy: {}'.format(sub_task, i, acc)) with open(result_file, 'a') as f: # Current server parameter (task) + device parameter (task) --> training a given task f.write('{}\t{}\t{}\t{}\t{}\t{}\n'.format(task, iteration, loss, sub_task, i, acc)) ewcs.append(splitEWC(model, train_loader[task], cut_idx, gpu))
def cb(generator, progress, batch_index, result): iteration = (current_task - 1) * total_iterations + batch_index progress.set_description( ('<Training Generator> ' 'task: {task}/{tasks} | ' 'progress: [{trained}/{total}] ({percentage:.0f}%) | ' 'loss => ' 'g: {g_loss:.4} / ' 'w: {w_dist:.4}').format( task=current_task, tasks=total_tasks, trained=batch_size * batch_index, total=batch_size * total_iterations, percentage=(100. * batch_index / total_iterations), g_loss=result['g_loss'], w_dist=-result['c_loss'], )) # log the losses of the generator. if iteration % loss_log_interval == 0: visual.visualize_scalar(result['g_loss'], 'generator g loss', iteration, env=env) visual.visualize_scalar(-result['c_loss'], 'generator w distance', iteration, env=env) # log the generated images of the generator (to visdom). if iteration % sample_log_interval == 0: visual.visualize_images( generator.sample(sample_size).data, 'generated samples ({replay_mode})'.format( replay_mode=replay_mode), env=env, ) # log the sample images of the generator (to file) if iteration % sample_log_interval == 0 and sample_log: utils.test_model(generator, sample_size, os.path.join(sample_dir, env + '-sample-logs', str(iteration)), verbose=False)
def main(): for model in [ 'LinearRegression', 'LogisticRegression', 'RandomForest', 'DecisionTree', 'SVM' ]: display(f'Training model {model}') def func(): # training model return train_model(model, x_train, y_train) trained_model, training_time = timed_func(func) display(f'time used: {training_time}') display(f'Testing model') results, confusion_matrix, accuracy_score = test_model( trained_model, x_test, y_test, lambda predictions: [int(i) for i in predictions]) display(f'confusion_matrix: \n{confusion_matrix}') display(f'accuracy_score: {accuracy_score}\n\n')
def train(continue_from,train_loader,test_loader): print(continue_from) #continue_from = "best_transformer_age5.pth" modeltype = "age" model = MyModel(modeltype,use_col) model = model.cuda() #warm up optimizer optimizer = TransformerOptimizer( torch.optim.Adam(model.parameters(), betas=(0.9, 0.98), eps=1e-08), 0.8, 512, 8000) start_epoch = 0 print("start load") if os.path.exists(continue_from ): naive_model = torch.load(continue_from , map_location=lambda storage, loc:storage) new_state_dict = OrderedDict() for k, v in naive_model["state"].items(): new_state_dict[k] = v model.load_state_dict(new_state_dict) start_epoch = naive_model["epoch"]+1 optimizer.load_state_dict(naive_model["optimizer"]) optimizer.step_num = naive_model['optimizer_step'] logger.info("load model finish") print("end load") losses_age = [] losses_gender= [] best_acc_age = 0 best_acc_gender = 0 entroy=nn.CrossEntropyLoss()#mseloss = nn.MSELoss() model = nn.DataParallel(model) best_age_acc = 0 best_gender_acc = 0 #acc_age,acc_gender,loss_age,loss_gender = test_model(model,test_loader,entroy) for epo in range(start_epoch,30): model.train() #start = time.time() for i,(data) in enumerate(train_loader): raw_arr,click,times,label_age,label_gender,user_id,user_id_emb = data label_age = label_age.cuda() label_gender = label_gender.cuda() click = click.cuda() times= times.cuda() user_id_emb = user_id_emb.cuda() raw_arr= raw_arr.cuda() embs1,embs2 = model(user_id_emb,raw_arr,click,times) loss_age = entroy(embs1,label_age-1) loss_gender = entroy(embs2,label_gender-1) loss = loss_age+loss_gender optimizer.zero_grad() loss.backward() #torch.nn.utils.clip_grad_norm_(model.parameters(), 400) optimizer.step() losses_age.append(loss_age.item()) losses_gender.append(loss_gender.item()) if (i+1)%200==0: #print((time.time()-start)/200*len(train_loader)) for g in optimizer.optimizer.param_groups: logger.info("now learn rate : "+str(g['lr']) ) mean_loss_age = sum(losses_age)/len(losses_age) mean_loss_gender = sum(losses_gender)/len(losses_gender) logger.info('cpEpoch:{0} \t step:{1}/{2} \t ' 'mean loss:{3} \t{4}'.format( (epo + 1), (i + 1),len(train_loader),mean_loss_age,mean_loss_gender)) acc_age,acc_gender,loss_age,loss_gender = test_model(model,test_loader,entroy) logger.info("acc:{},{} loss:{}{}".format(acc_age,acc_gender,loss_age,loss_gender)) if acc_age>best_acc_age: best_acc_age = acc_age save_model(model,epo,optimizer,model_name = continue_from) if acc_gender>best_gender_acc: best_gender_acc = acc_gender save_model(model,epo,optimizer,model_name = continue_from + "gender.pth")
scholar.generator.set_critic_optimizer(generator_c_optimizer) # set additional settings for the scholar's generator scholar.generator.set_lambda(args.lamda) scholar.generator.set_critic_updates_per_batch(args.cu) # run the experiment. if args.train: train( scholar, train_datasets, test_datasets, replay_mode=args.replay_mode, generator_iterations=args.gen_iter if train_generator else 0, solver_iterations=args.sol_iter, importance_of_new_task=args.rnt, batch_size=args.batch, test_size=args.test_n, sample_size=args.s_size, loss_log_interval=args.loss_log, eval_log_interval=args.eval_log, sample_log_interval=args.s_log, sample_log=args.samples, sample_dir=args.s_dir, checkpoint_dir=args.c_dir, collate_fn=utils.label_squeezing_collate_fn, cuda=cuda, valid_proportion=valid_proportion ) else: path = os.path.join(args.s_dir, '{}-sample'.format(scholar.name)) utils.load_checkpoint(scholar, args.c_dir) utils.test_model(scholar.generator, args.s_size, path)
if frozen_predictor: for param in model.predictor.parameters(): param.requires_grad_(False) datasets = utils.get_datasets(datasets, 9, 1) scaler = utils.ZScoreScaler(datasets['train'].mean, datasets['train'].std) optimizer = optim.Adam([{ 'params': model.adaptor.parameters() }, { 'params': model.predictor.parameters(), 'lr': 1e-5 }], lr=learning_rate) loss = utils.get_loss('MaskedMAELoss') trainer = utils.OursTrainer(model, loss, scaler, device, optimizer, weight_decay, 2, 5) utils.train_model(datasets=datasets, batch_size=64, folder=saved_folder, trainer=trainer, scheduler=None, epochs=100, early_stop_steps=10) utils.test_model(datasets=datasets, batch_size=64, trainer=trainer, folder=saved_folder)
def ewc_process(train_loader, test_loader, labels, class_incremental, online=False, result_file='./ewc.txt'): gpu = torch.device('cuda:0') new_model = model_retrieval().cuda(gpu) models, cur_label = [ copy.deepcopy(new_model) for _ in range(args.num_task) ], [] temp_model = copy.deepcopy(new_model) cut_idx = generate_cut_layer(args.split, temp_model) optimizers = [ optim.SGD(params=models[idx].parameters(), lr=args.lr) for idx in range(args.num_task) ] ewcs = [] for task in range(args.num_task): print('Training Task {}... Labels: {}'.format(task, labels[task])) model, optimizer = models[task] if task == 0 else models_copy( models[task], models[task - 1], cut_idx ), optimizers[ task] #model为tmp变量循环用,model:copy了上一次task中model的param(与cut_idx有关) if task == 0: cur_label = cur_label + labels[ task] if class_incremental else labels[task] for param_group in optimizer.param_groups: param_group['lr'] = args.first_lr for iteration in range(args.iterations): loss = normal_train(model, cur_label, optimizer, train_loader[task], gpu) print('Iteration: {}\tLoss:{}'.format(iteration, loss)) acc = test_model(model, cur_label, test_loader[task], gpu) print('Device Task: {}\tTest Task: {}\tAccuracy: {}'.format( task, task, acc)) with open(result_file, 'a') as f: f.write('{}\t{}\t{}\t{}\t{}\t{}\n'.format( task, iteration, loss, task, task, acc)) if iteration % 20 == 0: for param_group in optimizer.param_groups: param_group['lr'] *= 0.95 else: cur_label = cur_label + labels[ task] if class_incremental else labels[task] for iteration in range(args.iterations): if online: loss = ewc_train(model, cur_label, optimizer, train_loader[task], ewcs[-1:], args.lam, gpu) #与normal_train相比多了ewcs[]与args.lam else: loss = ewc_train( model, cur_label, optimizer, train_loader[task], ewcs, args.lam, gpu) #与online区别是ewcs[-1:]为ewcs只取最后一个元素构成的列表 print('Iteration: {}\tLoss:{}'.format(iteration, loss)) for sub_task in range(task + 1): #循环不同model temp_model = copy.deepcopy(models[sub_task]) temp_model = models_copy( temp_model, model, cut_idx) #temp_model 用的是当前model后半部分,models[]前半部分 for i in range(task + 1): #循环不同task cur_label = cur_label if class_incremental else labels[ i] acc = test_model(temp_model, cur_label, test_loader[i], gpu) print('Device Task: {}\tTest Task: {}\tAccuracy: {}'. format(sub_task, i, acc)) with open(result_file, 'a') as f: # Current server parameter (task) + device parameter (task) --> training a given task f.write('{}\t{}\t{}\t{}\t{}\t{}\n'.format( task, iteration, loss, sub_task, i, acc)) if iteration % 20 == 0: for param_group in optimizer.param_groups: param_group['lr'] *= 0.95 ewcs.append(splitEWC(model, train_loader[task], cut_idx, gpu))
#rf_classifier = sk_learn_classifiers.train_random_forest(X_train, y_train) rf_classifier = sk_learn_classifiers.load_model('outputs/random_forest2') #xgboost_classifier = sk_learn_classifiers.train_xgboost(X_train, y_train) xgboost_classifier = sk_learn_classifiers.load_model('outputs/xgboost1') df_X_train_preds_classifiers = get_X_predictions_classifiers( deep_classifier, rf_classifier, xgboost_classifier, X_train) df_X_test_preds_classifiers = get_X_predictions_classifiers( deep_classifier, rf_classifier, xgboost_classifier, X_test) X_train_ensemble = df_X_train_preds_classifiers.to_numpy(copy=True) y_train = y_train.to_numpy(copy=True) X_test_ensemble = df_X_test_preds_classifiers.to_numpy(copy=True) y_test = y_test.to_numpy(copy=True) # train --------------------------------------------------- #ensemble_classifier = data_science.train_deep_classifier(X_train_ensemble, y_train, X_test_ensemble, y_test, lr=0.0001) #ensemble_classifier = data_science.train_deep_small_classifier(X_train_ensemble, y_train, X_test_ensemble, y_test, lr=0.0001) ensemble_classifier = deep_classifiers.load_model_classifier( 'outputs/2019-06-03_14:31:16_model.hdf5') y_pred_ensemble = deep_classifiers.predict(ensemble_classifier, X_test_ensemble) y_pred_deep = deep_classifiers.predict(deep_classifier, X_test) y_pred_rf = sk_learn_classifiers.predict(rf_classifier, X_test) y_pred_xgboost = sk_learn_classifiers.predict(xgboost_classifier, X_test) utils.test_model(y_test, y_pred_deep) utils.test_model(y_test, y_pred_rf) utils.test_model(y_test, y_pred_xgboost) utils.test_model(y_test, y_pred_ensemble)
import sklearn.model_selection as sk import tensorflow as tf from tensorflow.keras import Sequential from tensorflow.keras.layers import Conv2D, BatchNormalization, Dropout, Flatten, Dense, MaxPool2D import cv2 from utils import load_images, output_model, enable_cuda, test_model, load_model, get_char images, labels = load_images() model = load_model("model1") print(test_model(model, "hello_world.png", True)) x_train, x_test, y_train, y_test = sk.train_test_split(images, labels, test_size=.15) model = tf.keras.Sequential() model.add(Conv2D(32, kernel_size=(3, 3))) model.add(Conv2D(64, kernel_size=(4, 4))) model.add(BatchNormalization()) model.add(MaxPool2D(pool_size=(3, 3))) model.add(Dropout(0.1)) model.add(Conv2D(128, kernel_size=(5, 5), strides=(2, 2), padding="same")) model.add(BatchNormalization()) model.add(Flatten()) model.add(Dense(2048)) model.add(Dropout(0.1)) model.add(Dense(1024)) model.add(Dense(62, activation="softmax"))
from utils import append_timestamp, test_model if __name__ == '__main__': report_folder = "../reports/" model_folder = "../models/" models = { "AE_classifier_29-Mar-2019__17-49-01.hdf5": { "report": "AE" }, "classifier_DANN_final_07-Apr-2019__00-43-33.hdf5": { "report": "DANN" }, "SDAE_classifier_06-Apr-2019__15-51-40.hdf5": { "report": "SDAE" } } data = { "source": "../data/test_movies_vectors_balanced.csv", "target": "../data/test_electr_vectors_balanced.csv" } for model_name in models: model_path = model_folder + model_name model = load_model(model_path) for k in data: report = "{}_test_{}_V2.csv".format(k, models[model_name]["report"]) report_path = append_timestamp(report_folder + report) comment = "Testing {} on {} domain with new predict processing" comment = comment.format(model_name, k) test_model(model, data[k], report_path, 1000, comment=comment, norm=True)
('drop2',nn.Dropout(0.2)), ('h3', nn.Linear(512, 133)), ('output', nn.LogSoftmax(dim=1)) ])) model_transfer.fc = classifier if use_cuda: model_transfer = model_transfer.cuda() # Set optimization criterion criterion_transfer = nn.NLLLoss() optimizer_transfer = optim.RMSprop(model_transfer.fc.parameters(), lr=0.001) loaders_transfer = {'train': trainLoad,'val': valLoad,'test': testLoad} model_transfer = train_model(35, loaders_transfer, model_transfer, optimizer_transfer, criterion_transfer, use_cuda, 'model_transfer_best.pt') # Test the model using the testing set model_transfer.load_state_dict(torch.load('model_transfer_best.pt')) test_model(loaders_transfer, model_transfer, criterion_transfer, use_cuda) # Save the classes class_names = [item[4:].replace("_", " ") for item in train.classes] class_numbers = [item_element for item_element,item in enumerate(train.classes)] # Save the class names and indices with open('Dog_names.pkl', 'wb') as handle: pkl.dump(class_names, handle) with open('Dog_indices.pkl', 'wb') as handle: pkl.dump(class_numbers, handle)
def main(args): # Generate data generate_data = get_data_generator( N=args.n_samples, mode=None) # Not applying over or under sampling technique train_loader, test_loader = generate_data() # Choose device to use device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") print(f"Using {device}") # Define criterion to optimize if args.model_version == 4: criterion = digit_prediction_criterion else: criterion = get_auxiliary_loss_model_criterion() # Grid Search if args.grid_search: print("Grid Search of learning rate and regularization term") model_class = NetSiamese model_params = { "input_channels": INPUT_CHANNELS, "output_class_channels": OUTPUT_CLASS_CHANNELS, "output_digit_channels": OUTPUT_DIGIT_CHANNELS, "activation": "leakyrelu", "auxiliary_loss": True, "version": args.model_version } lr, reg = grid_search([0.05, 0.01, 0.005, 0.001, 0.0005, 0.0001], [0.25, 0.1, 0.05, 0.01], train, train_loader, test_loader, device, model_class, model_params, criterion, print_info=True) else: lr, reg = args.lr, args.reg # Train print(f"Training Siamese model version {args.model_version}" + (f" ({args.n_rounds} rounds)" if args.n_rounds > 1 else "")) if args.n_rounds > 1: model_class = NetSiamese model_params = { "input_channels": INPUT_CHANNELS, "output_class_channels": OUTPUT_CLASS_CHANNELS, "output_digit_channels": OUTPUT_DIGIT_CHANNELS, "activation": "leakyrelu", "auxiliary_loss": True, "version": args.model_version } accuracy_values, loss_values = test_model(train, generate_data, device, model_class, model_params, criterion, lr, reg, nb_tests=args.n_rounds, epochs=args.n_epoch) if args.plot_curves: plot_test_results( accuracy_values, loss_values, title=f"Model's assessment over {args.n_rounds} rounds") print( f"Mean accuracy is {np.mean(np.max(accuracy_values, axis=1)):0.3f}, " f"Standard deviation of accuracy is {np.std(np.max(accuracy_values, axis=1)):0.3f}" ) else: net_auxiliary_loss = NetSiamese(INPUT_CHANNELS, OUTPUT_CLASS_CHANNELS, OUTPUT_DIGIT_CHANNELS, activation="leakyrelu", auxiliary_loss=True, version=args.model_version) optimizer = optim.Adam(net_auxiliary_loss.parameters(), lr=lr, weight_decay=reg) # Train model _, accuracies, losses = train(train_loader, test_loader, net_auxiliary_loss, optimizer, criterion, device=device, epochs=args.n_epoch, print_info=True) accuracy_train_class, accuracy_test_class, accuracy_train_digit, accuracy_test_digit = accuracies if args.plot_curves: plot_accuracy_and_loss(accuracy_train_class, accuracy_test_class, losses, title="Class prediction accuracy")
def _train(model, optim_inf, scheduler_inf, checkpointer, epochs, train_loader, test_loader, stat_tracker, log_dir, device, args): ''' Training loop for optimizing encoder ''' # If mixed precision is on, will add the necessary hooks into the model # and optimizer for half() conversions model, optim_inf = mixed_precision.initialize(model, optim_inf) optim_raw = mixed_precision.get_optimizer(optim_inf) # get target LR for LR warmup -- assume same LR for all param groups for pg in optim_raw.param_groups: lr_real = pg['lr'] # IDK, maybe this helps? # but it makes the training slow # torch.cuda.empty_cache() # prepare checkpoint and stats accumulator next_epoch, total_updates = checkpointer.get_current_position() fast_stats = AverageMeterSet() # run main training loop for epoch in range(next_epoch, epochs): epoch_stats = AverageMeterSet() epoch_updates = 0 time_start = time.time() for _, ((images1, images2), labels) in enumerate(train_loader): # get data and info about this minibatch labels = torch.cat([labels, labels]).to(device) images1 = images1.to(device) images2 = images2.to(device) # run forward pass through model to get global and local features res_dict = model(args, x1=images1, x2=images2, class_only=False) lgt_glb_mlp, lgt_glb_lin = res_dict['class'] # compute costs for all self-supervised tasks loss_g2l = (res_dict['g2l_1t5'] + res_dict['g2l_1t7'] + res_dict['g2l_5t5']) loss_inf = loss_g2l + res_dict['lgt_reg'] # compute loss for online evaluation classifiers loss_cls = (loss_xent(lgt_glb_mlp, labels) + loss_xent(lgt_glb_lin, labels)) # do hacky learning rate warmup -- we stop when LR hits lr_real if (total_updates < 500): lr_scale = min(1., float(total_updates + 1) / 500.) for pg in optim_raw.param_groups: pg['lr'] = lr_scale * lr_real # reset gradient accumlators and do backprop loss_opt = loss_inf + loss_cls optim_inf.zero_grad() mixed_precision.backward( loss_opt, optim_inf) # backwards with fp32/fp16 awareness if args.grad_clip_value and (total_updates >= 500): torch.nn.utils.clip_grad_value_(model.parameters(), args.grad_clip_value) optim_inf.step() # record loss and accuracy on minibatch epoch_stats.update_dict( { 'loss_inf': loss_inf.detach().item(), 'loss_cls': loss_cls.detach().item(), 'loss_g2l': loss_g2l.detach().item(), 'lgt_reg': res_dict['lgt_reg'].detach().item(), 'loss_g2l_1t5': res_dict['g2l_1t5'].detach().item(), 'loss_g2l_1t7': res_dict['g2l_1t7'].detach().item(), 'loss_g2l_5t5': res_dict['g2l_5t5'].detach().item() }, n=1) update_train_accuracies(epoch_stats, labels, lgt_glb_mlp, lgt_glb_lin) # shortcut diagnostics to deal with long epochs total_updates += 1 epoch_updates += 1 # this command makes the training slow # torch.cuda.empty_cache() if (total_updates % 100) == 0: # IDK, maybe this helps? time_stop = time.time() spu = (time_stop - time_start) / 100. print( 'Epoch {0:d}, {1:d} updates -- {2:.4f} sec/update'.format( epoch, epoch_updates, spu)) time_start = time.time() if (total_updates % 500) == 0: # record diagnostics eval_start = time.time() fast_stats = AverageMeterSet() test_model(args, model, test_loader, device, fast_stats, max_evals=100000) stat_tracker.record_stats( fast_stats.averages(total_updates, prefix='fast/')) eval_time = time.time() - eval_start stat_str = fast_stats.pretty_string(ignore=model.tasks) stat_str = '-- {0:d} updates, eval_time {1:.2f}: {2:s}'.format( total_updates, eval_time, stat_str) print(stat_str) # update learning rate scheduler_inf.step(epoch) test_model(args, model, test_loader, device, epoch_stats, max_evals=500000) epoch_str = epoch_stats.pretty_string(ignore=model.tasks) diag_str = '{0:d}: {1:s}'.format(epoch, epoch_str) print(diag_str) sys.stdout.flush() stat_tracker.record_stats(epoch_stats.averages(epoch, prefix='costs/')) checkpointer.update(epoch + 1, total_updates)
# X = torch.sqrt(X + 1e-5) X = torch.nn.functional.normalize(X) X = self.fc(X) assert X.size() == (N, 11) return X data_transform = transforms.Compose([ transforms.Resize(448), # transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])]) # data_dir = '/home/mcc/data/xuelang' data_dir = '../data/512' image_dataset = datasets.ImageFolder(os.path.join(data_dir, 'test_r2'), data_transform) dataloader = torch.utils.data.DataLoader(image_dataset, batch_size=32, shuffle=False, num_workers=12) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") model_ft = BCNN() model_ft.load_state_dict(torch.load('../output/bcnn_step2_20180830_0953.pth')) model_ft = model_ft.to(device) all_prob = test_model(model_ft, dataloader, device) img_names = sorted(os.listdir(os.path.join(data_dir, 'test_r2', 'norm'))) # submit_csv('bcnn', img_names, all_prob) with open('../output/prob_bcnn.pkl', 'wb') as f: pickle.dump(all_prob, f)
W_new_new = new_W.reshape(Wshape) new_layer = Convolution1D(output_dim, f_dim[1], strides=1, activation=layer.activation, padding='valid', weights=[new_W, b], name='converted_conv') flattened_ipt = False else: shape = (1, 1, input_shape[1], output_dim) new_W = W.reshape(shape) new_layer = Convolution1D(output_dim, (1, 1), strides=(1, 1), activation=layer.activation, padding='valid', weights=[new_W, b]) else: new_layer = layer new_model.add(new_layer) new_model.add(Lambda(lambda x: K.batch_flatten(x))) X, Y = utils.load_data('Dataframes/Testing24.pickle') score = utils.test_model(new_model, X, Y) print("%s: %.2f%%" % (new_model.metrics_names[1], score))
memlog_fname=mem_logfile_AE, epochs=epochs, ae=True) model.layers.pop() model.save("../models/AE_layers_popped_{}.hdf5".format(timestamp)) hidden_size1 = 64 hidden_size2 = 128 model.add(LSTM(hidden_size1, return_sequences=True, input_shape=(None, 128))) # model.add(LSTM(hidden_size2, return_sequences=True)) # model.add(Dense(hidden_size2, activation="hard_sigmoid")) model.add(Dense(1, activation='hard_sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam') steps_per_epoch = int(movies_lines / batch_size) epochs = 20 log_fname = '{}/training_AE_classifier.csv'.format(report_folder) print("Training classifier") train_model(model, train_path["movies"], batch_size=batch_size, steps_per_epoch=steps_per_epoch, log_fname=log_fname, memlog_fname=mem_logfile_classifier, epochs=epochs) print("Testing model") report_path = "{}/report_LSTM_AE_source.csv".format(report_folder) test_model(model, test_path["movies"], report_path, batch_size) report_path = "{}/report_LSTM_AE_target.csv".format(report_folder) test_model(model, test_path["electr"], report_path, batch_size) model.save("../models/LSTM_AE_{}_{}.hdf5".format(hidden_size1, timestamp)) print("Done!")
# Command Line ardguments ap.add_argument('--engine', default="cuda") ap.add_argument('--hidden_layers', type=int, nargs='+', default=[4096,2048, 512], help='Number of units per hidden layer') ap.add_argument('--output_size', type=int, default=102, help='Number of possible classifications') ap.add_argument('--lr', type=float, action="store", default=0.001) ap.add_argument('--dropout', type=float, action = "store", default = 0.25) ap.add_argument('--epochs', action="store", type=int, default = 1) ap.add_argument('--arch', action="store", default="vgg19", type = str) ap.add_argument('--print_every', action='store', default=10 ) ap.add_argument('--input_dir', nargs='*', action="store", default="flowers") ap.add_argument('--save_dir', dest="save_dir", action="store", default="./checkpoint.pth") args = ap.parse_args() train_dataloader, val_dataloader, test_dataloader, class_to_idx_train = utils.load_data(args.input_dir) model, criterion, optimizer = utils.build_model(args.arch,args.hidden_layers,args.output_size,args.dropout,args.lr, args.engine) utils.train_model(model, train_dataloader, val_dataloader, criterion, optimizer, args.epochs, args.print_every, args.engine) utils.test_model(model, test_dataloader) utils.save_model(model, args.epochs, optimizer, class_to_idx_train) print("All Done. The Model is trained")
def GAN(): # Graph Part # print("Graph initialization...") with tf.device(FLAGS.device): with tf.variable_scope("model", reuse=None): m_train = G.BEGAN(batch_size=FLAGS.tr_batch_size, is_training=True, num_keys=FLAGS.num_keys, input_length=FLAGS.hidden_state_size, output_length=FLAGS.predict_size, learning_rate=learning_rate) with tf.variable_scope("model", reuse=True): m_valid = G.BEGAN(batch_size=FLAGS.val_batch_size, is_training=False, num_keys=FLAGS.num_keys, input_length=FLAGS.hidden_state_size, output_length=FLAGS.predict_size, learning_rate=learning_rate) with tf.variable_scope("model", reuse=True): m_test = G.BEGAN(batch_size=FLAGS.test_batch_size, is_training=False, num_keys=FLAGS.num_keys, input_length=FLAGS.hidden_state_size, output_length=FLAGS.predict_size, learning_rate=learning_rate) print("Done") # Summary Part # print("Setting up summary op...") g_loss_ph = tf.placeholder(dtype=tf.float32) d_loss_ph = tf.placeholder(dtype=tf.float32) loss_summary_op_d = tf.summary.scalar("discriminatr_loss", d_loss_ph) loss_summary_op_g = tf.summary.scalar("generator_loss", g_loss_ph) valid_summary_writer = tf.summary.FileWriter(logs_dir + '/valid/', max_queue=2) train_summary_writer = tf.summary.FileWriter(logs_dir + '/train/', max_queue=2) print("Done") # Model Save Part # print("Setting up Saver...") saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(logs_dir) print("Done") # Session Part # print("Setting up Data Reader...") validation_dataset_reader = mt.Dataset( directory=test_dir, batch_size=FLAGS.val_batch_size, is_batch_zero_pad=FLAGS.is_batch_zero_pad, hidden_state_size=FLAGS.hidden_state_size, predict_size=FLAGS.predict_size, num_keys=FLAGS.num_keys, tick_interval=tick_interval, step=FLAGS.slice_step) test_dataset_reader = mt.Dataset(directory=test_dir, batch_size=FLAGS.test_batch_size, is_batch_zero_pad=FLAGS.is_batch_zero_pad, hidden_state_size=FLAGS.hidden_state_size, predict_size=FLAGS.predict_size, num_keys=FLAGS.num_keys, tick_interval=tick_interval, step=FLAGS.slice_step) print("done") sess_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) sess_config.gpu_options.allow_growth = True sess = tf.Session(config=sess_config) if ckpt and ckpt.model_checkpoint_path: # model restore saver.restore(sess, ckpt.model_checkpoint_path) print("Model restored...") else: sess.run(tf.global_variables_initializer() ) # if the checkpoint doesn't exist, do initialization if FLAGS.mode == "train": train_dataset_reader = mt.Dataset( directory=train_dir, batch_size=FLAGS.tr_batch_size, is_batch_zero_pad=FLAGS.is_batch_zero_pad, hidden_state_size=FLAGS.hidden_state_size, predict_size=FLAGS.predict_size, num_keys=FLAGS.num_keys, tick_interval=tick_interval, step=FLAGS.slice_step) for itr in range(MAX_EPOCH): feed_dict = utils.run_epoch(train_dataset_reader, FLAGS.tr_batch_size, m_train, sess) if itr % 100 == 0: if FLAGS.use_began_loss: train_loss_d, train_loss_g, train_pred = sess.run( [m_train.loss_d, m_train.loss_g, m_train.predict], feed_dict=feed_dict) train_summary_str_d, train_summary_str_g = sess.run( [loss_summary_op_d, loss_summary_op_g], feed_dict={ g_loss_ph: train_loss_g, d_loss_ph: train_loss_d }) train_summary_writer.add_summary(train_summary_str_g, itr) print("Step : %d TRAINING LOSS *****************" % (itr)) print("Dicriminator_loss: %g\nGenerator_loss: %g" % (train_loss_d, train_loss_g)) if itr % 1000 == 0: if FLAGS.use_began_loss: valid_loss_d, valid_loss_g, valid_pred = utils.validation( validation_dataset_reader, FLAGS.val_batch_size, m_valid, FLAGS.hidden_state_size, FLAGS.predict_size, sess, logs_dir, itr, tick_interval) valid_summary_str_d, valid_summary_str_g = sess.run( [loss_summary_op_d, loss_summary_op_g], feed_dict={ g_loss_ph: valid_loss_g, d_loss_ph: valid_loss_d }) valid_summary_writer.add_summary(valid_summary_str_d, itr) print("Step : %d VALIDATION LOSS ***************" % (itr)) print("Dicriminator_loss: %g\nGenerator_loss: %g" % (valid_loss_d, valid_loss_g)) if itr % 1000 == 0 and itr != 0: utils.test_model(test_dataset_reader, FLAGS.test_batch_size, m_test, FLAGS.predict_size, sess, logs_dir, itr, tick_interval, 5) if itr % 1000 == 0: saver.save(sess, logs_dir + "/model.ckpt", itr) if FLAGS.mode == "test": utils.test_model(test_dataset_reader, FLAGS.test_batch_size, m_test, FLAGS.predict_size, sess, logs_dir, 9999, tick_interval, 10)
validation_split=0.3, shuffle=True, callbacks=[reduce_lr, model_checkpoint, early_stopping]) # print('predict test data') # # imgs_mask_test = model.predict({'inputs_modality1': imgs_test, 'inputs_modality2': imgs_test, 'inputs_modality3': imgs_test}, # batch_size=1, verbose=1) # # np.save(save_path+'/imgs_mask_test.npy', imgs_mask_test) def save_img(self): print("array to image") imgs = np.load(save_path + '/imgs_mask_test.npy') for i in range(imgs.shape[0]): #img = imgs[i] img = np.squeeze(imgs[i]) * 255 img = np.clip(img, 0, 255).astype('uint8') save_images(os.path.join(save_path, '%d.jpg' % (i)), img) #img = array_to_img(img) #img.save("./results/%d.jpg"%(i)) if __name__ == '__main__': myunet = myUnet() myunet.train() # myunet.save_img() print('predict test model') test_model(myunet)
self.save(fname) print('Training completed in', int((time.time() - start_time) / 60), "minutes &", int(duration % 60), "seconds") #@tf.function def train_step(self, source, optimizer): with tf.GradientTape() as tape: w, w_mean, w_log_var = self.encode(source) prediction = self(source[0]) loss_identity = identity_lr * tf.reduce_mean(tf.reduce_sum(cross_entropy(source[0], prediction))) #+ 0.1 * tf.reduce_mean((source[0]-prediction)**2)) kl_loss = -0.5 * (1 + w_log_var - tf.square(w_mean) - tf.exp(w_log_var)) kl_loss = kl_lr * tf.reduce_mean(tf.reduce_sum(kl_loss, axis=1)) loss = (loss_identity + kl_loss) * learning_rate grads = tape.gradient(loss, model.trainable_variables) optimizer.apply_gradients(zip(grads, model.trainable_variables)) return loss_identity, kl_loss def build_graph(self): source = Input(shape=(256, 256, 3)) return tf.keras.Model(inputs=source, outputs=self.call(source), name=model_name) model = AE_A() tf.keras.utils.plot_model(model.build_graph(), model_name+".png", show_shapes=True, expand_nested=True) model.train_model(model_name) model.load(model_name) image_dataset, _ = utils.create_dataset(batch_size=batch_size) for source, style in zip(image_dataset.take(1), image_dataset.take(1)): utils.test_model(model, source, style, test=True, name=model_name) break
def main(): """Run training.""" config = yaml.safe_load(open("config.yml")) #training hyperparameters num_epochs = config['num_epochs'] learning_rate = config['learning_rate'] batch_size = config['batch_size'] valid_period = config['valid_period'] #data hyperparameters #extract sample (input sequences with length = stride_len) #for RNN from the givent full trajectory window_len = config['window_len'] stride_len = config['stride_len'] n_times = config['n_times'] x_train, y_train, x_dev, y_dev, x_test, y_test = read_data( window_len, stride_len, n_times) print('Dataset:', x_train.shape, y_train.shape, x_dev.shape, y_dev.shape, x_test.shape, y_test.shape) #create dataset transformed_control_dataset_train = ControlDataset( x_train, y_train, transform=transforms.Compose([ToTensor()])) transformed_control_dataset_dev = ControlDataset( x_dev, y_dev, transform=transforms.Compose([ToTensor()])) #transformed_control_dataset_test = ControlDataset(x_test, y_test, #transform=transforms.Compose([ToTensor()])) # create batch data_train = DataLoader(transformed_control_dataset_train, batch_size=batch_size, shuffle=True, num_workers=1, drop_last=True) data_dev = DataLoader(transformed_control_dataset_dev, batch_size=batch_size, shuffle=False, num_workers=1, drop_last=True) #data_test = DataLoader(transformed_control_dataset_test, batch_size=1, #shuffle=False, num_workers=1, drop_last=True) # Device device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") print('Device:', device) # save model save_model_name = config['model_name'] print('Save Model name: ', save_model_name) model_save_path = '../checkpoints/' + save_model_name #+ '.pt' ## not pth #test/train mode = int(sys.argv[1]) if mode == 0: #train mode # Model preparation model = SeRNN_FWXX(batch_size, device) print('Model: ', model) model.to(device) pytorch_total_params = sum(p.numel() for p in model.parameters()) print('# of params: ', pytorch_total_params) # multiple GPUs if torch.cuda.device_count() > 1: print("Model uploaded. Number of GPUs: ", torch.cuda.device_count()) #model = nn.DataParallel(model) #set the training loss and optimizer criterion = nn.MSELoss() optimizer = optim.RMSprop(model.parameters(), lr=learning_rate) train_model(model, device, data_train, data_dev, x_dev, y_dev, optimizer, criterion, num_epochs, model_save_path, window_len, stride_len, valid_period) else: # test mode # upload saved model print('Saved Model evaluation with test set') model = torch.jit.load(model_save_path) test_model(model, device, x_test, y_test, 'True', window_len, stride_len)
labels[seq["seqid"]] = label for label in seqs.keys(): random.shuffle(seqs[label]) errors = [] sizes = [] ratios = [.01, .05, .1, .25, .5, .75, .9]; for ratio in ratios: training = [] testing = [] for label in seqs.keys(): #random.shuffle(seqs[label]) #So our training sets are not proper subsets of eachother k = int(ratio * len(seqs[label])) training.extend(seqs[label][:k]) testing.extend(seqs[label][k:]) print >>sys.stderr, "Ratio %s, training: %s, testing: %s" % (ratio, len(training), len(testing)) sizes.append(len(training)) models = utils.build_models(training, labels) error, log_odds = utils.test_model(testing, labels, models) errors.append(error) #print errors print sizes print errors