def main(): parser = argparse.ArgumentParser() parser.add_argument("-batch_size", type=int, default=30) parser.add_argument("-N", type=int, default=3) parser.add_argument("-embedding_dim", type=int, default=512) parser.add_argument("-lr", type=float, default=1e-3) parser.add_argument("-weight_decay", type=float, default=1e-5) parser.add_argument("-iterator", type=int, default=30) args = parser.parse_args() train_batch, val, test, vocab = dataloader(batch_size=args.batch_size, memory_size=5, task=1, joint=False, tenK=False) model = MemoryNetwork(vocab_size=len(vocab), d=args.embedding_dim, N=args.N) criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) train(model=model, iterator=args.iterator, optimizer=optimizer, criterion=criterion, train_loader=train_batch, test_loader=None)
def fit(self): # Load data self.dl = dataloader(dataset=self.dataset, norm_method=self.norm_method, val_fold=self.val_fold, crop=self.crop, inv_thresh=self.inv_thresh, custom_data=self.custom_data, verbose=2) # Init network self.dtc = DTC(n_clusters=self.n_clusters, input_dim=1, timesteps=self.dl.data_train.shape[1], n_filters=self.n_filters, kernel_size=self.kernel_size, strides=self.strides, pool_size=self.pool_size, n_units=self.n_units, alpha=self.alpha, dist_metric=self.dist_metric, cluster_init=self.cluster_init, heatmap=self.heatmap) # Initialize and compile model self.dtc.initialize() self.dtc.model.summary() self.dtc.compile(gamma=self.gamma, optimizer=self.optimizer, initial_heatmap_loss_weight=self.initial_heatmap_loss_weight, final_heatmap_loss_weight=self.final_heatmap_loss_weight) # Pretrain if (self.pretrain_epochs > 0): self.dtc.pretrain(X=self.dl.data_train, optimizer=self.optimizer, epochs=self.pretrain_epochs, batch_size=self.batch_size, save_dir=self.save_dir) # Initialize clusters self.dtc.init_cluster_weights(self.dl.data_train) # Fit model self.dtc.fit(X_train=self.dl.data_train, y_train=self.dl.label_train, X_val=self.dl.data_val, y_val=self.dl.label_val, epochs=self.epochs, eval_epochs=self.eval_epochs, save_epochs=self.save_epochs, batch_size=self.batch_size, tol=0.001, patience=5, finetune_heatmap_at_epoch=self.finetune_heatmap_at_epoch, save_dir=self.save_dir) self.model = self.dtc.model
def train(self, task_id, trainData, trainLabels, trainLabelsVectors): self.discriminator = self.get_discriminator(task_id) self.model_optimizer = self.get_model_optimizer(task_id) self.d_optimizer = self.get_dis_optimizer() train_data, train_label, train_attr, no_steps = utils.dataloader(trainData, trainLabels, trainLabelsVectors, self.args.train_batch_size) for e in range(self.args.num_epochs): loss = self.train_epoch(task_id, train_data, train_label, train_attr, no_steps) print('epoch', e + 1, 'Total loss:', loss)
def __init__(self, args): self.initial_lr = args.learning_rate self.lr = args.learning_rate self.test_only = args.test_only self.dump_statistics = args.dump_statistics self.modelName = args.model self.experiment = args.experiment self.log_path = args.log_path self.save_path = args.save_path if not os.path.isdir(self.log_path): os.makedirs(self.log_path) self.logger = Logger( '%s/%s_%s.csv' % (self.log_path, self.modelName, args.experiment), 'epoch, time, learning_rate, tr_loss, tr_acc, val_loss, val_acc') self.progress_bar = ProgressBar() self.chrono = Chrono() self.trainset, self.testset, self.trainloader, self.testloader = dataloader( ) print('==> Building model..') self.ae = AutoEncoder() self.model = getattr(models, self.modelName)() if self.modelName == 'bit': self.model.load_from( numpy.load('./state_dicts/%s.npz' % self.modelName)) if torch.cuda.is_available(): self.ae = torch.nn.DataParallel(self.ae) self.model = torch.nn.DataParallel(self.model) torch.backends.cudnn.benchmark = True self.optimizer = torch.optim.SGD(self.model.parameters(), lr=self.lr, momentum=0.9) self.load_ae() if args.resume or self.test_only or self.dump_statistics: self.load() self.criterion = torch.nn.CrossEntropyLoss() self.criterion = get_torch_vars(self.criterion, False) self.ae = get_torch_vars(self.ae, False) self.model = get_torch_vars(self.model, False)
def train(self): opt = tf.keras.optimizers.Adam(amsgrad=True) model = self.model #apply binary cross entropy for the loss of confidence scores and custom smooth l1 loss for regressing the anchor coordinates. model.compile(loss=['binary_crossentropy', smooth_l1_loss], optimizer=opt) X, Y = dataloader() for epoch in range(self.nb_epoch): res = model.fit(x=X, y=Y, batch_size=self.batch_size, initial_epoch=epoch, epochs=epoch + 1) if epoch % 100 == 0: model.save_weights( os.path.join(config.checkpoint_path, str(epoch)))
def __init__(self, args): # Parametres self.epoch = args.epoch self.sample_num = 100 self.batch_size = args.batch_size self.input_size = args.input_size self.z_dim = 62 self.result_dir = 'results' self.save_dir = 'models' self.model_name = 'WGAN' self.dataset = args.dataset self.c = 0.01 # clipping value self.n_critic = 5 # the number of iterations of the critic per generator iteration # load dataset self.data_loader = utils.dataloader(self.input_size, self.batch_size, self.dataset) data = self.data_loader.__iter__().__next__()[0] # networks init self.G = generator(input_dim=self.z_dim, output_dim=data.shape[1], input_size=self.input_size) self.D = discriminator(input_dim=data.shape[1], output_dim=1, input_size=self.input_size) self.G_optimizer = optim.Adam(self.G.parameters(), lr=args.lrG, betas=(args.beta1, args.beta2)) self.D_optimizer = optim.Adam(self.D.parameters(), lr=args.lrD, betas=(args.beta1, args.beta2)) self.G.cpu() self.D.cpu() print('---------- Networks architecture -------------') utils.print_network(self.G) utils.print_network(self.D) print('-----------------------------------------------') # fixed noise self.sample_z_ = torch.rand((self.batch_size, self.z_dim)) self.sample_z_ = self.sample_z_.cpu()
def fit(self): # Prepare data self.dl = dataloader(dataset=self.dataset, norm_method=self.norm_method, val_fold=self.val_fold, crop=self.crop, inv_thresh=self.inv_thresh, custom_data=self.custom_data, format='Squashed', verbose=2) # Init network self.input_dim = 1 self.timesteps = int(self.dl.data_train.shape[1]) self.build_ae() # Prepare callbacks early_stop = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=5, min_delta=0.000) filepath = "model-ae-epoch-{epoch:02d}-{val_loss:.2f}.hdf5" checkpoint = ModelCheckpoint(filepath, monitor='val_loss', mode='min', verbose=1, save_best_only=False, period=10) # Fit model self.history = self.autoencoder.fit( x=self.dl.data_train, y=self.dl.data_train, validation_data=[self.dl.data_val, self.dl.data_val], nb_epoch=self.epochs, batch_size=self.batch_size, verbose=1, callbacks=[early_stop, checkpoint]).history
def fit(self): # Prepare data self.dl = dataloader(dataset=self.dataset, norm_method=self.norm_method, val_fold=self.val_fold, crop=self.crop, inv_thresh=self.inv_thresh, custom_data=self.custom_data, verbose=2) # Init network self.autoencoder, self.encoder, self.decoder = temporal_autoencoder_v3(input_dim=1, timesteps=self.dl.data_train.shape[1], n_filters=self.n_filters, kernel_size=self.kernel_size, strides=self.strides, pool_size=self.pool_size, n_units=self.n_units, latent_dim=self.latent_dim) # DTC model (autoencoder only) self.model = Model(inputs=self.autoencoder.input, outputs=self.autoencoder.output) # Compile model self.model.compile(loss=self.loss, optimizer=self.optimizer) # Prepare callbacks early_stop = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=5, min_delta=0.000) filepath = "model-ae-epoch-{epoch:02d}-{val_loss:.2f}.hdf5" checkpoint = ModelCheckpoint(filepath, monitor='val_loss', mode='min', verbose=1, save_best_only=False, period=10) # Fit model self.history = self.model.fit(x=self.dl.data_train, y=self.dl.data_train, validation_data=[self.dl.data_val, self.dl.data_val], nb_epoch=self.epochs, batch_size=self.batch_size, verbose=2, callbacks=[early_stop, checkpoint]).history
def __init__(self, args): # Parametres self.epoch = args.epoch self.sample_num = 100 self.batch_size = args.batch_size self.input_size = args.input_size self.z_dim = 62 self.result_dir = 'results' self.save_dir = 'models' self.dataset = args.dataset self.model_name = 'GAN' # Chargement des données self.data_loader = utils.dataloader(self.input_size, self.batch_size, self.dataset) data = self.data_loader.__iter__().__next__()[0] # Initialisation du générateur et du discriminant self.G = generator(input_dim=self.z_dim, output_dim=data.shape[1], input_size=self.input_size) self.D = discriminator(input_dim=data.shape[1], output_dim=1, input_size=self.input_size) # Optimisation avec Adam self.G_optimizer = optim.Adam(self.G.parameters(), lr=args.lrG, betas=(args.beta1, args.beta2)) self.D_optimizer = optim.Adam(self.D.parameters(), lr=args.lrD, betas=(args.beta1, args.beta2)) self.G.cpu() self.D.cpu() self.BCE_loss = nn.BCELoss().cpu() print('---------- Networks architecture -------------') utils.print_network(self.G) utils.print_network(self.D) print('-----------------------------------------------') # Bruit self.sample_z_ = torch.rand((self.batch_size, self.z_dim)) self.sample_z_ = self.sample_z_.cpu()
def main(args): model = AutoEncoder() use_gpu = torch.cuda.is_available() if use_gpu: print('cuda is available!') model.cuda() weight_file = args.path_weight_file model = AutoEncoder() model.load_state_dict( torch.load(weight_file, map_location=lambda storage, loc: storage)) test_dataset = dataloader('dogs_cats', 'test') test_loader = torch.utils.data.DataLoader( test_dataset, batch_size=args.batch_size, shuffle=False) images, _ = iter(test_loader).next() images = Variable(images, volatile=True) imshow(torchvision.utils.make_grid(images.data[:25], nrow=5)) outputs = model(images) imshow(torchvision.utils.make_grid(outputs.data[:25], nrow=5))
def __init__(self, args): self.initial_lr = args.learning_rate self.lr = args.learning_rate self.test_only = args.test_only self.dump_statistics = args.dump_statistics self.modelName = "malconv" self.experiment = args.experiment self.log_path = args.log_path self.save_path = args.save_path if not os.path.isdir(self.log_path): os.makedirs(self.log_path) self.logger = Logger('%s/%s_%s.csv' % (self.log_path, self.modelName, args.experiment), 'epoch, time, learning_rate, tr_loss, tr_acc, val_loss, val_acc') self.progress_bar = ProgressBar() self.chrono = Chrono() self.trainset, self.testset, self.trainloader, self.testloader = dataloader(args.first_n_byte) print('==> Building model..') self.model = MalConv(input_length=args.first_n_byte) if torch.cuda.is_available(): self.model = torch.nn.DataParallel(self.model) torch.backends.cudnn.benchmark = True self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.lr) if args.resume or self.test_only or self.dump_statistics: self.load() self.criterion = torch.nn.BCELoss() self.criterion = get_torch_vars(self.criterion, False) self.model = get_torch_vars(self.model, False)
def process_and_train_FL(model: BasicModel, opt: config.Option): # preparing saving files. save_path = os.path.join(opt.save_dir + '/model_file', opt.save_model_name).replace('\\', '/') print("model file save path: ", save_path) if not os.path.exists(save_path): os.makedirs(save_path) msger = messager( save_path=save_path, types=[ 'train_data_file', 'val_data_file', 'test_data_file', 'load_model_name', 'save_model_name', 'trainset_loss_type', 'testset_loss_type', 'class_num_ratio' ], json_name='train_information_msg_' + time.strftime('%m{}%d{}_%H:%M'.format('月', '日')) + '.json') msger.record_message([ opt.train_data_file, opt.val_data_file, opt.test_data_file, opt.load_model_name, opt.save_model_name, opt.train_loss_type, opt.testset_loss_type, opt.class_num_ratio ]) msger.save_json() # train data loading print('-----Data Loading-----') if opt.BERT: dataloader_train = dataloader_BERT(opt.train_data_file, opt.wordvec_file, opt.rel2id_file, opt.similarity_file, opt.same_level_pair_file, max_len=opt.max_len, random_init=opt.random_init, seed=opt.seed) dataloader_val = dataloader_BERT(opt.val_data_file, opt.wordvec_file, opt.rel2id_file, opt.similarity_file, max_len=opt.max_len) dataloader_test = dataloader_BERT(opt.test_data_file, opt.wordvec_file, opt.rel2id_file, opt.similarity_file, max_len=opt.max_len) else: dataloader_train = dataloader(opt.train_data_file, opt.wordvec_file, opt.rel2id_file, opt.similarity_file, opt.same_level_pair_file, max_len=opt.max_len, random_init=opt.random_init, seed=opt.seed, data_type=opt.data_type) dataloader_val = dataloader(opt.val_data_file, opt.wordvec_file, opt.rel2id_file, opt.similarity_file, max_len=opt.max_len, data_type=opt.data_type) dataloader_test = dataloader(opt.test_data_file, opt.wordvec_file, opt.rel2id_file, opt.similarity_file, max_len=opt.max_len, data_type=opt.data_type) word_emb_dim = dataloader_train._word_emb_dim_() word_vec_mat = dataloader_train._word_vec_mat_() # numpy.array float32 print('word_emb_dim is {}'.format(word_emb_dim)) # compile model print('-----Model Initializing-----') if opt.BERT != True: model.set_embedding_weight(word_vec_mat) if opt.load_model_name is not None: model.load_model(opt.load_model_name) os.environ["CUDA_VISIBLE_DEVICES"] = opt.gpu if torch.cuda.is_available(): torch.cuda.set_device(int(opt.gpu)) model, cuda_flag = tl.cudafy(model) if not cuda_flag: print("There is no gpu,use default cpu") count = tl.count_parameters(model) print("num of parameters:", count) # if the datasets are imbalanced such as nyt_su or trex , we load all test/dev data to perform open setting print('-----Validation Data Preparing-----') try: opt.data_type.index('imbalance') print("try load all imbalance dev data!") val_data, val_data_label = dataloader_val._data_() except: print("load part of data") if opt.data_type.startswith('fewrel'): val_data, val_data_label = dataloader_val._part_data_( 100 ) # 16 relation classes in validation data,each class has 100 sample in fewrel else: # other data sets has the problem of label imbalance val_data, val_data_label = dataloader_val._part_data_( 100 ) # for nyt_fb :sample 5 instance per relation, will get 490 dev instance print("-------Test Data Preparing--------") try: opt.data_type.index('imbalance') print("try load all imbalance test data!") test_data, test_data_label = dataloader_test._data_() except: print("load part of data") if opt.data_type.startswith('fewrel'): test_data, test_data_label = dataloader_test._data_() else: test_data, test_data_label = dataloader_test._data_( 100) # sample as the dev setting print("val_data:", len(val_data)) print("val_data_label:", len(set(val_data_label))) print("test_data:", len(test_data)) print("test_data_label:", len(set(test_data_label))) # intializing parameters batch_num_list = opt.batch_num msger_cluster = messager( save_path=save_path, types=[ 'method', 'temp_epoch', 'temp_batch_num', 'temp_batch_size', 'temp_lr', 'NMI', 'F1', 'precision', 'recall', 'msg' ], json_name='Validation_cluster_msg_' + time.strftime('%m{}%d{}_%H:%M'.format('月', '日')) + '.json') if opt.record_test: msger_test = messager( save_path=save_path, types=[ 'temp_globle_step', 'temp_batch_size', 'temp_learning_rate', 'NMI', 'F1', 'precision', 'recall', 'msg' ], json_name='Test_cluster_msg_' + time.strftime('%m{}%d{}_%H:%M'.format('月', '日')) + '.json') if opt.whether_visualize: loger = SummaryWriter(comment=opt.save_model_name) else: loger = None best_batch_step = 0 best_epoch = 0 batch_size_chose = -1 print_flag = opt.print_losses best_validation_f1 = 0 best_test_f1 = 0 loss_list = [] global_step = 0 for epoch in range(opt.epoch_num): print('------epoch {}------'.format(epoch)) print('max batch num to train is {}'.format(batch_num_list[epoch])) loss_reduce = 10000. early_stop_record = 0 for i in range(1, batch_num_list[epoch] + 1): global_step += 1 loss_list = model.train_self(opt, dataloader_train, loss_list, loger, batch_chose=batch_size_chose, global_step=global_step, temp_epoch=epoch) # print loss & record loss if i % 100 == 0: ave_loss = sum(loss_list) / 100. print('temp_batch_num: ', i, ' total_batch_num: ', batch_num_list[epoch], " ave_loss: ", ave_loss, ' temp learning rate: ', opt.lr[opt.lr_chose]) # empty the loss list loss_list = [] # visualize if opt.whether_visualize: loger.add_scalar('all_epoch_loss', ave_loss, global_step=global_step) # early stop if opt.early_stop is not None: if ave_loss < loss_reduce: early_stop_record = 0 loss_reduce = ave_loss else: early_stop_record += 1 if early_stop_record == opt.early_stop: print( "~~~~~~~~~ The loss can't be reduced in {} step, early stop! ~~~~~~~~~~~~" .format(opt.early_stop * 100)) cluster_result, cluster_msg, cluster_center, features = K_means_BERT( test_data, model.pred_vector, test_data_label, opt) if opt.BERT else K_means( test_data, model.pred_vector, len(np.unique(test_data_label)), opt) cluster_test_b3 = ClusterEvaluation( test_data_label, cluster_result).printEvaluation(extra_info=True, print_flag=True) print("learning rate decay num:", opt.lr_decay_num) print("learning rate decay step:", opt.lr_decay_record) print("best_epoch:", best_epoch) print("best_step:", best_batch_step) print("best_batch_size:", best_batch_size) print("best_cluster_eval_b3:", best_validation_f1) print("seed:", opt.seed) # clustering & validation if i % 200 == 0: print(opt.save_model_name, 'epoch:', epoch) with torch.no_grad(): # fewrel -> K-means ; nyt+su -> Mean-Shift if opt.dataset.startswith("fewrel"): print("chose k-means >>>") F_score = -1.0 best_cluster_result = None best_cluster_msg = None best_cluster_center = None best_features = None best_cluster_eval_b3 = None for iterion in range(opt.eval_num): K_num = opt.K_num if opt.K_num != 0 else len( np.unique(val_data_label)) cluster_result, cluster_msg, cluster_center, features = K_means_BERT( val_data, model.pred_vector, val_data_label, opt) if opt.BERT else K_means( val_data, model.pred_vector, K_num, opt) cluster_eval_b3 = ClusterEvaluation( val_data_label, cluster_result).printEvaluation( print_flag=False) if F_score < cluster_eval_b3['F1']: F_score = cluster_eval_b3['F1'] best_cluster_result = cluster_result best_cluster_msg = cluster_msg best_cluster_center = cluster_center best_features = features best_cluster_eval_b3 = cluster_eval_b3 cluster_result = best_cluster_result cluster_msg = best_cluster_msg cluster_center = best_cluster_center features = best_features cluster_eval_b3 = best_cluster_eval_b3 else: print("chose mean-shift >>>") cluster_result, cluster_msg, cluster_center, features = mean_shift_BERT( val_data, model.pred_vector, val_data_label, opt) if opt.BERT else mean_shift( val_data, model.pred_vector, opt) cluster_eval_b3 = ClusterEvaluation( val_data_label, cluster_result).printEvaluation(print_flag=False, extra_info=True) NMI_score = normalized_mutual_info_score( val_data_label, cluster_result) print("NMI:{} ,F1:{} ,precision:{} ,recall:{}".format( NMI_score, cluster_eval_b3['F1'], cluster_eval_b3['precision'], cluster_eval_b3['recall'], )) msger_cluster.record_message([ opt.select_cluster, epoch, i, opt.batch_size[batch_size_chose], model.lr, NMI_score, cluster_eval_b3['F1'], cluster_eval_b3['precision'], cluster_eval_b3['recall'], cluster_msg ]) msger_cluster.save_json() two_f1 = cluster_eval_b3['F1'] if two_f1 > best_validation_f1: # acc if opt.record_test == False: model.save_model(model_name=opt.save_model_name, global_step=global_step) best_batch_step = i best_epoch = epoch best_batch_size = opt.batch_size[batch_size_chose] best_validation_f1 = two_f1 if opt.whether_visualize: loger.add_embedding(features, metadata=val_data_label, label_img=None, global_step=global_step, tag='ground_truth', metadata_header=None) loger.add_embedding(features, metadata=cluster_result, label_img=None, global_step=global_step, tag='prediction', metadata_header=None) loger.add_scalar('all_epoch_NMI', NMI_score, global_step=global_step) loger.add_scalar('all_epoch_F1', cluster_eval_b3['F1'], global_step=global_step) loger.add_scalar('all_epoch_precision', cluster_eval_b3['precision'], global_step=global_step) loger.add_scalar('all_epoch_recall', cluster_eval_b3['recall'], global_step=global_step) if opt.record_test: if opt.dataset.startswith("fewrel"): cluster_result, cluster_msg, cluster_center, features = K_means_BERT( test_data, model.pred_vector, test_data_label, opt) if opt.BERT else K_means( test_data, model.pred_vector, len(np.unique(test_data_label)), opt) cluster_test_b3 = ClusterEvaluation( test_data_label, cluster_result).printEvaluation( print_flag=False) else: cluster_result, cluster_msg, cluster_center, features = mean_shift_BERT( test_data, model.pred_vector, test_data_label, opt) if opt.BERT else mean_shift( test_data, model.pred_vector, opt) cluster_test_b3 = ClusterEvaluation( test_data_label, cluster_result).printEvaluation( print_flag=False, extra_info=True) msger_test.record_message([ global_step, opt.batch_size[batch_size_chose], opt.lr[opt.lr_chose], NMI_score, cluster_test_b3['F1'], cluster_test_b3['precision'], cluster_test_b3['recall'], cluster_msg ]) msger_test.save_json() print('test messages saved.') if cluster_test_b3['F1'] > best_test_f1: model.save_model(model_name=opt.save_model_name, global_step=global_step) best_batch_step = i best_epoch = epoch best_batch_size = opt.batch_size[batch_size_chose] best_test_f1 = cluster_test_b3['F1'] model.lr_decay(opt) opt.lr_decay_record.append(global_step) print('End: The model is:', opt.save_model_name, opt.train_loss_type, opt.testset_loss_type) if opt.dataset.startswith("fewrel"): print('\n-----K-means Clustering test-----') best_test_b3, NMI_score = k_means_cluster_evaluation( model, opt, test_data, test_data_label, loger) else: print("\n-----------Mean_shift Clustering test:---------------") model.load_model(opt.save_model_name + "_best.pt") cluster_result_ms, cluster_msg_ms, _, _ = mean_shift_BERT( test_data, model.pred_vector, test_data_label, opt) if opt.BERT else mean_shift(test_data, model.pred_vector, opt) cluster_eval_b3_ms = ClusterEvaluation( test_data_label, cluster_result_ms).printEvaluation(print_flag=opt.print_losses, extra_info=True) NMI_score_ms = normalized_mutual_info_score(test_data_label, cluster_result_ms) best_test_b3 = cluster_eval_b3_ms NMI_score = NMI_score_ms if opt.whether_visualize: loger.add_scalar('test_NMI_MeanShift', NMI_score_ms, global_step=0) loger.add_scalar('test_F1_MeanShift', cluster_eval_b3_ms['F1'], global_step=0) print("learning rate decay num:", opt.lr_decay_num) print("learning rate decay step:", opt.lr_decay_record) print("best_epoch:", best_epoch) print("best_step:", best_batch_step) print("best_batch_size:", best_batch_size) print("best_cluster_eval_b3:", best_validation_f1) print("best_cluster_test_b3:", best_test_b3) print("best_NMI_score:", NMI_score) print("seed:", opt.seed)
def train(classifier, segmenter, epochs=10): data_loader = dataloader() loss = torch.nn.CrossEntropyLoss() gen_opt = torch.optim.Adam(segmenter.parameters()) clas_opt = torch.optim.Adam(classifier.parameters()) for i in range(epochs): for j, (imgs, ground_truths) in enumerate(data_loader): #Extract class label from ground truth label lbls = [] for k in range(len(ground_truths)): lbl = ground_truths[k] classes, mask = torch.unique(lbl, return_inverse=True) classes *= 255 max = 0 #Choose the class that has the largest area for l in range(len(classes)): if classes[l] != 0 and classes[l] != 255 and torch.sum( mask == l) > max: max = l lbls.append(classes[max]) imgs = imgs.cuda() lbls = torch.from_numpy(np.array(lbls)).long().cuda() - 1 #Obtain the Guided Backprop and GradCAM gc = grad_cam(classifier, imgs) gbp = guided_bp(classifier, imgs) #Input to the segmenter is the original image + gc + gbp (5 channels) seg_input = torch.cat((imgs, gc, gbp), 1).cuda() #Invert the output mask to leave only the background seg_output = segmenter(seg_input)[:, lbls[0], :, :] masks = (seg_output < 0.9).type(torch.cuda.FloatTensor) seg_imgs = [] for k in range(imgs.shape[0]): seg_imgs.append(imgs[k] * masks[k, k]) #Segmentor tries to increase the classifier loss after extracting the target class seg_imgs = torch.stack(seg_imgs) s_loss = -loss(classifier(seg_imgs), lbls) gen_opt.zero_grad() s_loss.backward() gen_opt.step() #Classifier tries to decrease loss on both original images and segmented images c_loss = loss(classifier(imgs), lbls) + loss( classifier(seg_imgs), lbls) clas_opt.zero_grad() c_loss.backward() clas_opt.step() if j % 20 == 0: print( "Epoch {} Iter {} - Segmentation Loss: {} Classificaton Loss: {}" .format(i, j, s_loss * -1, c_loss)) return classifier, segmenter
def test_stardardize(self): x, y = dataloader(mode='train', reduced=False) x = standardize(x) npt.assert_array_almost_equal(np.mean(x, axis=0), np.zeros(30)) npt.assert_array_almost_equal(np.std(x, axis=0), np.ones(30))
def train(train_x, train_y, learning_rate=0.1, num_epochs=50, batch_size=1): # Flatten input (num_samples, 28, 28) -> (num_samples, 784) x = train_x.reshape(train_x.shape[0], -1) num_samples = x.shape[0] # Turn labels into their one-hot representations y = one_hot_encoder(train_y) # Make a data loader trainloader = dataloader(x, y, batch_size=batch_size, shuffle=True) # Initialize weights w1, b1 = initialize_weight((784, 256), bias=True) w2, b2 = initialize_weight((256, 10), bias=True) loss_history = [] for epoch in range(1, num_epochs + 1): print("Epoch {}/{}\n===============".format(epoch, num_epochs)) batch_loss = 0 acc = 0 for inputs, labels in trainloader: # Number of samples per batch m = inputs.shape[0] # Forward Prop h1 = np.dot(inputs, w1) + b1 a1 = sigmoid(h1) h2 = np.dot(a1, w2) + b2 a2 = softmax(h2) out = a2 # Cross Entropy Loss batch_loss += cross_entropy_loss( out, labels.argmax(axis=1).reshape(m, 1)) # Compute Accuracy pred = np.argmax(out, axis=1) pred = pred.reshape(pred.shape[0], 1) acc += np.sum(pred == labels.argmax(axis=1).reshape(m, 1)) # Backward Prop dh2 = a2 - labels dw2 = (1 / m) * np.dot(a1.T, dh2) db2 = (1 / m) * np.sum(dh2, axis=0, keepdims=True) dh1 = np.dot(dh2, w2.T) * sigmoid_prime(a1) dw1 = (1 / m) * np.dot(inputs.T, dh1) db1 = (1 / m) * np.sum(dh1, axis=0, keepdims=True) # Weight (and bias) update w1 -= learning_rate * dw1 b1 -= learning_rate * db1 w2 -= learning_rate * dw2 b2 -= learning_rate * db2 loss_history.append(batch_loss / num_samples) print("Loss: {:.6f}".format(batch_loss / num_samples)) print("Accuracy: {:.2f}%\n".format(acc / num_samples * 100)) return w1, b1, w2, b2, loss_history
def __init__(self, args, SUPERVISED=True): # Parametres self.epoch = args.epoch self.batch_size = args.batch_size self.save_dir = 'models' self.result_dir = 'results' self.dataset = args.dataset self.model_name = 'InfoGAN' self.input_size = args.input_size self.z_dim = 62 self.SUPERVISED = SUPERVISED # if it is true, label info is directly used for code self.len_discrete_code = 10 # categorical distribution (i.e. label) self.len_continuous_code = 2 # gaussian distribution (e.g. rotation, thickness) self.sample_num = self.len_discrete_code ** 2 # load dataset self.data_loader = utils.dataloader(self.input_size, self.batch_size, self.dataset) data = self.data_loader.__iter__().__next__()[0] # networks init self.G = generator(input_dim=self.z_dim, output_dim=data.shape[1], input_size=self.input_size, len_discrete_code=self.len_discrete_code, len_continuous_code=self.len_continuous_code) self.D = discriminator(input_dim=data.shape[1], output_dim=1, input_size=self.input_size, len_discrete_code=self.len_discrete_code, len_continuous_code=self.len_continuous_code) self.G_optimizer = optim.Adam(self.G.parameters(), lr=args.lrG, betas=(args.beta1, args.beta2)) self.D_optimizer = optim.Adam(self.D.parameters(), lr=args.lrD, betas=(args.beta1, args.beta2)) self.info_optimizer = optim.Adam(itertools.chain(self.G.parameters(), self.D.parameters()), lr=args.lrD, betas=(args.beta1, args.beta2)) self.G.cpu() self.D.cpu() self.BCE_loss = nn.BCELoss().cpu() self.CE_loss = nn.CrossEntropyLoss().cpu() self.MSE_loss = nn.MSELoss().cpu() print('---------- Networks architecture -------------') utils.print_network(self.G) utils.print_network(self.D) print('-----------------------------------------------') # fixed noise & condition self.sample_z_ = torch.zeros((self.sample_num, self.z_dim)) for i in range(self.len_discrete_code): self.sample_z_[i * self.len_discrete_code] = torch.rand(1, self.z_dim) for j in range(1, self.len_discrete_code): self.sample_z_[i * self.len_discrete_code + j] = self.sample_z_[i * self.len_discrete_code] temp = torch.zeros((self.len_discrete_code, 1)) for i in range(self.len_discrete_code): temp[i, 0] = i temp_y = torch.zeros((self.sample_num, 1)) for i in range(self.len_discrete_code): temp_y[i * self.len_discrete_code: (i + 1) * self.len_discrete_code] = temp self.sample_y_ = torch.zeros((self.sample_num, self.len_discrete_code)).scatter_(1, temp_y.type(torch.LongTensor), 1) self.sample_c_ = torch.zeros((self.sample_num, self.len_continuous_code)) # manipulating two continuous code self.sample_z2_ = torch.rand((1, self.z_dim)).expand(self.sample_num, self.z_dim) self.sample_y2_ = torch.zeros(self.sample_num, self.len_discrete_code) self.sample_y2_[:, 0] = 1 temp_c = torch.linspace(-1, 1, 10) self.sample_c2_ = torch.zeros((self.sample_num, 2)) for i in range(self.len_discrete_code): for j in range(self.len_discrete_code): self.sample_c2_[i*self.len_discrete_code+j, 0] = temp_c[i] self.sample_c2_[i*self.len_discrete_code+j, 1] = temp_c[j] self.sample_z_, self.sample_y_, self.sample_c_, self.sample_z2_, self.sample_y2_, self.sample_c2_ = \ self.sample_z_.cpu(), self.sample_y_.cpu(), self.sample_c_.cpu(), self.sample_z2_.cpu(), \ self.sample_y2_.cpu(), self.sample_c2_.cpu()
model.to(device) optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.decay) if args.checkpoint is not None: try: load_checkpoint(args.checkpoint, model, optimizer) print('load checkpoint', args.checkpoint) except FileNotFoundError: print('checkpoint not found:', args.checkpoint) params = list(model.parameters()) params = list(filter(lambda p: p.requires_grad, params)) nparams = sum([np.prod(p.size()) for p in params]) print ('total nubmer of trainable parameters:', nparams) train_loader, valid_loader, test_loader = dataloader(args.dataset, args.Batchsize, args.cuda) if not args.train: model.eval() data, _ = list(test_loader)[0] data = data.to(device).view(-1, dim) #gaussianization(data, model) reversibility(data, model) if args.show: plt.show() else: plt.savefig(args.outname, dpi=300, transparent=True) else: if args.interactive:
chrono.remove("step_time") if __name__ == '__main__': parser = argparse.ArgumentParser(description="Train Autoencoder") parser.add_argument("--valid", action="store_true", default=False, help="Perform validation only.") args = parser.parse_args() chrono = Chrono() progress_bar = ProgressBar() _, _, trainloader, testloader = dataloader() autoencoder = get_torch_vars(AutoEncoder(), False) if torch.cuda.is_available(): autoencoder = torch.nn.DataParallel(autoencoder) torch.backends.cudnn.benchmark = True if args.valid: valid() exit(0) criterion = nn.BCEWithLogitsLoss() optimizer = optim.Adam(autoencoder.parameters()) train()
def test_dataloader(self): x, y = dataloader(mode='train') self.assertEqual(np.shape(x), (250000, 30)) self.assertEqual(np.shape(y), (250000, ))
@Software: PyCharm """ from model import Perceptron from utils import setup_seed, dataloader import random import numpy as np from sklearn.metrics import accuracy_score # hyperparameter iters = 50000 lr = 0.0001 seed = 1234 # main process setup_seed(seed) x_train, x_test, y_train, y_test = dataloader(ratio=0.9) perceptron = Perceptron(feat_dim=30, learning_rate=lr) train_num = x_train.shape[0] test_num = x_test.shape[0] # train for iter in range(iters): # 每次随机从训练集中选择一个样本进行训练 instance_id = random.randint(0, train_num - 1) x = x_train[instance_id] y_true = y_train[instance_id] y_pred = perceptron.forward(x) perceptron.backward(x, y_pred=y_pred, y_true=y_true) print("iteration {} finished. y_true: {}, y_pred: {}.".format( iter, y_true, y_pred)) # test
def train(self, task_id, traindata, trainlabels): train_data, train_label, train_attr, no_steps = utils.dataloader( traindata, trainlabels, traindata, self.args.test_batch_size) for e in range(self.args.class_epochs): loss = self.train_epoch(task_id, train_data, train_label, no_steps) print('epoch:', e + 1, 'class_loss:', loss)
state_dict.update(pretrain_state_dict) self.model.load_state_dict(state_dict) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--lr', type=float, required=True) parser.add_argument('--gpu', type=str, default='0') parser.add_argument('--save-model-path', type=str, required=True) parser.add_argument('--load-model-path', type=str) parser.add_argument('--is-train', action='store_true', default=False) parser.add_argument('--batch-size', type=int, default=20) parser.add_argument('--data-root', type=str, required=True) args = parser.parse_args() os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu torch.manual_seed(0) torch.cuda.manual_seed_all(0) train_loader, eval_loader = dataloader(args.batch_size, args.data_root, '../../data/diagnosis.txt') model = Model() model = nn.DataParallel(model.cuda()) if not os.path.exists(args.save_model_path): os.makedirs(args.save_model_path) manager = Manager(train_loader, eval_loader, model, args.lr, args.save_model_path) if args.load_model_path: manager.load_model_state_dict(args.load_model_path) if args.is_train: manager.fit()
def main(args): ## load datasets train_dataset = dataloader('dogs_cats', 'train') ## split train and validation num_train = len(train_dataset) indices = list(range(num_train)) split = 5000 validation_idx = np.random.choice(indices, size=split, replace=False) train_idx = list(set(indices) - set(validation_idx)) train_sampler = SubsetRandomSampler(train_idx) validation_sampler = SubsetRandomSampler(validation_idx) ## train and validation loader train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=args.batch_size, sampler=train_sampler) valid_loader = torch.utils.data.DataLoader( train_dataset, batch_size=args.batch_size, sampler=validation_sampler) ## debug if args.debug: images, _ = next(iter(train_loader)) grid = torchvision.utils.make_grid(images[:25], nrow=5) imshow(grid, 'train') images, _ = next(iter(valid_loader)) grid = torchvision.utils.make_grid(images[:25], nrow=5) imshow(grid, 'valid') ## define model model = AutoEncoder() use_gpu = torch.cuda.is_available() if use_gpu: print('cuda is available!') model.cuda() ## loss and optimizer criterion = nn.MSELoss() optimizer = torch.optim.SGD( model.parameters(), lr=0.001, momentum=0.9, weight_decay=1e-5) ## log log_dir = 'logs' if not os.path.isdir('logs'): os.mkdir('logs') ## train and valid best_val = 5 loss_list = [] val_loss_list = [] for epoch in range(args.n_epochs): loss = train(model, criterion, optimizer, train_loader, use_gpu) val_loss = valid(model, criterion, valid_loader, use_gpu) print('epoch {:d}, loss: {:.4f} val_loss: {:.4f}'.format(epoch, loss, val_loss)) if val_loss < best_val: print('val_loss improved from {:.5f} to {:.5f}!'.format(best_val, val_loss)) best_val = val_loss model_file = 'epoch{:03d}-{:.3f}.pth'.format(epoch, val_loss) torch.save(model.state_dict(), os.path.join(log_dir, model_file)) loss_list.append(loss) val_loss_list.append(val_loss)
def main(): args = parser.parse_args() if args.exp_name == '': print( 'ERROR: USE \'--exp-name\' or \'-n\' option to define this experiment\'s name.' ) sys.exit() # directories settings os.makedirs('../logs/outputs', exist_ok=True) os.makedirs('../logs/models/{}'.format(args.exp_name), exist_ok=True) OUTPUT_PATH = '../logs/outputs/{}.log'.format(args.exp_name) MODEL_PATH = '../logs/models/{}/'.format(args.exp_name) if os.path.exists(OUTPUT_PATH): print( 'ERROR: This \'--exp-name\' is already used. Use another name for this experiment.' ) sys.exit() # recording outputs sys.stdout = open(OUTPUT_PATH, 'w') sys.stderr = open(OUTPUT_PATH, 'a') # tensorboardX writer = SummaryWriter(log_dir='../logs/tb/{}'.format(args.exp_name)) # cuda settings args.cuda = not args.no_cuda and torch.cuda.is_available() device = torch.device("cuda:0" if args.cuda else "cpu") print('device: {}'.format(device)) # for fast training cudnn.benchmark = True np.random.seed(args.seed) torch.manual_seed(args.seed) if args.cuda: torch.cuda.manual_seed(args.seed) # data settings trainloader, testloader, _ = dataloader(batch_size=args.batch_size) # Model, Criterion, Optimizer model = AlexNetCifar10().to(device) criterion = nn.CrossEntropyLoss().to(device) optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=0.9, weight_decay=args.weight_decay) # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) print("=> loaded checkpoint '{}' (epoch {})".format( args.resume, checkpoint['epoch'])) else: print("=> no checkpoint found at '{}'".format(args.resume)) # print settings print('=' * 5 + ' settings ' + '=' * 5) print('TRAINING MODE: {}'.format(args.mode)) if args.mode == 'blur-step': print('### BLUR CHANGING STEPS ###') print('Step: 1-10 -> 11-20 -> 21-30 -> 31-40 -> 41-{}'.format( args.epochs)) print('Sigma: 4 -> 3 -> 2 -> 1 -> none') # print('Kernel-size: (25, 25) -> (19, 19) -> (13, 13) -> (7, 7) -> none') print('#' * 20) elif args.mode == 'blur-half-epochs': print('### NO BLUR FROM EPOCH {:d} ###'.format(args.epochs // 2)) print('Sigma: {}'.format(args.sigma)) # print('Kernel-size: {}'.format(tuple(args.kernel_size))) # radius = sigma * 3 * 2 + 1 elif args.mode == 'blur-all': print('Sigma: {}'.format(args.sigma)) # print('Kernel-size: {}'.format(tuple(args.kernel_size))) # radius = sigma * 3 * 2 + 1 if args.blur_val: print('VALIDATION MODE: blur-val') print('Random seed: {}'.format(args.seed)) print('Epochs: {}'.format(args.epochs)) print('Learning rate: {}'.format(args.lr)) print('Weight_decay: {}'.format(args.weight_decay)) print() print(model) print('=' * 20) print() # training print('Start Training...') train_time = time.time() blur = True for epoch in range(args.start_epoch, args.epochs): # loop over the dataset multiple times if args.mode == 'normal': blur = False elif args.mode == 'blur-step': ### BLUR-STEP SETTINGS ### if epoch < 10: args.sigma = 4 # args.kernel_size = (25, 25) # radius = sigma * 3 * 2 + 1 elif epoch < 20: args.sigma = 3 # args.kernel_size = (19, 19) elif epoch < 30: args.sigma = 2 # args.kernel_size = (13, 13) elif epoch < 40: args.sigma = 1 # args.kernel_size = (7, 7) else: blur = False elif args.mode == 'blur-half-epochs': if epoch >= args.epochs // 2: blur = False # ===== train mode ===== train_acc = AverageMeter('train_acc', ':6.2f') train_loss = AverageMeter('train_loss', ':.4e') model.train() for i, data in enumerate(trainloader, 0): # get the inputs; data is a list of [inputs, labels] inputs, labels = data[0], data[1].to(device) # Blur images if blur: if args.mode == 'blur-half-data': half1, half2 = inputs.chunk(2) # blur first half images half1 = GaussianBlurAll(half1, (0, 0), args.sigma) inputs = torch.cat((half1, half2)) else: inputs = GaussianBlurAll(inputs, (0, 0), args.sigma) inputs = inputs.to(device) # zero the parameter gradients optimizer.zero_grad() # forward + record outputs = model(inputs) loss = criterion(outputs, labels) acc1 = accuracy(outputs, labels, topk=(1, )) train_loss.update(loss.item(), inputs.size(0)) train_acc.update(acc1[0], inputs.size(0)) # backward + optimize loss.backward() optimizer.step() # record the values in tensorboard writer.add_scalar('loss/train', train_loss.avg, epoch + 1) # average loss writer.add_scalar('acc/train', train_acc.avg, epoch + 1) # average acc # ===== val mode ===== val_acc = AverageMeter('val_acc', ':6.2f') val_loss = AverageMeter('val_loss', ':.4e') model.eval() with torch.no_grad(): for data in testloader: inputs, labels = data[0], data[1].to(device) if args.blur_val: inputs = GaussianBlurAll(inputs, (0, 0), args.sigma) inputs = inputs.to(device) outputs = model(inputs) loss = criterion(outputs, labels) acc1 = accuracy(outputs, labels, topk=(1, )) val_loss.update(loss.item(), inputs.size(0)) val_acc.update(acc1[0], inputs.size(0)) # record the values in tensorboard writer.add_scalar('loss/val', val_loss.avg, epoch + 1) # average loss writer.add_scalar('acc/val', val_acc.avg, epoch + 1) # average acc # ===== save the model ===== if (epoch + 1) % 10 == 0: save_model( { 'epoch': epoch + 1, 'arch': 'alexnet-cifar10', 'val_loss': val_loss.avg, 'val_acc': val_acc.avg, 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict() }, MODEL_PATH, epoch + 1) print('Finished Training') print("Training time elapsed: {:.4f}mins".format( (time.time() - train_time) / 60)) print() writer.close() # close tensorboardX writer
'''Here, we will see whether we could use PCA as our regression feature to classifier a stock's class''' from utils import dataloader, accuracy, confusion_map from sklearn.decomposition import PCA from sklearn.linear_model import LogisticRegression import matplotlib.pyplot as plt from mpl_toolkits.mplot3d import Axes3D import numpy as np np.random.seed(5) # prepare data X, X_class, X_index, Y, cl2co, class2bbr, bbr2class = dataloader() X_train, X_test = X[:300], X[300:] Y_train, Y_test = Y[:300], Y[300:] # PCA n = 150 pca = PCA(n_components=n) pca.fit(X_train) X_train = pca.transform(X_train) X_test = pca.transform(X_test) print('Number of Component:', n) print('Explain Variance: {:.3f}'.format(sum(pca.explained_variance_ratio_))) # logistic regression clf = LogisticRegression(C=60, solver='newton-cg') clf.fit(X_train, Y_train)
print("WARNING: You have a CUDA device, so you should probably set cuda in params.py to True") # ----------------------------------------------- """ In this block Get train and val data_loader """ def loader_param(): img_x = 32 img_y = 128 batch_size = 4 return(img_x, img_y, batch_size) img_x, img_y, batch_size = loader_param() dataset = dataset.dataset(image_root="../IAM Dataset/words/", label_root = "../IAM Dataset/ascii/labels.txt", img_x = img_x, img_y = img_y) train_loader, val_loader = utils.dataloader(dataset = dataset, batch_size = 16, validation_split = 0.2, shuffle_dataset = True) # def data_loader(): # # train # train_dataset = dataset.lmdbDataset(root=args.trainroot) # assert train_dataset # if not params.random_sample: # sampler = dataset.randomSequentialSampler(train_dataset, params.batchSize) # else: # sampler = None # train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=params.batchSize, \ # shuffle=True, sampler=sampler, num_workers=int(params.workers), \ # collate_fn=dataset.alignCollate(imgH=params.imgH, imgW=params.imgW, keep_ratio=params.keep_ratio)) # # val