def prep_data(df): new_df = df.copy() fraud_labels = ['fraudster', 'fraudster_att', 'fraudster_event'] new_df = EDA.create_response_label(new_df, fraud_labels) df_num = new_df.select_dtypes(include=[np.number]) df_num.fillna(value=0, inplace=True) X = df_num.drop(EDA.get_fraud_label(), axis=1) y = df_num[EDA.get_fraud_label()] return (X, y) def get_dataframe_from_zip(filename): #returns dataframe of zipped JSON file zip = ZipFile(filename) zip.extractall('files/') # file should be data.json return pd.read_json('files/data.{}'.format('json')) if __name__ == '__main__': from model import Classifier df = get_dataframe_from_zip("files/data.zip") X, y = prep_data(df) modeler = Classifier() modeler.fit(X, y) score = modeler.score(X, y) print('Self score: ', score) with open('files/model.pkl', 'w') as f: pickle.dump(modeler, f)
class Solver(object): def __init__(self, Msceleb_loader, config): # Data loader self.Msceleb_loader = Msceleb_loader # Model hyper-parameters self.c_dim = config.c_dim self.image_size = config.image_size self.g_conv_dim = config.g_conv_dim self.d_conv_dim = config.d_conv_dim self.g_repeat_num = config.g_repeat_num self.d_repeat_num = config.d_repeat_num self.d_train_repeat = config.d_train_repeat # Hyper-parameteres self.lambda_cls = config.lambda_cls self.lambda_rec = config.lambda_rec self.lambda_gp = config.lambda_gp self.g_lr = config.g_lr self.d_lr = config.d_lr self.beta1 = config.beta1 self.beta2 = config.beta2 # Training settings # self.dataset = config.dataset self.num_epochs = config.num_epochs self.num_epochs_decay = config.num_epochs_decay self.num_iters = config.num_iters self.num_iters_decay = config.num_iters_decay self.batch_size = config.batch_size self.use_tensorboard = config.use_tensorboard self.pretrained_model = config.pretrained_model # Test settings self.test_model = config.test_model # Path self.log_path = config.log_path self.sample_path = config.sample_path self.model_save_path = config.model_save_path self.result_path = config.result_path # Step size self.log_step = config.log_step self.sample_step = config.sample_step self.model_save_step = config.model_save_step # self.lambda_face = 0.0 # if self.lambda_face > 0.0: # self.Face_recognition_network = face_recognition_networks.LightCNN_29Layers(num_classes=79077) # self.Face_recognition_network = torch.nn.DataParallel(self.Face_recognition_network).cuda() # checkpoint = torch.load(r'/data5/shentao/LightCNN/CNN_29.pkl') # self.Face_recognition_network.load_state_dict(checkpoint) # for param in self.Face_recognition_network.parameters(): # param.requires_grad = False # self.Face_recognition_network.eval() # Build tensorboard if use self.build_model() if self.use_tensorboard: self.build_tensorboard() # Start with trained model if self.pretrained_model: self.load_pretrained_model() def build_model(self): self.G = Generator(self.g_conv_dim, self.g_repeat_num) self.D = Discriminator(self.d_conv_dim, self.d_repeat_num) self.C = Classifier(self.image_size, self.d_conv_dim, self.c_dim, self.d_repeat_num) # Optimizers self.g_optimizer = torch.optim.Adam(self.G.parameters(), self.g_lr, [self.beta1, self.beta2]) self.d_optimizer = torch.optim.Adam(self.D.parameters(), self.d_lr, [self.beta1, self.beta2]) self.c_optimizer = torch.optim.Adam(self.C.parameters(), self.d_lr, [self.beta1, self.beta2]) # Print networks self.print_network(self.G, 'G') self.print_network(self.D, 'D') self.print_network(self.C, 'C') if torch.cuda.is_available(): self.G.cuda() self.D.cuda() self.C.cuda() def print_network(self, model, name): num_params = 0 for p in model.parameters(): num_params += p.numel() print(name) print(model) print("The number of parameters: {}".format(num_params)) def load_pretrained_model(self): self.G.load_state_dict(torch.load(os.path.join( self.model_save_path, '{}_G.pth'.format(self.pretrained_model)))) self.D.load_state_dict(torch.load(os.path.join( self.model_save_path, '{}_D.pth'.format(self.pretrained_model)))) self.C.load_state_dict(torch.load(os.path.join( self.model_save_path, '{}_C.pth'.format(self.pretrained_model)))) print('loaded trained models (step: {})..!'.format(self.pretrained_model)) def build_tensorboard(self): from logger import Logger self.logger = Logger(self.log_path) def update_lr(self, g_lr, d_lr): for param_group in self.g_optimizer.param_groups: param_group['lr'] = g_lr for param_group in self.d_optimizer.param_groups: param_group['lr'] = d_lr for param_group in self.c_optimizer.param_groups: param_group['lr'] = d_lr def reset_grad(self): self.g_optimizer.zero_grad() self.d_optimizer.zero_grad() self.c_optimizer.zero_grad() def to_var(self, x, volatile=False): if torch.cuda.is_available(): x = x.cuda() return Variable(x, volatile=volatile) def denorm(self, x): out = (x + 1) / 2 return out.clamp_(0, 1) def threshold(self, x): x = x.clone() x[x >= 0.5] = 1 x[x < 0.5] = 0 return x def compute_accuracy(self, x, y): _, predicted = torch.max(x, dim=1) correct = (predicted == y).float() accuracy = torch.mean(correct) * 100.0 return accuracy def one_hot(self, labels, dim): """Convert label indices to one-hot vector""" batch_size = labels.size(0) out = torch.zeros(batch_size, dim) out[torch.from_numpy(np.arange(batch_size).astype(np.int64)), labels.long()] = 1 return out def train(self): """Train StarGAN within a single dataset.""" self.criterionL1 = torch.nn.L1Loss() # self.criterionL2 = torch.nn.MSELoss() self.criterionTV = TVLoss() self.data_loader = self.Msceleb_loader # The number of iterations per epoch iters_per_epoch = len(self.data_loader) fixed_x = [] real_c = [] for i, (aug_images, aug_labels, _, _) in enumerate(self.data_loader): fixed_x.append(aug_images) real_c.append(aug_labels) if i == 3: break # Fixed inputs and target domain labels for debugging fixed_x = torch.cat(fixed_x, dim=0) fixed_x = self.to_var(fixed_x, volatile=True) # lr cache for decaying g_lr = self.g_lr d_lr = self.d_lr # Start with trained model if exists if self.pretrained_model: start = int(self.pretrained_model.split('_')[0]) else: start = 0 # Start training start_time = time.time() for e in range(start, self.num_epochs): for i, (aug_x, aug_label, origin_x, origin_label) in enumerate(self.data_loader): # Generat fake labels randomly (target domain labels) # aug_c = self.one_hot(aug_label, self.c_dim) # origin_c = self.one_hot(origin_label, self.c_dim) aug_c_V = self.to_var(aug_label) origin_c_V = self.to_var(origin_label) aug_x = self.to_var(aug_x) origin_x = self.to_var(origin_x) # # ================== Train D ================== # # Compute loss with real images out_src = self.D(origin_x) out_cls = self.C(origin_x) d_loss_real = - torch.mean(out_src) c_loss_cls = F.cross_entropy(out_cls, origin_c_V) # Compute classification accuracy of the discriminator if (i+1) % self.log_step == 0: accuracies = self.compute_accuracy(out_cls, origin_c_V) log = ["{:.2f}".format(acc) for acc in accuracies.data.cpu().numpy()] print('Classification Acc (75268 ids): ') print(log) # Compute loss with fake images fake_x = self.G(aug_x) fake_x = Variable(fake_x.data) out_src = self.D(fake_x) d_loss_fake = torch.mean(out_src) # Backward + Optimize d_loss = d_loss_real + d_loss_fake c_loss = self.lambda_cls * c_loss_cls self.reset_grad() d_loss.backward() c_loss.backward() self.d_optimizer.step() self.c_optimizer.step() # Compute gradient penalty alpha = torch.rand(origin_x.size(0), 1, 1, 1).cuda().expand_as(origin_x) interpolated = Variable(alpha * origin_x.data + (1 - alpha) * fake_x.data, requires_grad=True) out = self.D(interpolated) grad = torch.autograd.grad(outputs=out, inputs=interpolated, grad_outputs=torch.ones(out.size()).cuda(), retain_graph=True, create_graph=True, only_inputs=True)[0] grad = grad.view(grad.size(0), -1) grad_l2norm = torch.sqrt(torch.sum(grad ** 2, dim=1)) d_loss_gp = torch.mean((grad_l2norm - 1)**2) # Backward + Optimize d_loss = self.lambda_gp * d_loss_gp self.reset_grad() d_loss.backward() self.d_optimizer.step() # Logging loss = {} loss['D/loss_real'] = d_loss_real.data[0] loss['D/loss_fake'] = d_loss_fake.data[0] loss['D/loss_gp'] = d_loss_gp.data[0] loss['C/loss_cls'] = c_loss_cls.data[0] # ================== Train G ================== # if (i+1) % self.d_train_repeat == 0: # Original-to-target and target-to-original domain fake_x = self.G(aug_x) # Compute losses out_src = self.D(fake_x) out_cls = self.C(fake_x) g_loss_fake = - torch.mean(out_src) g_loss_cls = F.cross_entropy(out_cls, aug_c_V) # Backward + Optimize recon_loss = self.criterionL1(fake_x, aug_x) TV_loss = self.criterionTV(fake_x) * 0.001 g_loss = g_loss_fake + self.lambda_cls * g_loss_cls + 5* recon_loss + TV_loss # if self.lambda_face > 0.0: # self.criterionFace = nn.L1Loss() # # real_input_x = (torch.sum(real_x, 1, keepdim=True) / 3.0 + 1) / 2.0 # fake_input_x = (torch.sum(fake_x, 1, keepdim=True) / 3.0 + 1) / 2.0 # rec_input_x = (torch.sum(rec_x, 1, keepdim=True) / 3.0 + 1) / 2.0 # # _, real_x_feature_fc, real_x_feature_conv = self.Face_recognition_network.forward( # real_input_x) # _, fake_x_feature_fc, fake_x_feature_conv = self.Face_recognition_network.forward( # fake_input_x) # _, rec_x1_feature_fc, rec_x1_feature_conv = self.Face_recognition_network.forward(rec_input_x) # # x1_loss = (self.criterionFace(fake_x1_feature_fc, Variable(real_x1_feature_fc.data,requires_grad=False)) + # # self.criterionFace(fake_x1_feature_conv,Variable(real_x1_feature_conv.data,requires_grad=False)))\ # # * self.lambda_face # x_loss = (self.criterionFace(fake_x_feature_fc,Variable(real_x_feature_fc.data, requires_grad=False))) \ # * self.lambda_face # # rec_x_loss = (self.criterionFace(rec_x1_feature_fc, Variable(real_x_feature_fc.data, requires_grad=False))) # # self.id_loss = x_loss + rec_x_loss # loss['G/id_loss'] = self.id_loss.data[0] # g_loss += self.id_loss self.reset_grad() g_loss.backward() self.g_optimizer.step() # Logging loss['G/loss_fake'] = g_loss_fake.data[0] loss['G/loss_cls'] = g_loss_cls.data[0] # Print out log info if (i+1) % self.log_step == 0: elapsed = time.time() - start_time elapsed = str(datetime.timedelta(seconds=elapsed)) log = "Elapsed [{}], Epoch [{}/{}], Iter [{}/{}]".format( elapsed, e+1, self.num_epochs, i+1, iters_per_epoch) for tag, value in loss.items(): log += ", {}: {:.4f}".format(tag, value) print(log) # Translate fixed images for debugging if (i+1) % self.sample_step == 0: fake_image_list = [fixed_x] fake_image_list.append(self.G(fixed_x)) fake_images = torch.cat(fake_image_list, dim=3) save_image(self.denorm(fake_images.data), os.path.join(self.sample_path, '{}_{}_fake.png'.format(e+1, i+1)),nrow=1, padding=0) print('Translated images and saved into {}..!'.format(self.sample_path)) # Save model checkpoints if (i+1) % self.model_save_step == 0: torch.save(self.G.state_dict(), os.path.join(self.model_save_path, '{}_{}_G.pth'.format(e+1, i+1))) torch.save(self.D.state_dict(), os.path.join(self.model_save_path, '{}_{}_D.pth'.format(e+1, i+1))) torch.save(self.C.state_dict(), os.path.join(self.model_save_path, '{}_{}_C.pth'.format(e+1, i+1))) # Decay learning rate if (e+1) > (self.num_epochs - self.num_epochs_decay): g_lr -= (self.g_lr / float(self.num_epochs_decay)) d_lr -= (self.d_lr / float(self.num_epochs_decay)) self.update_lr(g_lr, d_lr) print ('Decay learning rate to g_lr: {}, d_lr: {}.'.format(g_lr, d_lr)) torch.save(self.G.state_dict(), os.path.join(self.model_save_path, '{}_final_G.pth'.format(e + 1))) torch.save(self.D.state_dict(), os.path.join(self.model_save_path, '{}_final_D.pth'.format(e + 1))) torch.save(self.C.state_dict(), os.path.join(self.model_save_path, '{}_final_C.pth'.format(e + 1)))
add_output(modify_text) return if __name__ == '__main__': preparation() ae_model = get_cuda( make_model( d_vocab=args.vocab_size, N=args.num_layers_AE, d_model=args.transformer_model_size, latent_size=args.latent_size, d_ff=args.transformer_ff_size, )) dis_model = get_cuda( Classifier(latent_size=args.latent_size, output_size=args.label_size)) if args.if_load_from_checkpoint: # Load models' params from checkpoint ae_model.load_state_dict( torch.load(args.current_save_path + 'ae_model_params.pkl')) dis_model.load_state_dict( torch.load(args.current_save_path + 'dis_model_params.pkl')) else: train_iters(ae_model, dis_model) # eval_iters(ae_model, dis_model) print("Done!")
parser.add_argument("--num_epochs", type=int, default=200) parser.add_argument('--train', action="store_true") parser.add_argument('--evaluate', action="store_true") args = parser.parse_args() tf.logging.set_verbosity(tf.logging.INFO) if __name__ == "__main__": estimator = tf.estimator.Estimator( model_fn=lambda features, labels, mode, params: Classifier(network=ResNet( conv_param=Param(filters=32, kernel_size=[3, 3], strides=[1, 1]), pool_param=None, residual_params=[ Param(filters=32, strides=[1, 1], blocks=3), Param(filters=64, strides=[2, 2], blocks=3), Param(filters=128, strides=[2, 2], blocks=3), ], num_classes=10))(features, labels, mode, Param(params)), model_dir=args.model_dir, config=tf.estimator.RunConfig(save_summary_steps=100, save_checkpoints_steps=1000), params=dict( weight_decay=2e-4, learning_rate=lambda global_step: tf.train.exponential_decay( learning_rate=0.1 * args.batch_size / 64, global_step=global_step, decay_steps=50000 * args.num_epochs / args.batch_size / 4, decay_rate=0.1), momentum=0.9,
import tensorflow as tf from model import Classifier from dataset import get_dataset # Set some hyper-parameters. BATCH_SIZE = 12 MAX_LEN = 512 # The max sequence length that BERT can handle is 512. # Get the dataset print('Preparing dataset...') dataset, n = get_dataset(mode='test', batch_size=BATCH_SIZE) # Build the BERT model. model = Classifier() model(tf.zeros((6, BATCH_SIZE, MAX_LEN), dtype=tf.int32)) model.summary() # Load the latest checkpoint. checkpoint = tf.train.Checkpoint(model=model) checkpoint.restore(tf.train.latest_checkpoint('ckpt')) @tf.function def _test_step(inputs): x, y = inputs[:-1], inputs[-1] pred = model.predict(x) accuracy.update_state(y, pred) accuracy = tf.keras.metrics.CategoricalAccuracy()
train_data_size = 200 if debug else int(n_dataset * 0.9) valid_data_size = 100 if debug else int(n_dataset - train_data_size) perm = np.random.RandomState(random_seed).permutation(n_dataset) train_dataset = BengaliAIDataset(train_images, train_labels, transform=Transform(train_aug_config), indices=perm[:train_data_size]) valid_dataset = BengaliAIDataset( train_images, train_labels, transform=Transform(valid_aug_config), indices=perm[train_data_size:train_data_size + valid_data_size]) predictor = Predictor(weights_path=trained_weights_path) classifier = Classifier(predictor).to(device) train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) valid_loader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False) optimizer = torch.optim.Adam(classifier.parameters(), lr=initial_lr) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=2, min_lr=1e-10)
summary=SummaryWriter('./runs/{}'.format(args.name)) ###################################################################################### # End of hyper parameters ###################################################################################### if __name__ == '__main__': preparation(args) ae_model = get_cuda(make_model(d_vocab=args.vocab_size, N=args.num_layers_AE, d_model=args.transformer_model_size, latent_size=args.latent_size, gpu=args.gpu, d_ff=args.transformer_ff_size), args.gpu) dis_model=get_cuda(Classifier(1, args),args.gpu) ae_optimizer = NoamOpt(ae_model.src_embed[0].d_model, 1, 2000, torch.optim.Adam(ae_model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9)) dis_optimizer=torch.optim.Adam(dis_model.parameters(), lr=0.0001) if args.load_model: # Load models' params from checkpoint ae_model.load_state_dict(torch.load(args.current_save_path + '/{}_ae_model_params.pkl'.format(args.load_iter), map_location=device)) dis_model.load_state_dict(torch.load(args.current_save_path + '/{}_dis_model_params.pkl'.format(args.load_iter), map_location=device)) start=args.load_iter+1 else: start=0 train_data_loader=non_pair_data_loader( batch_size=args.batch_size, id_bos=args.id_bos, id_eos=args.id_eos, id_unk=args.id_unk,
hypothesis = "" for id in sent2_batch[i]: if id2token[id.item()] == ",": hypothesis += "<unk> " elif id2token[id.item()] != "999994": hypothesis += id2token[id.item()] + " " print("Hypothesis: " + hypothesis) print("Actual label: " + dict_labels_ulta[label_batch[i].item()]) print("Predicted label: " + dict_labels_ulta[predicted[i].item()] + "\n\n") if ic == 3 and c == 3: break #total += labels.size(0) #correct.append(predicted.eq(label_batch.view_as(predicted))) #print("Correctly Classified thus far:\n\n ",correct) #return (100 * correct / total),loss args = parser.parse_args() state_dict = torch.load(args.model, map_location=lambda storage, loc: storage) model = Classifier('RNN', len(token2id), pretrained_vecs) model.load_state_dict(state_dict) model.eval() criterion = torch.nn.CrossEntropyLoss() #list_correct,list_incorrect= test_model(val_loader, model)
import os import re import csv import time import requests from bs4 import BeautifulSoup from collections import deque from model import Classifier from utils import extract_text_from_html, id_from_url model = Classifier("data/model.pkl") visited = set() with open("data/classes.csv", "r") as visited_file: reader = csv.DictReader(visited_file, delimiter=";") visited = set([row["url"] for row in reader]) visited_file = open("data/classes.csv", "a") # Set one seed to start the crawler to_visit = ["Metallica"] to_visit_queue = deque(to_visit) to_visit_set = set(to_visit) print("Already collected : %s" % len(visited)) while len(to_visit_queue) > 0: article = to_visit_queue.popleft() to_visit_set.remove(article)
def main(): torch.manual_seed(seed) # 设置随机种子 train_dataset = SpeakerTrainDataset() # 设置训练集读取 n_classes = train_dataset.n_classes # 说话人数 batch_sampler = BalancedBatchSampler(train_dataset.labels, train_dataset.count, spks_per_batch, utts_per_spk) print('Num of classes: {}'.format(n_classes)) encoder = Encoder(expansion, blocks, embedding_dim).to(device) generator = Generator(expansion, blocks, embedding_dim, FEATURE_LEN, latent_dim).to(device) discriminator = Discriminator(expansion, blocks, embedding_dim).to(device) classifier = Classifier(expansion, blocks, n_classes).to(device) if optimizer == 'sgd': # 优化器使用sgd generator_optimizer = optim.SGD([{ 'params': generator.parameters() }, { 'params': encoder.parameters() }, { 'params': discriminator.parameters() }], lr=lr, momentum=momentum, weight_decay=weight_decay) discriminator_optimizer = optim.SGD( [{ 'params': discriminator.parameters() }, { 'params': classifier.parameters() }], lr=lr, momentum=momentum, weight_decay=weight_decay) elif optimizer == 'adagrad': # 优化器使用adagrad generator_optimizer = optim.Adagrad( [{ 'params': generator.parameters() }, { 'params': encoder.parameters() }, { 'params': discriminator.parameters() }], lr=lr, weight_decay=weight_decay) discriminator_optimizer = optim.Adagrad( [{ 'params': discriminator.parameters() }, { 'params': classifier.parameters() }], lr=lr, weight_decay=weight_decay) else: # 优化器使用adam generator_optimizer = optim.Adam([{ 'params': generator.parameters() }, { 'params': encoder.parameters() }, { 'params': discriminator.parameters() }], lr=lr, weight_decay=weight_decay) discriminator_optimizer = optim.Adam( [{ 'params': discriminator.parameters() }, { 'params': classifier.parameters() }], lr=lr, weight_decay=weight_decay) gen_scheduler = lr_scheduler.StepLR(generator_optimizer, 200, gamma=0.1, last_epoch=-1) dis_scheduler = lr_scheduler.StepLR(discriminator_optimizer, 200, gamma=0.1, last_epoch=-1) selector = RandomNegativeTripletSelector(margin) triplet_criterion = OnlineTripletLoss(margin, selector).to(device) bce_criterion = nn.BCELoss().to(device) softmax_criterion = nn.CrossEntropyLoss().to(device) start = 1 metric = AverageNonzeroTripletsMetric() if resume: # 是否从之前保存的模型开始 if os.path.isfile(os.path.join(final_dir, "net.pth")): print('=> loading checkpoint {}'.format( os.path.join(final_dir, "net.pth"))) checkpoint = torch.load(os.path.join(final_dir, "net.pth")) start = checkpoint['epoch'] + 1 if load_optimizer: generator_optimizer.load_state_dict( checkpoint['generator_optimizer']) discriminator_optimizer.load_state_dict( checkpoint['discriminator_optimizer']) generator.load_state_dict(checkpoint['generator_state_dict']) discriminator.load_state_dict( checkpoint['discriminator_state_dict']) else: print('=> no checkpoint found at {}'.format( os.path.join(final_dir, "net.pth"))) train_loader = DataLoader(train_dataset, batch_sampler=batch_sampler, num_workers=8, pin_memory=True) test_dataset = SpeakerTestDataset() test_loader = DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=8, pin_memory=True) for epoch in range(start, epochs + 1): real_data, fake_fbank = train(epoch, encoder, generator, discriminator, classifier, triplet_criterion, bce_criterion, softmax_criterion, generator_optimizer, discriminator_optimizer, train_loader, metric) save_image(torch.Tensor(random.sample(real_data.tolist(), 4)), 'real_fbank/{}.png'.format(epoch), nrow=2, normalize=True) save_image(torch.Tensor(random.sample(fake_fbank.tolist(), 4)), 'fake_fbank/{}.png'.format(epoch), nrow=2, normalize=True) test(encoder, test_loader) # 测试 gen_scheduler.step() dis_scheduler.step() task = pd.read_csv(TRIAL_FILE, header=None, delimiter='[,]', engine='python') pred = pd.read_csv(final_dir + prediction, engine='python') y_true = np.array(task.iloc[:, 0]) y_pred = np.array(pred.iloc[:, -1]) eer, thresh = cal_eer(y_true, y_pred) logger.log_value('eer', eer, epoch) print('EER : {:.3%}'.format(eer)) print('Threshold: {:.5f}'.format(thresh))
import tweepy import pprint from PIL import Image import numpy as np import urllib.request import io from model import Classifier pp = pprint.PrettyPrinter(width=41, compact=True) classifier = Classifier() # credentials.py from credentials import * auth = tweepy.OAuthHandler(CONSUMER_TOKEN, CONSUMER_SECRET) auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET) api = tweepy.API(auth) # override tweepy.StreamListener to add logic to on_status class MyStreamListener(tweepy.StreamListener): def on_status(self, status): print(status.text) if status.in_reply_to_screen_name == api.me().screen_name: pp.pprint(status) if 'media' in status.entities: url = status.entities['media'][0]['media_url'] with urllib.request.urlopen(url) as u: f = io.BytesIO(u.read()) image = Image.open(f) # image = image.thumbnail((28, 28))
validation_mask = validation_input['input_mask'] validation_labels = validation_input['labels'] print( "Validation input shape: input_word_ids=>{}, input_mask=>{}, labels=>{}" .format(validation_word_ids.shape, validation_mask.shape, validation_labels.shape)) # Load the input data train_dataloader, validation_dataloader = data.get_data_loader( train_input, validation_input, args.batch_size) # Build the model by passing in the input params model_class = XLMRobertaForSequenceClassification model = Classifier(model_class, args.model_name, num_labels=args.num_labels, output_attentions=False, output_hidden_states=False) # Send the model to the device model.to(device) # Train the model train_losses, train_accuracies, validation_losses, validation_accuracies = train_fn( model, train_dataloader, validation_dataloader, args.epochs, args.lr, device, args.best_model_path, args.torch_manual_seed,
def learnAndSaveModel(filter_size, filter_depth, layer_num): #data set properties batch_size = 100 validation_split = .2 shuffle_dataset = True random_seed = 42 #load flat and normalize image dataset = Dataset(filter_size, filter_depth) # Creating data indices for training and validation splits: dataset_size = len(dataset) indices = list(range(dataset_size)) split = int(np.floor(validation_split * dataset_size)) if shuffle_dataset: np.random.seed(random_seed) np.random.shuffle(indices) train_indices, val_indices = indices[split:], indices[:split] # Creating PT data samplers and loaders: train_sampler = SubsetRandomSampler(train_indices) valid_sampler = SubsetRandomSampler(val_indices) #Loading datasets train, validation, test train_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=train_sampler) validation_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=valid_sampler) test_loader = torch.utils.data.DataLoader(dataset, batch_size=100, shuffle=True) #Loading model, and optimizer model = Classifier(layer_num) optimizer = optim.SGD(model.parameters(), lr=0.01) criterion = nn.NLLLoss() #Stop conition values last_min = 300 last_min_idx = -1 #Learning loop max_num_epochs = 20000 for x in range(0, max_num_epochs): running_loss = 0 for images, labels in train_loader: features = images.view(images.shape[0], -1) logps = model.forward(features) loss = criterion(logps, labels) optimizer.zero_grad() loss.backward() optimizer.step() with torch.set_grad_enabled(False): for images, labels in validation_loader: # Load image features = images.view(images.shape[0], -1) # Model computations logps = model.forward(features) loss = criterion(logps, labels) running_loss += loss.item() if loss.item() < last_min: last_min = loss.item() last_min_idx = x else: print(f"Running loss: {running_loss/len(validation_loader)}") #Stop condition if x - last_min_idx > 15: print(f'Epochs {x}') break #saving model torch.save(model.state_dict(), model_output_filename(filter_size, filter_depth, layer_num)) running_loss = 0 for images, labels in test_loader: # Transfer to GPU features = images.view(images.shape[0], -1) # Model computations logps = model.forward(features) loss = criterion(logps, labels) running_loss += loss.item() else: print(f"Final cost is: {running_loss/len(test_loader)}")
for i, (word_id, pos_id, label) in enumerate(valid_loader): begin = time.time() hidden = model.init_hidden(BATCH_SIZE) pred, attention = model(word_id, pos_id, hidden) loss = criterion(pred, label) losses.update(loss.item()) end = time.time() times.update(end-begin) print('epoch %d, %d/%d, validation loss: %f, time estimated: %.2f seconds'%(epoch, i+1,len(valid_loader),losses.avg, times.avg*len(valid_loader)), end='\r') print("\n") return losses.avg BATCH_SIZE = 50 cuda = torch.device('cuda:1') model = Classifier(config) model.encoder.bilstm.word_embedding.weight.requires_grad = True model.encoder.bilstm.pos_embedding.weight.requires_grad = True model.to(cuda) criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=0.001, betas=[0.9, 0.999], eps=1e-8, weight_decay=0) train_loader = DataLoader(Yelp_Dataset('train',cuda),batch_size=BATCH_SIZE,shuffle=True,num_workers=0) valid_loader = DataLoader(Yelp_Dataset('dev',cuda),batch_size=BATCH_SIZE,shuffle=True,num_workers=0) min_loss = float('inf') count = 0 for epoch in range(0,50): train(model, train_loader, optimizer, epoch) valid_loss = valid(model, valid_loader, optimizer, epoch) if valid_loss<min_loss: count = 0 min_loss = valid_loss
class Training(object): def __init__(self, config, logger=None): if logger is None: logger = logging.getLogger('logger') logger.setLevel(logging.DEBUG) logging.basicConfig(format='%(message)s', level=logging.DEBUG) self.logger = logger self.config = config self.classes = list(config.id2label.keys()) self.num_classes = config.num_classes self.embedder = Embedder(self.config) self.encoder = LSTMEncoder(self.config) self.clf = Classifier(self.config) self.clf_loss = SequenceCriteria(class_weight=None) if self.config.lambda_ae > 0: self.ae = AEModel(self.config) self.writer = SummaryWriter(log_dir="TFBoardSummary") self.global_steps = 0 self.enc_clf_opt = Adam(self._get_trainabe_modules(), lr=self.config.lr, betas=(config.beta1, config.beta2), weight_decay=config.weight_decay, eps=config.eps) if config.scheduler == "ReduceLROnPlateau": self.scheduler = lr_scheduler.ReduceLROnPlateau( self.enc_clf_opt, mode='max', factor=config.lr_decay, patience=config.patience, verbose=True) elif config.scheduler == "ExponentialLR": self.scheduler = lr_scheduler.ExponentialLR(self.enc_clf_opt, gamma=config.gamma) self._init_or_load_model() if config.multi_gpu: self.embedder.cuda() self.encoder.cuda() self.clf.cuda() self.clf_loss.cuda() if self.config.lambda_ae > 0: self.ae.cuda() self.ema_embedder = ExponentialMovingAverage(decay=0.999) self.ema_embedder.register(self.embedder.state_dict()) self.ema_encoder = ExponentialMovingAverage(decay=0.999) self.ema_encoder.register(self.encoder.state_dict()) self.ema_clf = ExponentialMovingAverage(decay=0.999) self.ema_clf.register(self.clf.state_dict()) self.time_s = time() def _get_trainabe_modules(self): param_list = list(self.embedder.parameters()) + \ list(self.encoder.parameters()) + \ list(self.clf.parameters()) if self.config.lambda_ae > 0: param_list += list(self.ae.parameters()) return param_list def _get_l2_norm_loss(self): total_norm = 0. for p in self._get_trainabe_modules(): param_norm = p.data.norm(p=2) total_norm += param_norm # ** 2 return total_norm # / 2. def _init_or_load_model(self): # if not self._load_model(): ensure_directory(self.config.output_path) self.epoch = 0 self.best_accuracy = -np.inf def _compute_vocab_freq(self, train_, dev_): counter = collections.Counter() for _, ids_ in train_: counter.update(ids_) for _, ids_ in dev_: counter.update(ids_) word_freq = np.zeros(self.config.n_vocab) for index_, freq_ in counter.items(): word_freq[index_] = freq_ return torch.from_numpy(word_freq).type(batch_utils.FLOAT_TYPE) def _save_model(self): state = { 'epoch': self.epoch, 'state_dict_encoder': self.ema_encoder.shadow_variable_dict, # self.encoder.state_dict(), 'state_dict_embedder': self.ema_embedder.shadow_variable_dict, # self.embedder.state_dict(), 'state_dict_clf': self.ema_clf.shadow_variable_dict, # self.clf.state_dict(), 'best_accuracy': self.best_accuracy } torch.save( state, os.path.join(self.config.output_path, self.config.model_file)) def _load_model(self): checkpoint_path = os.path.join(self.config.output_path, self.config.model_file) if self.config.load_checkpoint and os.path.isfile(checkpoint_path): # Code taken from here: https://github.com/pytorch/examples/blob/master/imagenet/main.py dict_ = torch.load(checkpoint_path) self.epoch = dict_['epoch'] self.best_accuracy = dict_['best_accuracy'] self.embedder.load_state_dict(dict_['state_dict_embedder']) self.encoder.load_state_dict(dict_['state_dict_encoder']) self.clf.load_state_dict(dict_['state_dict_clf']) self.logger.info("=> loaded checkpoint '{}' (epoch {})".format( checkpoint_path, self.epoch)) return True def __call__(self, train, dev, test, unlabel, addn, addn_un, addn_test): self.logger.info('Start training') self._train(train, dev, unlabel, addn, addn_un, addn_test) self._evaluate(test, addn_test) def _create_iter(self, data_, wbatchsize, random_shuffler=None): iter_ = data.iterator.pool(data_, wbatchsize, key=lambda x: len(x[1]), batch_size_fn=batch_size_fn, random_shuffler=None) return iter_ def _run_epoch(self, train_data, dev_data, unlabel_data, addn_data, addn_data_unlab, addn_dev): addn_dev.cuda() report_stats = utils.Statistics() cm = ConfusionMatrix(self.classes) _, seq_data = list(zip(*train_data)) total_seq_words = len(list(itertools.chain.from_iterable(seq_data))) iter_per_epoch = (1.5 * total_seq_words) // self.config.wbatchsize self.encoder.train() self.clf.train() self.embedder.train() # print(addn_data) # print(addn_data.shape) # print(train_data[:5]) train_iter = self._create_iter(train_data, self.config.wbatchsize) # addn_iter = self._create_iter(addn_data, self.config.wbatchsize) # train_iter = self._create_iter(zip(train_data, addn_data), self.config.wbatchsize) unlabel_iter = self._create_iter(unlabel_data, self.config.wbatchsize_unlabel) sofar = 0 sofar_1 = 0 for batch_index, train_batch_raw in enumerate(train_iter): seq_iter = list(zip(*train_batch_raw))[1] seq_words = len(list(itertools.chain.from_iterable(seq_iter))) report_stats.n_words += seq_words self.global_steps += 1 # self.enc_clf_opt.zero_grad() if self.config.add_noise: train_batch_raw = add_noise(train_batch_raw, self.config.noise_dropout, self.config.random_permutation) train_batch = batch_utils.seq_pad_concat(train_batch_raw, -1) # print(train_batch.shape) train_embedded = self.embedder(train_batch) memory_bank_train, enc_final_train = self.encoder( train_embedded, train_batch) if self.config.lambda_vat > 0 or self.config.lambda_ae > 0 or self.config.lambda_entropy: try: unlabel_batch_raw = next(unlabel_iter) except StopIteration: unlabel_iter = self._create_iter( unlabel_data, self.config.wbatchsize_unlabel) unlabel_batch_raw = next(unlabel_iter) if self.config.add_noise: unlabel_batch_raw = add_noise( unlabel_batch_raw, self.config.noise_dropout, self.config.random_permutation) unlabel_batch = batch_utils.seq_pad_concat( unlabel_batch_raw, -1) unlabel_embedded = self.embedder(unlabel_batch) memory_bank_unlabel, enc_final_unlabel = self.encoder( unlabel_embedded, unlabel_batch) # print(memory_bank_unlabel.shape[0]) addn_batch_unlab = retAddnBatch(addn_data_unlab, memory_bank_unlabel.shape[0], sofar_1).cuda() sofar_1 += addn_batch_unlab.shape[0] # print(addn_batch_unlab.shape) # print(memory_bank_train.shape[0]) addn_batch = retAddnBatch(addn_data, memory_bank_train.shape[0], sofar).cuda() sofar += addn_batch.shape[0] # print(addn_batch.shape) pred = self.clf(memory_bank_train, addn_batch, enc_final_train) # print(pred) accuracy = self.get_accuracy(cm, pred.data, train_batch.labels.data) lclf = self.clf_loss(pred, train_batch.labels) lat = Variable( torch.FloatTensor([-1.]).type(batch_utils.FLOAT_TYPE)) lvat = Variable( torch.FloatTensor([-1.]).type(batch_utils.FLOAT_TYPE)) if self.config.lambda_at > 0: lat = at_loss( self.embedder, self.encoder, self.clf, train_batch, addn_batch, perturb_norm_length=self.config.perturb_norm_length) if self.config.lambda_vat > 0: lvat_train = vat_loss( self.embedder, self.encoder, self.clf, train_batch, addn_batch, p_logit=pred, perturb_norm_length=self.config.perturb_norm_length) if self.config.inc_unlabeled_loss: lvat_unlabel = vat_loss( self.embedder, self.encoder, self.clf, unlabel_batch, addn_batch_unlab, p_logit=self.clf(memory_bank_unlabel, addn_batch_unlab, enc_final_unlabel), perturb_norm_length=self.config.perturb_norm_length) if self.config.unlabeled_loss_type == "AvgTrainUnlabel": lvat = 0.5 * (lvat_train + lvat_unlabel) elif self.config.unlabeled_loss_type == "Unlabel": lvat = lvat_unlabel else: lvat = lvat_train lentropy = Variable( torch.FloatTensor([-1.]).type(batch_utils.FLOAT_TYPE)) if self.config.lambda_entropy > 0: lentropy_train = entropy_loss(pred) if self.config.inc_unlabeled_loss: lentropy_unlabel = entropy_loss( self.clf(memory_bank_unlabel, addn_batch_unlab, enc_final_unlabel)) if self.config.unlabeled_loss_type == "AvgTrainUnlabel": lentropy = 0.5 * (lentropy_train + lentropy_unlabel) elif self.config.unlabeled_loss_type == "Unlabel": lentropy = lentropy_unlabel else: lentropy = lentropy_train lae = Variable( torch.FloatTensor([-1.]).type(batch_utils.FLOAT_TYPE)) if self.config.lambda_ae > 0: lae = self.ae(memory_bank_unlabel, enc_final_unlabel, unlabel_batch.sent_len, unlabel_batch_raw) ltotal = (self.config.lambda_clf * lclf) + \ (self.config.lambda_ae * lae) + \ (self.config.lambda_at * lat) + \ (self.config.lambda_vat * lvat) + \ (self.config.lambda_entropy * lentropy) report_stats.clf_loss += lclf.data.cpu().numpy() report_stats.at_loss += lat.data.cpu().numpy() report_stats.vat_loss += lvat.data.cpu().numpy() report_stats.ae_loss += lae.data.cpu().numpy() report_stats.entropy_loss += lentropy.data.cpu().numpy() report_stats.n_sent += len(pred) report_stats.n_correct += accuracy self.enc_clf_opt.zero_grad() ltotal.backward() params_list = self._get_trainabe_modules() # Excluding embedder form norm constraint when AT or VAT if not self.config.normalize_embedding: params_list += list(self.embedder.parameters()) norm = torch.nn.utils.clip_grad_norm(params_list, self.config.max_norm) report_stats.grad_norm += norm self.enc_clf_opt.step() if self.config.scheduler == "ExponentialLR": self.scheduler.step() self.ema_embedder.apply(self.embedder.named_parameters()) self.ema_encoder.apply(self.encoder.named_parameters()) self.ema_clf.apply(self.clf.named_parameters()) report_func(self.epoch, batch_index, iter_per_epoch, self.time_s, report_stats, self.config.report_every, self.logger) if self.global_steps % self.config.eval_steps == 0: cm_, accuracy, prc_dev = self._run_evaluate(dev_data, addn_dev) self.logger.info( "- dev accuracy {} | best dev accuracy {} ".format( accuracy, self.best_accuracy)) self.writer.add_scalar("Dev_Accuracy", accuracy, self.global_steps) pred_, lab_ = zip(*prc_dev) pred_ = torch.cat(pred_) lab_ = torch.cat(lab_) self.writer.add_pr_curve("Dev PR-Curve", lab_, pred_, self.global_steps) pprint.pprint(cm_) pprint.pprint(cm_.get_all_metrics()) if accuracy > self.best_accuracy: self.logger.info("- new best score!") self.best_accuracy = accuracy self._save_model() if self.config.scheduler == "ReduceLROnPlateau": self.scheduler.step(accuracy) self.encoder.train() self.embedder.train() self.clf.train() if self.config.weight_decay > 0: print(">> Square Norm: %1.4f " % self._get_l2_norm_loss()) cm, train_accuracy, _ = self._run_evaluate(train_data, addn_data) self.logger.info("- Train accuracy {}".format(train_accuracy)) pprint.pprint(cm.get_all_metrics()) cm, dev_accuracy, _ = self._run_evaluate(dev_data, addn_dev) self.logger.info("- Dev accuracy {} | best dev accuracy {}".format( dev_accuracy, self.best_accuracy)) pprint.pprint(cm.get_all_metrics()) self.writer.add_scalars("Overall_Accuracy", { "Train_Accuracy": train_accuracy, "Dev_Accuracy": dev_accuracy }, self.global_steps) return dev_accuracy @staticmethod def get_accuracy(cm, output, target): batch_size = output.size(0) predictions = output.max(-1)[1].type_as(target) correct = predictions.eq(target) correct = correct.float() if not hasattr(correct, 'sum'): correct = correct.cpu() correct = correct.sum() cm.add_batch(target.cpu().numpy(), predictions.cpu().numpy()) return correct def _predict_batch(self, cm, batch, addn_batch): self.embedder.eval() self.encoder.eval() self.clf.eval() one, two = self.encoder(self.embedder(batch), batch) pred = self.clf(one, addn_batch, two) accuracy = self.get_accuracy(cm, pred.data, batch.labels.data) return pred, accuracy def chunks(self, l, n=15): """Yield successive n-sized chunks from l.""" for i in range(0, len(l), n): yield l[i:i + n] def _run_evaluate(self, test_data, addn_test): pr_curve_data = [] cm = ConfusionMatrix(self.classes) accuracy_list = [] # test_iter = self._create_iter(test_data, self.config.wbatchsize, # random_shuffler=utils.identity_fun) test_iter = self.chunks(test_data) for batch_index, test_batch in enumerate(test_iter): addn_batch = addn_test[batch_index * 15:(batch_index + 1) * 15] test_batch = batch_utils.seq_pad_concat(test_batch, -1) # print(addn_batch.shape) try: pred, acc = self._predict_batch(cm, test_batch, addn_batch) except: continue accuracy_list.append(acc) pr_curve_data.append( (F.softmax(pred, -1)[:, 1].data, test_batch.labels.data)) accuracy = 100 * (sum(accuracy_list) / len(test_data)) return cm, accuracy, pr_curve_data def _train(self, train_data, dev_data, unlabel_data, addn_data, addn_data_unlab, addn_dev): addn_data = addn_data.cuda() addn_data_unlab = addn_data_unlab.cuda() addn_dev = addn_dev.cuda() # for early stopping nepoch_no_imprv = 0 epoch_start = self.epoch + 1 epoch_end = self.epoch + self.config.nepochs + 1 for self.epoch in range(epoch_start, epoch_end): self.logger.info("Epoch {:} out of {:}".format( self.epoch, self.config.nepochs)) # random.shuffle(train_data) # random.shuffle(unlabel_data) accuracy = self._run_epoch(train_data, dev_data, unlabel_data, addn_data, addn_data_unlab, addn_dev) # early stopping and saving best parameters if accuracy > self.best_accuracy: nepoch_no_imprv = 0 self.best_accuracy = accuracy self.logger.info("- new best score!") self._save_model() else: nepoch_no_imprv += 1 if nepoch_no_imprv >= self.config.nepoch_no_imprv: self.logger.info( "- early stopping {} epochs without improvement". format(nepoch_no_imprv)) break if self.config.scheduler == "ReduceLROnPlateau": self.scheduler.step(accuracy) def _evaluate(self, test_data, addn_test): addn_test = addn_test.cuda() self.logger.info("Evaluating model over test set") self._load_model() _, accuracy, prc_test = self._run_evaluate(test_data, addn_test) pred_, lab_ = zip(*prc_test) pred_ = torch.cat(pred_).cpu().tolist() lab_ = torch.cat(lab_).cpu().tolist() path_ = os.path.join(self.config.output_path, "{}_pred_gt.tsv".format(self.config.exp_name)) with open(path_, 'w') as fp: for p, l in zip(pred_, lab_): fp.write(str(p) + '\t' + str(l) + '\n') self.logger.info("- test accuracy {}".format(accuracy))
def __init__(self, param): self.device = torch.device(param.device) ######################## LOAD DATA ############################# self.scalar = MinMaxScaler() self.trainval_set = dataloader( param, transform=self.scalar, split='trainval') # AWA2: 23527 40类 CUB:7057 150类 self.test_set_unseen = dataloader( param, transform=self.scalar, split='test_unseen') # AWA2: 7913 10类 CUB:2967 50类 self.test_set_seen = dataloader( param, transform=self.scalar, split='test_seen') # AWA2: 5882 40类 CUB:1764 150类 self.trainloader = data.DataLoader(self.trainval_set, batch_size=param.batch_size, shuffle=True) self.input_dim = self.trainval_set.__getlen__() self.atts_dim = self.trainval_set.__get_attlen__() self.num_classes = self.trainval_set.__totalClasses__() print(30 * ('-')) print("Input_dimension=%d" % self.input_dim) print("Attribute_dimension=%d" % self.atts_dim) print("z=%d" % param.latent_size) print("num_classes=%d" % self.num_classes) print(30 * ('-')) ####################### INITIALIZE THE MODEL AND OPTIMIZER ##################### self.model_encoder = encoder_cada(input_dim=self.input_dim, atts_dim=self.atts_dim, z=param.latent_size).to(self.device) self.model_decoder = decoder_cada(input_dim=self.input_dim, atts_dim=self.atts_dim, z=param.latent_size).to(self.device) learnable_params = list(self.model_encoder.parameters()) + list( self.model_decoder.parameters()) self.optimizer = optim.Adam(learnable_params, lr=0.00015, betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=True) self.classifier = Classifier(input_dim=param.latent_size, num_class=self.num_classes).to( self.device) self.cls_optimizer = optim.Adam(self.classifier.parameters(), lr=0.001, betas=(0.5, 0.999)) print(self.model_encoder) print(self.model_decoder) print(self.classifier) ################### LOAD PRETRAINED MODEL ######################## if param.pretrained: if param.model_path == '': print("Please provide the path of the pretrained model.") else: checkpoint = torch.load(param.model_path) self.model_encoder.load_state_dict( checkpoint['model_encoder_state_dict']) self.model_decoder.load_state_dict( checkpoint['model_decoder_state_dict']) print(">> Pretrained model loaded!") ########## LOSS ############ self.l1_loss = nn.L1Loss(reduction='sum') self.lossfunction_classifier = nn.NLLLoss() ######### Hyper-params ####### self.gamma = torch.zeros(1, device=self.device).float() self.beta = torch.zeros(1, device=self.device).float() self.delta = torch.zeros(1, device=self.device).float()
def main(): args = docopt(__doc__) train_embedding = not args['--no-train-embedding'] handle_foreign = args['--handle-foreign'] enable_all_pools = args['--enable-all-pools'] np.random.seed(int(args['--seed'])) torch.manual_seed(int(args['--seed'])) torch.cuda.manual_seed_all(int(args['--seed'])) hidden = int(args['--hidden']) dropout = float(args['--dropout']) batch_size = int(args['--batch']) lr = float(args['--lr']) emb_lr = float(args['--emb-lr']) weight_decay = float(args['--weight-decay']) epochs = int(args['--epochs']) device = torch.device(int(args['--device'])) print(f"{device} will be used") ratio = float(args['--ratio']) threshold = float(args['--threshold']) dname = args['--dirname'] train_dset = QueryDataset(split='train', ratio=ratio, equally_handle_foreign_authors=handle_foreign, use_paper_author=args['--use-paper-author'], oversample_false_collabs=args['--oversample-false-collabs']) valid_dset = QueryDataset(split='valid', ratio=ratio, equally_handle_foreign_authors=handle_foreign) train_loader = DataLoader(train_dset, batch_size=1, num_workers=1, shuffle=True) valid_loader = DataLoader(valid_dset, batch_size=1, num_workers=1, shuffle=False) embedding_mode, embedding = load_embedding( args['--embedding'], train_embedding, device) classifier = Classifier(embedding, hidden, dropout, args['--deepset'], equally_handle_foreign_authors=handle_foreign, enable_all_pools=enable_all_pools) if torch.cuda.is_available(): classifier.to(device) emb_params = set(embedding.parameters()) cls_params = set(classifier.parameters()).difference(emb_params) optimizer1 = optim.SparseAdam(emb_params, lr=emb_lr) optimizer2 = optim.Adam(cls_params, lr=lr, weight_decay=weight_decay) train_embedding = 'on' if train_embedding else 'off' if dname == 'None': mode = f'{classifier.savename}_emb-{embedding_mode}'\ f'_trainemb-{train_embedding}' dname = get_dirname(mode) else: os.makedirs(dname, exist_ok=True) path = os.path.join(dname, 'log.txt') with open(path, 'a') as f: f.write(repr(args) + '\n') backup_path = os.path.join(dname, 'classifier.pth') # TODO: Add checkpoint training feature pbar = tqdm(total=epochs, initial=0, bar_format="{desc:<5}{percentage:3.0f}%|{bar:10}{r_bar}") best_acc = 0 for epoch in range(epochs): avg_loss, train_acc, val_acc, precision, recall, best_model = train_classifier( train_loader, valid_loader, classifier, [optimizer1, optimizer2], device, epoch, batch_size, dname, threshold) if val_acc > best_acc: torch.save(best_model.state_dict(), backup_path) best_acc = val_acc pbar.set_description( f'Train Loss: {avg_loss:.6f}, Train Acc:{train_acc:.2f} Valid Acc: {val_acc:.2f}% Precision: {precision:.2f} Recall: {recall:.2f}') pbar.update(1)
else: train_data = utils.get_data(train_file, word_dict, category_dicts, label_dict) pickle.dump(train_data, open(train_pickle, 'wb'), protocol=4) if os.path.isfile(dev_pickle): dev_data = pickle.load(open(dev_pickle, 'rb')) else: dev_data = utils.get_data(dev_file, word_dict, category_dicts, label_dict) pickle.dump(dev_data, open(dev_pickle, 'wb'), protocol=4) category_sizes = [len(category_dict) for category_dict in category_dicts] label_size = len(label_dict) word_size = len(word_dict) model = Model(word_size, label_size, category_sizes, word_dim, embed_dim, hidden_dim, category_dim, inject_type, inject_locs, chunk_ratio=basis, basis=basis) with torch.no_grad(): model.embedding.weight.set_(torch.from_numpy(word_vectors).float()) model.cuda() optimizer = torch.optim.Adadelta(model.parameters()) #if os.path.exists(model_file): # best_point = torch.load(model_file) # model.load_state_dict(best_point['state_dict']) # optimizer.load_state_dict(best_point['optimizer']) # best_dev_acc = best_point['dev_acc'] print("Total Parameters:", sum(p.numel() for p in model.parameters())) x_train = train_data['x'] c_train = train_data['c'] y_train = train_data['y']
def train(args): num_epoch = args.epoch lr = args.learning_rate batch_size = args.batch_size weight = args.weight alpha = args.alpha # preprocessing transform = transforms.Compose([ transforms.ToPILImage(), transforms.Pad(padding=4, padding_mode="reflect"), transforms.RandomCrop((32, 32)), transforms.ColorJitter(), #transforms.RandomRotation(degrees=20), transforms.RandomHorizontalFlip(p=0.5), transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), transforms.RandomErasing(), ]) # load training dataset X_train, y_train, _, _ = utils.load_cifar10(cifar10_path) assert X_train.shape[0] == y_train.shape[0] print(f"Dataset size (train) = {y_train.shape[0]}") train_data = utils.MyDataset(x=X_train, y=y_train, transform=transform) # weighted sampling weighted_sampler = utils.set_weighted_sampler(label_data=y_train) train_loader = DataLoader(train_data, batch_size, shuffle=False, sampler=weighted_sampler, num_workers=2) # normal sampling #train_loader = DataLoader(train_data, batch_size, shuffle=True, num_workers=2) # model, optimizer, loss function model = Classifier().to(device) if len(weight) > 0: model.load_state_dict(torch.load(weight)) # optimizer optimizer = torch.optim.Adam(model.parameters(), lr=lr) scheduler = torch.optim.lr_scheduler.LambdaLR( optimizer, lr_lambda=learning_rate_schedule) # loss function cross_entropy_loss = nn.CrossEntropyLoss().to(device) mse_loss = nn.MSELoss().to(device) # train train_loss = [] for epoch in range(num_epoch): train_tmp_loss = [] for i, data in enumerate(train_loader, 0): inputs, labels = data inputs = inputs.to(device) labels = labels.to(device) optimizer.zero_grad() # forward outputs, autoencoder_outputs = model(inputs) # calc categorical cross entropy loss1 = cross_entropy_loss(outputs, labels) # calc mean squared loss loss2 = mse_loss(inputs, autoencoder_outputs) loss = loss1 + alpha * loss2 # backpropagate losses loss.backward() train_tmp_loss.append(loss.item() * len(inputs)) optimizer.step() if i % 100 == 99: print( f"epoch: {epoch+1}, batch: {i+1}, loss: {loss.item():.3f}") train_loss.append(sum(train_tmp_loss) / y_train.shape[0]) # adjust learning rate according to the schedule scheduler.step() # save parameters weight_path = pathlib.Path(f"weight/epoch_{epoch+1:03d}_{alpha}.pth") torch.save(model.state_dict(), weight_path) print(train_loss)
class Trainer_Classifier(): def __init__(self, batch_size, device, index_to_start_conditioning): #need to do train/test/val split though girl self.batch_size = batch_size self.device = device self.dataset = EmotionDatasetTiming( "../prosodyProject/processed_midi_info_edited/", 20, self.device) self.loader = DataLoader(self.dataset, batch_size=self.batch_size, shuffle=True, num_workers=4, collate_fn=pad_collate, drop_last=True) self.classifier = Classifier( 4, batch_size, device, 4, hidden_size=50, num_layers=3) #not 4+4 because no emotions given self.classifier.to(device) self.index_to_start_conditioning = index_to_start_conditioning #probably 4 self.classifier_optimizer = torch.optim.Adam( self.classifier.parameters(), lr=0.0001) self.losses = [] self.bceloss = nn.BCELoss() self.crossentropy = nn.CrossEntropyLoss() def train(self, num_epochs, save_every): for epoch in range(num_epochs): for (x_padded, x_lens) in self.loader: real_data = self.dataset.get_data_with_noise(x_padded).to( self.device) #just leave as-is for now, simpler train_data = pack_padded_sequence( real_data[:, :, :self.index_to_start_conditioning], x_lens, batch_first=True, enforce_sorted=False) emotion_info = real_data[:, 0, self.index_to_start_conditioning:] output = self.classifier(train_data) loss = self.loss_quadrant(output, emotion_info) loss.backward() self.classifier_optimizer.step() self.losses.append(loss.item()) print("last output ", output) print("loss: ", self.losses[-1]) if epoch % save_every == 0: pass #torch.save({"epoch": epoch, "model_state_dict": self.classifier.state_dict(), # "optimizer_state_dict": self.classifier_optimizer.state_dict(), # "loss": self.losses[-1]}, # "checkpoints_classifier1/classifier_checkpoint" + str(epoch) + ".pth.tar") def loss_2(self, output, emotion_info): #print("output: ", output) #print("compare to: ", emotion_info[:,0:2]) emotion_loss = torch.dist(output, emotion_info[:, 0:2]) #print("emotion_loss: ", emotion_loss) return emotion_loss def loss_quadrant(self, output, emotion_info): quadrant_gt = torch.zeros(emotion_info.size(0), dtype=torch.long).to(self.device) for i in range(emotion_info.size(0)): if emotion_info[i, 2] > 0: if emotion_info[i, 3] > 0: quadrant = 0 else: quadrant = 3 else: if emotion_info[i, 3] > 0: quadrant = 1 else: quadrant = 2 quadrant_gt[i] = quadrant #print("quadrant_gt ", quadrant_gt) return self.crossentropy(output, quadrant_gt)
from model import Classifier from dataset import preprocess_images from tensorflow_addons.image import rotate import numpy as np import tensorflow as tf (train_set, train_labels), (test_dataset, test_labels) = tf.keras.datasets.mnist.load_data() train_images = preprocess_images(train_set) test_images = preprocess_images(test_dataset) classifier = Classifier(shape=(28, 28, 1)) classifier_path = checkpoint_path = "./checkpoints/cls_nor" cls = tf.train.Checkpoint(classifier=classifier) cls_manager = tf.train.CheckpointManager(cls, classifier_path, max_to_keep=5) if cls_manager.latest_checkpoint: cls.restore(cls_manager.latest_checkpoint) print('classifier checkpoint restored!!') c_t = train_images c_l = train_labels classifier.compile( optimizer='adam', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy']) classifier.fit(c_t, c_l, epochs=30, verbose=2) test_loss, test_acc = classifier.evaluate(test_images, test_labels, verbose=2) print('\nTest accuracy:', test_acc) filePath = "./cls_nor" checkpoint_path = "./checkpoints/" + filePath ckpt_save_path = cls_manager.save() print('Saving checkpoint for epoch {} at {}'.format(1, ckpt_save_path))
import os from flask import Flask, request, make_response, send_from_directory import torch import PIL from model import Classifier, image_transform app = Flask(__name__, static_folder="static") # initialize model from path net = Classifier() net.load_state_dict(torch.load("model.th", map_location=torch.device("cpu"))) net.eval() # IMPORTANT # map of model outputs to ones we can understand class_map = { 0: "cat", 1: "dog" } # outline routes @app.route("/", methods=["GET"]) def hello_world(path): return "Hello World!" @app.route("/classify", methods=["POST"]) def classify(): output = "idk" probability = -1 if "image" in request.files: f = request.files["image"]
img = cv2.imread(os.path.join(path, file)) if label: y[i] = int(file.split("_")[0]) x[i, :, :] = cv2.resize(img,(128, 128)) else: x[i, :, :] = cv2.resize(img,(128, 128)) if label: return x, y else: return x if __name__ == '__main__': #model_dir = sys.argv[1] workspace_dir = sys.argv[1] out_dir = sys.argv[2] model = Classifier().cuda() checkpoint = torch.load('VGG_150.pt') model.load_state_dict(checkpoint) train_paths, train_labels = readfile(workspace_dir, True) # 這邊在 initialize dataset 時只丟「路徑」和「class」,之後要從 dataset 取資料時 # dataset 的 __getitem__ method 才會動態的去 load 每個路徑對應的圖片 train_set = ImgDataset(train_paths, train_labels, mode='eval') # 指定想要一起 visualize 的圖片 indices img_indices = [83, 4218, 4707, 8598] img_indices2 = [993+200+709, 993+429+250+709] images, labels = train_set.getbatch(img_indices) images2, labels2 = train_set.getbatch2(img_indices) images3, labels3 = train_set.getbatch(img_indices2)
add_log( '| end of epoch {:3d} | time: {:5.2f}s |'.format( epoch, (time.time() - epoch_start_time))) # Save model torch.save(ae_model.state_dict(), args.current_save_path + 'ae_model_params.pkl') torch.save(dis_model.state_dict(), args.current_save_path + 'dis_model_params.pkl') return if __name__ == '__main__': preparation() ae_model = get_cuda(make_model(d_vocab=args.vocab_size, N=args.num_layers_AE, d_model=args.transformer_model_size, d_ff=args.transformer_ff_size, )) dis_model = get_cuda(Classifier(latent_size=args.latent_size, output_size=args.label_size)) if args.if_load_from_checkpoint: # Load models' params from checkpoint ae_model.load_state_dict(torch.load(args.current_save_path + 'ae_model_params.pkl')) dis_model.load_state_dict(torch.load(args.current_save_path + 'dis_model_params.pkl')) else: train_iters(ae_model, dis_model) # eval_iters(ae_model, dis_model) print("Done!")
class ModelRunner: def __init__(self): self.device = torch.device( "cuda:0" if torch.cuda.is_available() else "cpu") print(f"Device is {self.device}") model_name = 'bert-base-cased' self.model_path = 'NER_CONLL2003.pt' self.tokenizer = BertTokenizer.from_pretrained(model_name) self.model = Classifier(model_name) self.loader = NERDataLoader(tokenizer=self.tokenizer) self.optimizer = optim.Adam(params=self.model.parameters(), lr=LEARNING_RATE) def train_and_save(self): for epoch in range(EPOCHS): self.train_epoch(epoch) torch.save(self.model.state_dict(), self.model_path) def load_model(self): loaded_state = torch.load( self.model_path, map_location=torch.device( "cuda" if torch.cuda.is_available() else "cpu")) self.model.load_state_dict(loaded_state) def train_epoch(self, epoch): training_loader = self.loader.train() self.model.train() for _, data in enumerate(training_loader, 0): input_data = data['ids'].to(self.device, dtype=torch.long) mask_data = data['mask'].to(self.device, dtype=torch.long) labels = data['tags'].to(self.device, dtype=torch.long) loss = self.model(input_data, mask_data, labels=labels)[0] if _ % 10 == 0: print(f'Epoch: {epoch}, Loss: {loss.item()}') self.optimizer.zero_grad() loss.backward() self.optimizer.step() def test(self): testing_loader = self.loader.test() self.model.eval() eval_loss = 0 eval_accuracy = 0 predictions, true_labels = [], [] nb_eval_steps, nb_eval_examples = 0, 0 with torch.no_grad(): for _, data in enumerate(testing_loader, 0): input_data = data['ids'].to(self.device, dtype=torch.long) mask_data = data['mask'].to(self.device, dtype=torch.long) labels = data['tags'].to(self.device, dtype=torch.long) output = self.model(input_data, mask_data, labels=labels) loss, logits = output[:2] logits = logits.detach().cpu().numpy() label_ids = labels.to('cpu').numpy() predictions.extend( [list(p) for p in np.argmax(logits, axis=2)]) true_labels.append(label_ids) accuracy = self.flat_accuracy(logits, label_ids) eval_loss += loss.mean().item() eval_accuracy += accuracy nb_eval_examples += input_data.size(0) nb_eval_steps += 1 eval_loss = eval_loss / nb_eval_steps print("Validation loss: {}".format(eval_loss)) print("Validation Accuracy: {}".format(eval_accuracy / nb_eval_steps)) pred_tags = [TAGS[p_i] for p in predictions for p_i in p] valid_tags = [ TAGS[l_ii] for l in true_labels for l_i in l for l_ii in l_i ] clean_pred_tags, clean_valid_tags = self.remove_none_tags( pred_tags, valid_tags) print("Micro F1-Score: {}".format( f1_score(clean_pred_tags, clean_valid_tags, average='micro'))) print("Macro F1-Score: {}".format( f1_score(clean_pred_tags, clean_valid_tags, average='macro'))) def remove_none_tags(self, pred_tags, valid_tags): return zip(*[(pred, valid) for pred, valid in zip(pred_tags, valid_tags) if "None" != valid]) def flat_accuracy(self, preds, labels): flat_preds = np.argmax(preds, axis=2).flatten() flat_labels = labels.flatten() return np.sum(flat_preds == flat_labels) / len(flat_labels)
from model import Classifier from prepare_data import get_loaders import tensorflow as tf data_dir = "../data" batch_size = 32 image_size = (512, 1024, 3) phase = "test" image_URL = "../data/valid/handwritten/00002.png" class_dict = {0: "handwritten", 1: "printed"} model = Classifier() if phase == "train": model.compile( loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"] ) model.build((None,) + image_size) model.summary() train_loader, val_loader = get_loaders(data_dir, "train", image_size[:2], batch_size)
batch_size=batch_size, shuffle=shuffle, num_workers=num_workers, drop_last=True) t_loader_raw = torch.utils.data.DataLoader(t_set, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers, drop_last=True) t_loader_test = torch.utils.data.DataLoader(t_set_test, batch_size=batch_size, shuffle=False, num_workers=num_workers) extractor = Extractor() s1_classifier = Classifier(num_classes=num_classes) s2_classifier = Classifier(num_classes=num_classes) s3_classifier = Classifier(num_classes=num_classes) s1_t_discriminator = Discriminator() s2_t_discriminator = Discriminator() s3_t_discriminator = Discriminator() extractor.load_state_dict( torch.load( osp.join( MAIN_DIR, "MSDA/A_W_2_D_Open/bvlc_A_W_2_D/pretrain/bvlc_extractor.pth"))) extractor = nn.DataParallel(extractor) extractor = extractor.cuda() s1_classifier.load_state_dict(
config.NNBATCHSIZE, shuffle=True, num_workers=16) valid_dataset = IonDataset( train[val_index], train_tr[val_index], seq_len=config.GROUP_BATCH_SIZE, flip=False, ) valid_dataloader = DataLoader(valid_dataset, config.NNBATCHSIZE, shuffle=False) it = 0 model = Classifier() model = model.to(DEVICE) early_stopping = EarlyStopping( patience=40, is_maximize=True, checkpoint_path=os.path.join( STORE_DIR, f"gru_clean_checkpoint_fold_{index}_iter_{it}.pt"), ) weight = None # cal_weights() criterion = nn.CrossEntropyLoss(weight=weight) optimizer = torch.optim.Adam(model.parameters(), lr=config.LR) # base_opt = torch.optim.SGD(model.parameters(), lr=0.0015) # optimizer = torchcontrib.optim.SWA(base_opt, swa_start=10, swa_freq=5, swa_lr=0.05)
class Gzsl_vae(): """docstring for Gzsl_vae""" def __init__(self, param): self.device = torch.device(param.device) ######################## LOAD DATA ############################# self.scalar = MinMaxScaler() self.trainval_set = dataloader( param, transform=self.scalar, split='trainval') # AWA2: 23527 40类 CUB:7057 150类 self.test_set_unseen = dataloader( param, transform=self.scalar, split='test_unseen') # AWA2: 7913 10类 CUB:2967 50类 self.test_set_seen = dataloader( param, transform=self.scalar, split='test_seen') # AWA2: 5882 40类 CUB:1764 150类 self.trainloader = data.DataLoader(self.trainval_set, batch_size=param.batch_size, shuffle=True) self.input_dim = self.trainval_set.__getlen__() self.atts_dim = self.trainval_set.__get_attlen__() self.num_classes = self.trainval_set.__totalClasses__() print(30 * ('-')) print("Input_dimension=%d" % self.input_dim) print("Attribute_dimension=%d" % self.atts_dim) print("z=%d" % param.latent_size) print("num_classes=%d" % self.num_classes) print(30 * ('-')) ####################### INITIALIZE THE MODEL AND OPTIMIZER ##################### self.model_encoder = encoder_cada(input_dim=self.input_dim, atts_dim=self.atts_dim, z=param.latent_size).to(self.device) self.model_decoder = decoder_cada(input_dim=self.input_dim, atts_dim=self.atts_dim, z=param.latent_size).to(self.device) learnable_params = list(self.model_encoder.parameters()) + list( self.model_decoder.parameters()) self.optimizer = optim.Adam(learnable_params, lr=0.00015, betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=True) self.classifier = Classifier(input_dim=param.latent_size, num_class=self.num_classes).to( self.device) self.cls_optimizer = optim.Adam(self.classifier.parameters(), lr=0.001, betas=(0.5, 0.999)) print(self.model_encoder) print(self.model_decoder) print(self.classifier) ################### LOAD PRETRAINED MODEL ######################## if param.pretrained: if param.model_path == '': print("Please provide the path of the pretrained model.") else: checkpoint = torch.load(param.model_path) self.model_encoder.load_state_dict( checkpoint['model_encoder_state_dict']) self.model_decoder.load_state_dict( checkpoint['model_decoder_state_dict']) print(">> Pretrained model loaded!") ########## LOSS ############ self.l1_loss = nn.L1Loss(reduction='sum') self.lossfunction_classifier = nn.NLLLoss() ######### Hyper-params ####### self.gamma = torch.zeros(1, device=self.device).float() self.beta = torch.zeros(1, device=self.device).float() self.delta = torch.zeros(1, device=self.device).float() def train(self, epoch): ''' This function trains the cada_vae model ''' if epoch > 5 and epoch < 21: self.delta += 0.54 if epoch > 20 and epoch < 76: self.gamma += 0.044 if epoch < 93: self.beta += 0.0026 trainbar = tqdm(self.trainloader) self.model_encoder.train() self.model_decoder.train() train_loss = 0 for batch_idx, (x, y, sig) in enumerate(trainbar): x.requires_grad = False sig.requires_grad = False z_img, z_sig, mu_x, logvar_x, mu_sig, logvar_sig = self.model_encoder( x, sig) recon_x, recon_sig, sigDecoder_x, xDecoder_sig = self.model_decoder( z_img, z_sig) # loss vae_reconstruction_loss = self.l1_loss(recon_x, x) + self.l1_loss( recon_sig, sig) cross_reconstruction_loss = self.l1_loss( xDecoder_sig, x) + self.l1_loss(sigDecoder_x, sig) KLD_loss = ( 0.5 * torch.sum(1 + logvar_x - mu_x.pow(2) - logvar_x.exp()) ) + (0.5 * torch.sum(1 + logvar_sig - mu_sig.pow(2) - logvar_sig.exp())) distributed_loss = torch.sqrt( torch.sum((mu_x - mu_sig)**2, dim=1) + torch.sum((torch.sqrt(logvar_x.exp()) - torch.sqrt(logvar_sig.exp()))**2, dim=1)) distributed_loss = distributed_loss.sum() self.optimizer.zero_grad() loss = vae_reconstruction_loss - self.beta * KLD_loss if self.delta > 0: loss += distributed_loss * self.delta if self.gamma > 0: loss += cross_reconstruction_loss * self.gamma loss.backward() self.optimizer.step() train_loss += loss.item() trainbar.set_description('l:%.3f' % (train_loss / (batch_idx + 1))) #print("vae %f, da %f, ca %f"%(vae,da,ca)) print(train_loss / (batch_idx + 1)) if epoch % 100 == 0: if not os.path.exists('./checkpoints'): os.makedirs('./checkpoints') name = "./checkpoints/" + "checkpoint_cada_" + opt.dataset_name + ".pth" torch.save( { 'epoch': epoch, 'model_encoder_state_dict': self.model_encoder.state_dict(), 'model_decoder_state_dict': self.model_decoder.state_dict(), 'optimizer_state_dict': self.optimizer.state_dict(), 'loss': loss, }, name) ##################### FEATURE EXTRCTION ####################### def extract_features(self, params): print(30 * '-') print("Preparing dataset for the classifier..") self.model_encoder.eval() self.model_decoder.eval() img_seen_feats = params['img_seen'] # 200 img_unseen_feats = params['img_unseen'] # 0 att_seen_feats = params['att_seen'] # 0 att_unseen_feats = params['att_unseen'] # 400 seen_classes = self.trainval_set.__NumClasses__() # 可见类150类 unseen_classes = self.test_set_unseen.__NumClasses__() # 不可见类50类 #atts for unseen classes attribute_vector_unseen, labels_unseen = self.test_set_unseen.__attributeVector__( ) # shape:50*312 50 #for trainval features: features_seen = [] labels_seen = [] k = 0 for n in seen_classes: # 150类,每一类200张 perclass_feats = self.trainval_set.__get_perclass_feats__( n) # n=1, perclass_feats:45*2048 n=2,perclass_feats:38*2048 k += perclass_feats.shape[0] repeat_factor = math.ceil( img_seen_feats / perclass_feats.shape[0] ) # 向上取整 n=1, repeat_factor=5 n=2,repeat_factor=6 perclass_X = np.repeat( perclass_feats.cpu().numpy(), repeat_factor, axis=0 ) # n=1, 225*2048 n=2 228*2048 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!为什么复制重复的 perclass_labels = torch.from_numpy( np.repeat(n.cpu().numpy(), img_seen_feats, axis=0)).long() # 200 seen_feats = perclass_X[:img_seen_feats].astype(np.float32) # if seen_feats.shape[0] < 200: # print(n,"-------", seen_feats.shape) features_seen.append(torch.from_numpy(seen_feats)) labels_seen.append(perclass_labels) print("Number of seen features:", k) tensor_seen_features = torch.cat(features_seen).to( self.device) # 30000*2048 tensor_seen_feats_labels = torch.cat(labels_seen) tensor_unseen_attributes = torch.from_numpy( # 20000*312 np.repeat(attribute_vector_unseen, att_unseen_feats, axis=0)).float().to(self.device) # 复制400次:50*400 tensor_unseen_labels = torch.from_numpy( np.repeat(labels_unseen, att_unseen_feats, axis=0)).long() # 20000 test_unseen_X, test_unseen_Y = self.test_set_unseen.__Test_Features_Labels__( ) # 2967*2048 2967*1 test_seen_X, test_seen_Y = self.test_set_seen.__Test_Features_Labels__( ) # 1764*2048 1764*1 with torch.no_grad(): z_img, z_att, mu_x, logvar_x, mu_att, logvar_att = self.model_encoder( tensor_seen_features, tensor_unseen_attributes ) # 括号内的30000*2048,20000*312!!!!!!!!!!!!!!!!!!!!!!! z_unseen_test_img, z_unseen_test_att, mu_x_unseen, logvar_x, mu_att, logvar_att = self.model_encoder( test_unseen_X, tensor_unseen_attributes) # 括号内的:2967*2048,20000*312 z_seen_test_img, z_unseen_test_att, mu_x_seen, logvar_x, mu_att, logvar_att = self.model_encoder( test_seen_X, tensor_unseen_attributes) # 1764*2048, 20000*312 train_features = torch.cat((z_att, z_img)) # 50000*64 train_labels = torch.cat( (tensor_unseen_labels, tensor_seen_feats_labels)) # 50000 test_unseen_Y = torch.squeeze(test_unseen_Y) # 2967 test_seen_Y = torch.squeeze(test_seen_Y) # 1764 print( ">> Extraction of trainval, test seen, and test unseen features are complete!" ) print(train_features.shape, train_labels.shape) #return train_features, train_labels, z_unseen_test_img, test_unseen_Y, z_seen_test_img, test_seen_Y return train_features, train_labels, mu_x_unseen, test_unseen_Y, mu_x_seen, test_seen_Y ##################### TRAINING THE CLASSIFIER ####################### def train_classifier(self, epochs): train_features, train_labels, test_unseen_features, test_unseen_labels, test_seen_features, test_seen_labels = \ self.extract_features(params) if not os.path.exists('./datas/' + opt.dataset_name + '_features'): os.makedirs('./datas/' + opt.dataset_name + '_features') np.save( './datas/' + opt.dataset_name + '_features/' + 'test_novel_Y.npy', test_unseen_labels.cpu().numpy()) self.cls_trainData = classifier_dataloader( features_img=train_features, labels=train_labels, device=self.device) # 50000*64 self.cls_trainloader = data.DataLoader(self.cls_trainData, batch_size=32, shuffle=True) self.cls_test_unseen = classifier_dataloader( features_img=test_unseen_features, labels=test_unseen_labels, device=self.device) # test_unseen_features为VAE的均值 self.cls_test_unseenLoader = data.DataLoader(self.cls_test_unseen, batch_size=32, shuffle=False) self.test_unseen_target_classes = self.cls_test_unseen.__targetClasses__( ) self.cls_test_seen = classifier_dataloader( features_img=test_seen_features, labels=test_seen_labels, device=self.device) # test_seen_features为VAE的均值 self.cls_test_seenLoader = data.DataLoader(self.cls_test_seen, batch_size=32, shuffle=False) self.test_seen_target_classes = self.cls_test_seen.__targetClasses__() def val_gzsl(testbar_cls): with torch.no_grad(): self.classifier.eval() print("**Validation**") preds = [] target = [] for batch_idx, (x, y) in enumerate(testbar_cls): output = self.classifier(x) output_data = torch.argmax(output.data, 1) preds.append(output_data) target.append(y) predictions = torch.cat(preds) targets = torch.cat(target) return predictions, targets best_H = -1 best_seen = 0 best_unseen = 0 ############## TRAINING #################### for epoch in range(1, epochs + 1): # 1-41 print("Training: Epoch - ", epoch) self.classifier.train() trainbar_cls = tqdm( self.cls_trainloader) # cls_trainloader由50000*64的数据构成 train_loss = 0 for batch_idx, (x, y) in enumerate(trainbar_cls): # y从0开始 output = self.classifier(x.to(opt.device)) loss = self.lossfunction_classifier(output, y) self.cls_optimizer.zero_grad() loss.backward() self.cls_optimizer.step() train_loss += loss.item() trainbar_cls.set_description('l:%.3f' % (train_loss / (batch_idx + 1))) ########## VALIDATION ################## accu_unseen = 0 accu_seen = 0 testbar_cls_unseen = tqdm( self.cls_test_unseenLoader) # 由2967*64的数据构成 testbar_cls_seen = tqdm(self.cls_test_seenLoader) # 由1764*64的数据构成 preds_unseen, target_unseen = val_gzsl(testbar_cls_unseen) preds_seen, target_seen = val_gzsl(testbar_cls_seen) ########## ACCURACY METRIC ################## def compute_per_class_acc_gzsl(test_label, predicted_label, target_classes): per_class_accuracies = torch.zeros( target_classes.shape[0]).float() predicted_label = predicted_label for i in range(target_classes.shape[0]): is_class = test_label == target_classes[i] # 找出是该类的 per_class_accuracies[i] = torch.div( (predicted_label[is_class] == test_label[is_class] ).sum().float(), is_class.sum().float()) return per_class_accuracies.mean() ################################## ''' For NLLL loss the labels are mapped from 0-n, map them back to 1-n for calculating accuracies. ''' target_unseen = target_unseen + 1 preds_unseen = preds_unseen + 1 target_seen = target_seen + 1 preds_seen = preds_seen + 1 ################################## accu_unseen = compute_per_class_acc_gzsl( target_unseen, preds_unseen, self.test_unseen_target_classes) accu_seen = compute_per_class_acc_gzsl( target_seen, preds_seen, self.test_seen_target_classes) if (accu_seen + accu_unseen) > 0: H = (2 * accu_seen * accu_unseen) / (accu_seen + accu_unseen) else: H = 0 if H > best_H: best_seen = accu_seen best_unseen = accu_unseen best_H = H print(30 * '-') print('Epoch:', epoch) print('Best: u, s, h =%.4f,%.4f,%.4f' % (best_unseen, best_seen, best_H)) print('u, s, h =%.4f,%.4f,%.4f' % (accu_unseen, accu_seen, H)) print(30 * '-') return best_seen, best_unseen, best_H
trainDataRoot, transform=input_transform_tr), batch_size=batchSize, shuffle=True, num_workers=4) valloader = data.DataLoader(datasets.ImageFolder( "./data/Classification/test", transform=input_transform), batch_size=batchSize, shuffle=True, num_workers=4) numClass = 4 numFeat = 32 dropout = 0.25 modelConv = DownSampler(numFeat, False, dropout) modelClass = Classifier(numFeat * 2, numClass, 4) modelHess = BNNL() if hessMC: modelHess = BNNMC() weights = torch.ones(numClass) if torch.cuda.is_available(): modelConv = modelConv.cuda() modelClass = modelClass.cuda() modelHess = modelHess.cuda() weights = weights.cuda() criterion = torch.nn.CrossEntropyLoss(weights) mapLoc = None if torch.cuda.is_available() else {'cuda:0': 'cpu'} epochs = 80
from model import Classifier import matplotlib.pyplot as plt if __name__ == '__main__': # Load the utilities to clean up the data util = TitaticUtils() # load the feature vector (train_label, train_data, test_data) = util.create_feature_vector() # shuffle the data train_data, train_label = shuffle(train_data, train_label) # Load the classifier classifier = Classifier() # project the feature vector (train_pca, test_pca) = classifier.get_pca_projection(train_data, test_data, snum = 2) # Prediction predict_test_label = Classifier().classify('randomforest', train_label, train_pca, test_pca) util.write_file(predict_test_label, '../data/test_result.csv') # Visualization #plt.scatter(train_pca[:,0],train_pca[:,1], c = train_label) #plt.prism #plt.show()