def main(config='config/finetune/agnews/train.json'): cfg = Config(**json.load(open(config, "r"))) cfg_data = data.Config(**json.load(open(cfg.cfg_data, "r"))) cfg_model = models.Config(**json.load(open(cfg.cfg_model, "r"))) cfg_optim = trainer.Config(**json.load(open(cfg.cfg_optim, "r"))) set_seeds(cfg.seed) TaskDataset = data.get_class( cfg_data.task) # task dataset class according to the task tokenizer = tokenization.FullTokenizer(vocab_file=cfg_data.vocab_file, do_lower_case=True) dataset = TaskDataset( cfg_data.data_file[cfg.mode], pipelines=[ data.RemoveSymbols('\\'), data.Tokenizing(tokenizer.convert_to_unicode, tokenizer.tokenize), data.AddSpecialTokensWithTruncation(cfg_data.max_len), data.TokenIndexing(tokenizer.convert_tokens_to_ids, TaskDataset.labels, cfg_data.max_len) ], n_data=None) dataset = TensorDataset(*dataset.get_tensors()) # To Tensors data_iter = DataLoader(dataset, batch_size=cfg_optim.batch_size, shuffle=True) model = models.BlendCNN(cfg_model, len(TaskDataset.labels)) checkpoint.load_embedding(model.embed, cfg.pretrain_file) optimizer = optim.optim4GPU(cfg_optim, model) train_loop = trainer.TrainLoop(cfg_optim, model, data_iter, optimizer, cfg.save_dir, get_device()) def get_loss(model, batch, global_step): # make sure loss is a scalar tensor input_ids, segment_ids, input_mask, label_id = batch logits = model(input_ids, segment_ids, input_mask) loss = nn.CrossEntropyLoss()(logits, label_id) return loss def evaluate(model, batch): input_ids, segment_ids, input_mask, label_id = batch logits = model(input_ids, segment_ids, input_mask) _, label_pred = logits.max(1) result = (label_pred == label_id).float() #.cpu().numpy() accuracy = result.mean() return accuracy, result if cfg.mode == "train": train_loop.train(get_loss, cfg.model_file, None) # not use pretrain_file print("Training has been done properly.") elif cfg.mode == "eval": results = train_loop.eval(evaluate, cfg.model_file) total_accuracy = torch.cat(results).mean().item() print(f"Accuracy: {total_accuracy}")
def main(config='config/finetune/agnews/train.json'): cfg = Config(**json.load(open(config, "r"))) cfg_data = data.Config(**json.load(open(cfg.cfg_data, "r"))) cfg_model = models.Config(**json.load(open(cfg.cfg_model, "r"))) cfg_optim = trainer.Config(**json.load(open(cfg.cfg_optim, "r"))) set_seeds(cfg.seed) ### Prepare Dataset and Preprocessing ### TaskDataset = data.get_class(cfg_data.task) # task dataset class according to the task tokenizer = tokenization.FullTokenizer(vocab_file=cfg_data.vocab_file, do_lower_case=True) dataset = TaskDataset(cfg_data.data_file[cfg.mode], pipelines=[ data.RemoveSymbols('\\'), data.Tokenizing(tokenizer.convert_to_unicode, tokenizer.tokenize), data.AddSpecialTokensWithTruncation(cfg_data.max_len), data.TokenIndexing(tokenizer.convert_tokens_to_ids, TaskDataset.labels, cfg_data.max_len) ], n_data=None) tensors = TensorDataset(*dataset.get_tensors()) # To Tensors data_iter = DataLoader(tensors, batch_size=cfg_optim.batch_size, shuffle=False) ### Fetch Teacher's output and put it into the dataset ### def fetch_logits(model): def get_logits(model, batch): input_ids, segment_ids, input_mask, label_id = batch logits = model(input_ids, segment_ids, input_mask) return 0.0, logits train_loop = trainer.TrainLoop(cfg_optim, model, data_iter, None, None, get_device()) results = torch.cat(train_loop.eval(get_logits, cfg.model_file)) return results if cfg.mode == "train": print("Fetching teacher's output...") teacher = models.Classifier4Transformer(cfg_model, len(TaskDataset.labels)) teacher.load_state_dict(torch.load(cfg.model_file)) # use trained model with torch.no_grad(): teacher_logits = fetch_logits(teacher) tensors = TensorDataset(teacher_logits, *dataset.get_tensors()) # To Tensors data_iter = DataLoader(tensors, batch_size=cfg_optim.batch_size, shuffle=False) ### Models ### model = models.BlendCNN(cfg_model, len(TaskDataset.labels)) checkpoint.load_embedding(model.embed, cfg.pretrain_file) optimizer = optim.optim4GPU(cfg_optim, model) train_loop = trainer.TrainLoop( cfg_optim, model, data_iter, optimizer, cfg.save_dir, get_device() ) def get_loss(model, batch, global_step): # make sure loss is a scalar tensor teacher_logits, input_ids, segment_ids, input_mask, label_id = batch T = 1.0 logits = model(input_ids, segment_ids, input_mask) loss = 0.1*nn.CrossEntropyLoss()(logits, label_id) loss += 0.9*nn.KLDivLoss()( F.log_softmax(logits/T, dim=1), F.softmax(teacher_logits/T, dim=1) ) #loss = 0.9*nn.MSELoss()(logits, teacher_logits) return loss def evaluate(model, batch): input_ids, segment_ids, input_mask, label_id = batch logits = model(input_ids, segment_ids, input_mask) _, label_pred = logits.max(1) result = (label_pred == label_id).float() #.cpu().numpy() accuracy = result.mean() return accuracy, result if cfg.mode == "train": train_loop.train(get_loss, None, None) # not use pretrain file print("Training has been done properly.") elif cfg.mode == "eval": results = train_loop.eval(evaluate, cfg.model_file) total_accuracy = torch.cat(results).mean().item() print(f"Accuracy: {total_accuracy}")
def main(config='config/blendcnn/mrpc/eval.json', args=None): cfg = Config(**json.load(open(config, "r"))) cfg_data = data.Config(**json.load(open(cfg.cfg_data, "r"))) cfg_model = models.Config(**json.load(open(cfg.cfg_model, "r"))) cfg_optim = trainer.Config(**json.load(open(cfg.cfg_optim, "r"))) set_seeds(cfg.seed) TaskDataset = data.get_class( cfg_data.task) # task dataset class according to the task tokenizer = tokenization.FullTokenizer(vocab_file=cfg_data.vocab_file, do_lower_case=True) dataset = TaskDataset( args.dataset_location, pipelines=[ data.RemoveSymbols('\\'), data.Tokenizing(tokenizer.convert_to_unicode, tokenizer.tokenize), data.AddSpecialTokensWithTruncation(cfg_data.max_len), data.TokenIndexing(tokenizer.convert_tokens_to_ids, TaskDataset.labels, cfg_data.max_len) ], n_data=None) dataset = TensorDataset(*dataset.get_tensors()) # To Tensors data_iter = DataLoader(dataset, batch_size=args.batch_size, shuffle=False) model = models.BlendCNN(cfg_model, len(TaskDataset.labels)) checkpoint.load_embedding(model.embed, cfg.pretrain_file) optimizer = optim.optim4GPU(cfg_optim, model) train_loop = trainer.TrainLoop(cfg_optim, model, data_iter, optimizer, cfg.save_dir, get_device()) def get_loss(model, batch, global_step): # make sure loss is a scalar tensor input_ids, segment_ids, input_mask, label_id = batch logits = model(input_ids, segment_ids, input_mask) loss = nn.CrossEntropyLoss()(logits, label_id) return loss def evaluate(model, batch): input_ids, segment_ids, input_mask, label_id = batch logits = model(input_ids, segment_ids, input_mask) _, label_pred = logits.max(1) result = (label_pred == label_id).float() #.cpu().numpy() accuracy = result.mean() return accuracy, result class Bert_DataLoader(object): def __init__(self, loader=None, model_type=None, device='cpu', batch_size=1): self.loader = loader self.model_type = model_type self.device = device self.batch_size = batch_size def __iter__(self): for batch in self.loader: batch = tuple(t.to(self.device) for t in batch) outputs = { 'output_all': (batch[0], batch[1], batch[2]), 'labels': batch[3] } yield outputs['output_all'], outputs['labels'] def benchmark(model): total_samples = 0 total_time = 0 index = 0 class RandomDataset(object): def __init__(self, size, shape): self.len = size self.input_ids = torch.randint(low=0, high=30522, size=(size, shape), dtype=torch.int64) self.segment_ids = torch.randint(low=0, high=1, size=(size, shape), dtype=torch.int64) self.input_mask = torch.randint(low=0, high=1, size=(size, shape), dtype=torch.int64) self.data = (self.input_ids, self.segment_ids, self.input_mask) def __getitem__(self, index): return (self.data[0][index], self.data[1][index], self.data[2][index]) def __len__(self): return self.len rand_loader = DataLoader(dataset=RandomDataset(size=5000, shape=128), batch_size=args.batch_size, shuffle=True) for batch in rand_loader: index += 1 tic = time.time() if os.environ.get('BLENDCNN_PROFILING') is not None: with profiler.profile(record_shapes=True) as prof: with torch.no_grad(): input_ids, segment_ids, input_mask = batch _ = model(*batch) else: with torch.no_grad( ): # evaluation without gradient calculation input_ids, segment_ids, input_mask = batch _ = model(*batch) if index > args.warmup: total_samples += batch[0].size()[0] total_time += time.time() - tic throughput = total_samples / total_time print('Latency: %.3f ms' % (1 / throughput * 1000)) print('Throughput: %.3f images/sec' % (throughput)) if os.environ.get('BLENDCNN_PROFILING') is not None: print(prof.key_averages().table(sort_by="cpu_time_total", row_limit=10)) def eval_func(model): results = [] # prediction results total_samples = 0 total_time = 0 index = 0 model.eval() eval_dataloader = Bert_DataLoader(loader=data_iter, batch_size=args.batch_size) for batch, label in eval_dataloader: index += 1 tic = time.time() if os.environ.get('BLENDCNN_PROFILING') is not None: with profiler.profile(record_shapes=True) as prof: with torch.no_grad(): accuracy, result = evaluate(model, (*batch, label)) else: with torch.no_grad( ): # evaluation without gradient calculation accuracy, result = evaluate(model, (*batch, label)) results.append(result) if index > args.warmup: total_samples += batch[0].size()[0] total_time += time.time() - tic total_accuracy = torch.cat(results).mean().item() throughput = total_samples / total_time print('Latency: %.3f ms' % (1 / throughput * 1000)) print('Throughput: %.3f samples/sec' % (throughput)) print('Accuracy: %.3f ' % (total_accuracy)) if os.environ.get('BLENDCNN_PROFILING') is not None: print(prof.key_averages().table(sort_by="cpu_time_total", row_limit=10)) return total_accuracy if cfg.mode == "train": train_loop.train(get_loss, cfg.model_file, None) # not use pretrain_file print("Training has been done properly.") elif cfg.mode == "eval": # results = train_loop.eval(evaluate, cfg.model_file) # total_accuracy = torch.cat(results).mean().item() # print(f"Accuracy: {total_accuracy}") if args.tune: import lpot from lpot import common # lpot tune model.load_state_dict(torch.load(args.input_model)) eval_dataloader = Bert_DataLoader(loader=data_iter, batch_size=args.batch_size) quantizer = lpot.Quantization(args.tuned_yaml) quantizer.model = common.Model(model) quantizer.calib_dataloader = eval_dataloader quantizer.eval_func = eval_func q_model = quantizer() q_model.save(args.tuned_checkpoint) elif args.int8: from lpot.utils.pytorch import load int8_model = load( os.path.abspath(os.path.expanduser(args.tuned_checkpoint)), model) print(int8_model) if args.accuracy_only: eval_func(int8_model) elif args.benchmark: benchmark(int8_model) else: model.load_state_dict(torch.load(args.input_model)) print(model) if args.accuracy_only: eval_func(model) elif args.benchmark: benchmark(model)