def test(): parser = argparse.ArgumentParser() parser.add_argument("-d", "--device", default="cpu") parser.add_argument("-i", "--hidden-size", type=int, default=256) args = parser.parse_args() args.device = torch.device(args.device) model = MyModel0(len(VOCAB), 16, args.hidden_size).to(args.device) dataset = MyDataset(None, args.device, test_path="data/testdict.pth") model.load_state_dict(torch.load("Bi-LSTM_model.pth")) model.eval() with torch.no_grad(): for k in dataset.testdict.ks(): texttensor = dataset.get_test_data(k) oupt = model(texttensor) prob = torch.nn.functional.softmax(oupt, dim=2) prob, pred = torch.max(prob, dim=2) prob = prob.squeeze().cpu().numpy() pred = pred.squeeze().cpu().numpy() realtext = dataset.testdict[k] result = pred_to_dict(realtext, pred, prob) with open("results/" + k + ".json", "w", encoding="utf-8") as jsonopened: json.dump(result, jsonopened, indent=4) print(k)
def test(): parser = argparse.ArgumentParser() parser.add_argument("-d", "--device", default="cpu") parser.add_argument("-i", "--hidden-size", type=int, default=256) parser.add_argument("-o", "--outputFolder", type=str) args = parser.parse_args() args.device = torch.device(args.device) model = MyModel0(len(VOCAB), 16, args.hidden_size).to(args.device) dataset = MyDataset(None, args.device, test_path="/input/data/test_dict2.pth") k = [x for x in dataset.test_dict.keys()][0] dataset.test_dict[k] = dataset.test_dict[k].replace("\t", " ") model.load_state_dict(torch.load("/input/model.pth", map_location='cpu')) model.eval() with torch.no_grad(): for key in dataset.test_dict.keys(): text_tensor = dataset.get_test_data(key) oupt = model(text_tensor) prob = torch.nn.functional.softmax(oupt, dim=2) prob, pred = torch.max(prob, dim=2) prob = prob.squeeze().cpu().numpy() pred = pred.squeeze().cpu().numpy() real_text = dataset.test_dict[key] result = pred_to_dict(real_text, pred, prob) with open(args.outputFolder + 'result' + ".json", "w", encoding="utf-8") as json_opened: json.dump(result, json_opened, indent=4) print(key)
def main(): parser = argparse.ArgumentParser() parser.add_argument("-d", "--device", default="cuda") parser.add_argument("-b", "--batch_size", type=int, default=8) parser.add_argument("-e", "--max_epoch", type=int, default=2000) parser.add_argument("-v", "--val-at", type=int, default=100) parser.add_argument("-i", "--hidden-size", type=int, default=512) args = parser.parse_args() args.device = torch.device(args.device) # torch.backends.cudnn.enabled = False model = MyModel0(len(VOCAB), 20, args.hidden_size).to(args.device) dataset = MyDataset("data/data_dict.pth", args.device) criterion = nn.CrossEntropyLoss(weight=torch.tensor([0.1, 1, 1.2, 0.8, 5], device=args.device)) optimizer = optim.Adam(model.parameters()) scheduler = optim.lr_scheduler.StepLR(optimizer, 1000) for i in range(args.max_epoch // args.val_at): train( model, dataset, criterion, optimizer, (i * args.val_at + 1, (i + 1) * args.val_at + 1), args.batch_size, ) validate(model, dataset) validate(model, dataset, batch_size=10)
def main(): parser = argparse.ArgumentParser() parser.add_argument("-d", "--device", default="cpu") parser.add_argument("-b", "--batch_size", type=int, default=16) parser.add_argument("-e", "--max_epoch", type=int, default=1500) parser.add_argument("-v", "--val-at", type=int, default=100) parser.add_argument("-i", "--hidden-size", type=int, default=256) parser.add_argument("--val-size", type=int, default=76) args = parser.parse_args() args.device = torch.device(args.device) model = MyModel0(len(VOCAB), 16, args.hidden_size).to(args.device) dataset = MyDataset( "data/data_dict4.pth", args.device, val_size=args.val_size, test_path="data/test_dict.pth", ) criterion = nn.CrossEntropyLoss( weight=torch.tensor([0.1, 1, 1.2, 0.8, 1.5], device=args.device)) optimizer = optim.Adam(model.parameters()) scheduler = optim.lr_scheduler.StepLR(optimizer, 1000) for i in range(args.max_epoch // args.val_at): train( model, dataset, criterion, optimizer, (i * args.val_at + 1, (i + 1) * args.val_at + 1), args.batch_size, ) # validate(model, dataset) validate(model, dataset, batch_size=76) torch.save(model.state_dict(), "model.pth") model.eval() with torch.no_grad(): for key in dataset.test_dict.keys(): text_tensor = dataset.get_test_data(key) oupt = model(text_tensor) prob = torch.nn.functional.softmax(oupt, dim=2) prob, pred = torch.max(prob, dim=2) prob = prob.squeeze().cpu().numpy() pred = pred.squeeze().cpu().numpy() real_text = dataset.test_dict[key] result = pred_to_dict(real_text, pred, prob) with open("results/" + key + ".json", "w", encoding="utf-8") as json_opened: json.dump(result, json_opened, indent=4)
def test(): parser = argparse.ArgumentParser() parser.add_argument("-d", "--device", default="cpu") parser.add_argument("-i", "--hidden-size", type=int, default=256) parser.add_argument('--name', type=str, default="") opt = parser.parse_args() savename = opt.name.split(".")[0] args = parser.parse_args() args.device = torch.device(args.device) model = MyModel0(len(VOCAB), 16, args.hidden_size).to(args.device) dataset = MyDataset(None, args.device, test_path="C:\\Users\\lenovo\\Desktop\\Class\\Network\\Project\\task3\\data/test_dict.pth") model.load_state_dict(torch.load("C:\\Users\\lenovo\\Desktop\\Class\\Network\\Project\\task3\\src\\model.pth")) model.eval() with torch.no_grad(): for key in dataset.test_dict.keys(): if key != savename: continue text_tensor = dataset.get_test_data(key) oupt = model(text_tensor) prob = torch.nn.functional.softmax(oupt, dim=2) prob, pred = torch.max(prob, dim=2) prob = prob.squeeze().cpu().numpy() pred = pred.squeeze().cpu().numpy() real_text = dataset.test_dict[key] result = pred_to_dict(real_text, pred, prob) with open("results/" + key + ".json", "w", encoding="utf-8") as json_opened: json.dump(result, json_opened, indent=4) print(key)
def inference(text): text[0] = preprocess(text[0]) device = torch.device("cpu") hidden_size = 256 model = MyModel0(len(VOCAB), 16, hidden_size).to(device) model.load_state_dict( torch.load("model.pth", map_location=torch.device('cpu'))) #text = ["shubham bisht, something happens"] text_tensor = torch.zeros(len(text[0]), 1, dtype=torch.long) text_tensor[:, 0] = torch.LongTensor([VOCAB.find(c) for c in text[0].upper()]) #print(text_tensor) inp = text_tensor.to(device) oupt = model(inp) prob = torch.nn.functional.softmax(oupt, dim=2) prob, pred = torch.max(prob, dim=2) color_print(text[0], pred) json = pred_to_dict(text[0], pred, prob) print("\n###########################\n") print(json) return json
def main(): # read the command line arguments parser = argparse.ArgumentParser() parser.add_argument("-d", "--device", default="cpu") parser.add_argument("-b", "--batch_size", type=int, default=10) parser.add_argument("-e", "--max_epoch", type=int, default=2500) parser.add_argument("-v", "--val-at", type=int, default=100) parser.add_argument("-i", "--hidden-size", type=int, default=256) parser.add_argument("--val-size", type=int, default=76) args = parser.parse_args() args.device = torch.device(args.device) # build model model = MyModel0(len(VOCAB), 16, args.hidden_size).to(args.device) # prepare the dataset by initializing MyDataset class dataset = MyDataset( "data/data_dict4.pth", args.device, val_size=args.val_size, test_path="data/test_dict.pth", ) # Use CrossEntropy as loss function criterion = nn.CrossEntropyLoss( weight=torch.tensor([0.1, 1, 1.2, 0.8, 1.5], device=args.device)) # Adam optimizer gives best performance in this case optimizer = optim.Adam(model.parameters()) # Added learning rate scheduler for future callbacks scheduler = optim.lr_scheduler.StepLR(optimizer, 1000) for i in range(args.max_epoch // args.val_at): train( model, dataset, criterion, optimizer, (i * args.val_at + 1, (i + 1) * args.val_at + 1), args.batch_size, ) # validate(model, dataset) # validate(model, dataset, batch_size=10) torch.save(model.state_dict(), "Bi-LSTM_model.pth") model.eval() with torch.no_grad(): for key in dataset.test_dict.keys(): text_tensor = dataset.get_test_data(key) oupt = model(text_tensor) prob = torch.nn.functional.softmax(oupt, dim=2) prob, pred = torch.max(prob, dim=2) prob = prob.squeeze().cpu().numpy() pred = pred.squeeze().cpu().numpy() tealtext = dataset.test_dict[key] result = pred_to_dict(tealtext, pred, prob) with open("results/" + key + ".json", "w", encoding="utf-8") as json_opened: json.dump(result, json_opened, indent=4) print(key)