def main(load, savestr='default', lr=1e-3, beta=0.01, kill_time=True): """ :param load: :param savestr: :param lr: :param curri: :return: """ total_epochs = 40 iter_per_epoch = int(saturation / param_bs) optim = None starting_epoch = 0 starting_iteration = 0 logfile = "log/dnc_" + savestr + "_" + datetime_filename() + ".txt" num_workers = 32 ig = InputGenJ(no_underlying=True, death_only=True, debug=True) # ig = InputGenH(no_underlying=True) param_x = ig.input_dim param_v_t = ig.output_dim target_dim = param_v_t trainds = ig.get_train() validds = ig.get_valid() traindl = DataLoader(dataset=trainds, batch_size=param_bs, num_workers=num_workers, collate_fn=pad_collate, pin_memory=True) validdl = DataLoader(dataset=validds, batch_size=param_bs, num_workers=num_workers // 2, collate_fn=pad_collate, pin_memory=True) print("Generating prior") prior_probability = get_death_code_proportion(ig) print("Using", num_workers, "workers for training set") computer = PriorDNC(x=param_x, h=param_h, L=param_L, v_t=param_v_t, W=param_W, R=param_R, N=param_N, prior=prior_probability) # load model: if load: print("loading model") computer, optim, starting_epoch, starting_iteration = load_model( computer, optim, starting_epoch, starting_iteration, savestr) computer = computer.cuda() if optim is None: # does adamax perform better with sparse output? optimizer = torch.optim.Adam(computer.parameters(), lr=lr) else: # print('use Adadelta optimizer with learning rate ', lr) # optimizer = torch.optim.Adadelta(computer.parameters(), lr=lr) optimizer = optim for group in optimizer.param_groups: print("Currently using a learing rate of ", group["lr"]) # creating the positive_weights # with open("/infodev1/rep/projects/jason/pickle/dcc.pkl","rb") as f: # # loaded here is a vector where v_i is the number of times death label i has occured # weights=pickle.load(f) # negs=59652-weights # weights[weights<4]=3 # weights=negs/weights # weights=torch.from_numpy(weights).float().cuda() # weights=Variable(weights) real_criterion = TOELoss() # this parameter does not appear in PyTorch 0.3.1 # binary_criterion = WeightedBCELLoss(pos_weight=None) binary_criterion = nn.BCEWithLogitsLoss() # starting with the epoch after the loaded one train(computer, optimizer, real_criterion, binary_criterion, traindl, validdl, int(starting_epoch), total_epochs, int(starting_iteration), iter_per_epoch, target_dim, savestr, beta, logfile, kill_time)
def main(load, savestr='default', lr=1e-3, beta=0.01): """ :param load: :param savestr: :param lr: :param curri: :return: """ total_epochs = 1 iter_per_epoch = int(saturation / param_bs) optim = None starting_epoch = 0 starting_iteration = 0 logfile = "log/dnc_" + savestr + "_" + datetime_filename() + ".txt" num_workers = 16 # ig=InputGenG(small_target=True) ig = InputGenH(small_target=True) trainds = ig.get_train() validds = ig.get_valid() traindl = DataLoader(dataset=trainds, batch_size=param_bs, num_workers=num_workers, collate_fn=pad_collate, pin_memory=True) validdl = DataLoader(dataset=validds, batch_size=param_bs, num_workers=num_workers // 2, collate_fn=pad_collate, pin_memory=True) print("Using", num_workers, "workers for training set") computer = SeqDNC(x=param_x, h=param_h, L=param_L, v_t=param_v_t, W=param_W, R=param_R, N=param_N) # load model: if load: print("loading model") computer, optim, starting_epoch, starting_iteration = load_model( computer, optim, starting_epoch, starting_iteration, savestr) computer = computer.cuda() if optim is None: optimizer = torch.optim.Adam(computer.parameters(), lr=lr) else: # print('use Adadelta optimizer with learning rate ', lr) # optimizer = torch.optim.Adadelta(computer.parameters(), lr=lr) optimizer = optim for group in optimizer.param_groups: print("Currently using a learing rate of ", group["lr"]) # creating the positive_weights with open("/infodev1/rep/projects/jason/pickle/dcc.pkl", "rb") as f: # loaded here is a vector where v_i is the number of times death label i has occured weights = pickle.load(f) negs = 59652 - weights weights[weights < 4] = 3 weights = negs / weights weights = torch.from_numpy(weights).float().cuda() weights = Variable(weights) real_criterion = TOELoss() # this parameter does not appear in PyTorch 0.3.1 binary_criterion = WeightedBCELLoss(pos_weight=weights) # starting with the epoch after the loaded one train(computer, optimizer, real_criterion, binary_criterion, traindl, validdl, int(starting_epoch), total_epochs, int(starting_iteration), iter_per_epoch, savestr, beta, logfile)
def main(load, savestr, lr=1e-3, beta=1e-3): total_epochs = 10 iter_per_epoch = 2019 optim = None starting_epoch = 0 starting_iteration = 0 logfile = "log/lstm_" + savestr + "_" + datetime_filename() + ".txt" num_workers = 16 ig = InputGenJ() trainds = ig.get_train() validds = ig.get_valid() testds = ig.get_test() validdl = DataLoader(dataset=validds, batch_size=8, num_workers=num_workers // 2, collate_fn=pad_collate, shuffle=True) traindl = DataLoader(dataset=trainds, batch_size=8, num_workers=num_workers, collate_fn=pad_collate, shuffle=True) print("Using", num_workers, "workers for training set") # testing whether this LSTM works is basically a question whether lstm = lstmwrapperJ(input_size=ig.input_dim, output_size=ig.output_dim) # load model: if load: print("loading model") lstm, optim, starting_epoch, starting_iteration = load_model( lstm, optim, starting_epoch, starting_iteration, savestr) lstm = lstm.cuda() if optim is None: optimizer = torch.optim.Adam(lstm.parameters(), lr=lr) else: # print('use Adadelta optimizer with learning rate ', lr) # optimizer = torch.optim.Adadelta(computer.parameters(), lr=lr) optimizer = optim for group in optimizer.param_groups: print("Currently using a learing rate of ", group["lr"]) real_criterion = TOELoss() # creating the positive_weights # with open("/infodev1/rep/projects/jason/pickle/dcc.pkl","rb") as f: # # loaded here is a vector where v_i is the number of times death label i has occured # weights=pickle.load(f) # negs=59652-weights # weights[weights<4]=3 # weights=negs/weights # weights=torch.from_numpy(weights).float().cuda() # weights=Variable(weights) # binary_criterion = WeightedBCELLoss(pos_weight=weights) binary_criterion = nn.BCEWithLogitsLoss() # starting with the epoch after the loaded one train(lstm, optimizer, real_criterion, binary_criterion, traindl, validdl, int(starting_epoch), total_epochs, int(starting_iteration), iter_per_epoch, savestr, beta, ig.output_dim, logfile)
def main(load=False, lr=1e-3, beta=1e-3, savestr="", kill_time=True): total_epochs = 1 iter_per_epoch = 10000 lr = lr optim = None starting_epoch = 0 starting_iteration = 0 logfile = "log/taco_" + savestr + "_" + datetime_filename() + ".txt" num_workers = 4 ig = InputGenH(small_target=True) validds = ig.get_valid() trainds = ig.get_train() validdl = DataLoader(dataset=validds, batch_size=8, num_workers=num_workers, collate_fn=pad_collate, pin_memory=True) traindl = DataLoader(dataset=trainds, batch_size=8, num_workers=num_workers, collate_fn=pad_collate, pin_memory=True) print("Using", num_workers, "workers for training set") computer = Tacotron() # load model: if load: print("loading model") computer, optim, starting_epoch, starting_iteration = load_model( computer, optim, starting_epoch, starting_iteration, savestr) computer = computer.cuda() if optim is None: print("Using Adam with lr", lr) optimizer = torch.optim.Adam( [i for i in computer.parameters() if i.requires_grad], lr=lr) else: # print('use Adadelta optimizer with learning rate ', lr) # optimizer = torch.optim.Adadelta(computer.parameters(), lr=lr) optimizer = optim for group in optimizer.param_groups: print("Currently using a learing rate of ", group["lr"]) # creating the positive_weights with open("/infodev1/rep/projects/jason/pickle/dcc.pkl", "rb") as f: # loaded here is a vector where v_i is the number of times death label i has occured weights = pickle.load(f) negs = 59652 - weights weights[weights < 4] = 3 weights = negs / weights weights = torch.from_numpy(weights).float().cuda() weights = Variable(weights) binary_criterion = WeightedBCELLoss(pos_weight=weights) # starting with the epoch after the loaded one real_criterion = TOELoss() train(computer, optimizer, real_criterion, binary_criterion, traindl, validdl, int(starting_epoch), total_epochs, int(starting_iteration), iter_per_epoch, savestr, beta, logfile, kill_time)
:param afterpipe: :return: """ beforep = torch.sigmoid(beforepipe) afterp = torch.sigmoid(afterpipe) beforep = beforep.clamp(1e-6, 1 - 1e-6) afterp = afterp.clamp(1e-6, 1 - 1e-6) pos = beforepipe * torch.log(beforep / afterp) neg = (1 - beforep) * torch.log((1 - beforep) / (1 - afterp)) return pos + neg if __name__ == '__main__': from death.post.inputgen_planJ import InputGenJ, pad_collate from death.final.losses import TOELoss real_criterion = TOELoss() binary_criterion = DiscreteCrossEntropy() igj = InputGenJ() trainig = igj.get_train() d1 = trainig[10] d2 = trainig[11] # input=torch.empty(64,400,7298).uniform_() # target=torch.empty(64,435).uniform_() input, target, loss_type, time_length = [ t.cuda() for t in pad_collate((d1, d2)) ] model = TransformerMixedAttnSoftmax(binary_criterion=binary_criterion, real_criterion=real_criterion, input_size=7298, output_size=435, prior=None).cuda()