def main(load, savestr='default', lr=1e-3, beta=0.01, kill_time=True):
"""
:param load:
:param savestr:
:param lr:
:param curri:
:return:
"""
total_epochs = 40
iter_per_epoch = int(saturation / param_bs)
optim = None
starting_epoch = 0
starting_iteration = 0
logfile = "log/dnc_" + savestr + "_" + datetime_filename() + ".txt"
num_workers = 32
ig = InputGenJ(no_underlying=True, death_only=True, debug=True)
# ig = InputGenH(no_underlying=True)
param_x = ig.input_dim
param_v_t = ig.output_dim
target_dim = param_v_t
trainds = ig.get_train()
validds = ig.get_valid()
traindl = DataLoader(dataset=trainds,
batch_size=param_bs,
num_workers=num_workers,
collate_fn=pad_collate,
pin_memory=True)
validdl = DataLoader(dataset=validds,
batch_size=param_bs,
num_workers=num_workers // 2,
collate_fn=pad_collate,
pin_memory=True)
print("Generating prior")
prior_probability = get_death_code_proportion(ig)
print("Using", num_workers, "workers for training set")
computer = PriorDNC(x=param_x,
h=param_h,
L=param_L,
v_t=param_v_t,
W=param_W,
R=param_R,
N=param_N,
prior=prior_probability)
# load model:
if load:
print("loading model")
computer, optim, starting_epoch, starting_iteration = load_model(
computer, optim, starting_epoch, starting_iteration, savestr)
computer = computer.cuda()
if optim is None:
# does adamax perform better with sparse output?
optimizer = torch.optim.Adam(computer.parameters(), lr=lr)
else:
# print('use Adadelta optimizer with learning rate ', lr)
# optimizer = torch.optim.Adadelta(computer.parameters(), lr=lr)
optimizer = optim
for group in optimizer.param_groups:
print("Currently using a learing rate of ", group["lr"])
# creating the positive_weights
# with open("/infodev1/rep/projects/jason/pickle/dcc.pkl","rb") as f:
# # loaded here is a vector where v_i is the number of times death label i has occured
# weights=pickle.load(f)
# negs=59652-weights
# weights[weights<4]=3
# weights=negs/weights
# weights=torch.from_numpy(weights).float().cuda()
# weights=Variable(weights)
real_criterion = TOELoss()
# this parameter does not appear in PyTorch 0.3.1
# binary_criterion = WeightedBCELLoss(pos_weight=None)
binary_criterion = nn.BCEWithLogitsLoss()
# starting with the epoch after the loaded one
train(computer, optimizer, real_criterion, binary_criterion,
traindl, validdl, int(starting_epoch), total_epochs,
int(starting_iteration), iter_per_epoch, target_dim, savestr, beta,
logfile, kill_time)
def main(load, savestr='default', lr=1e-3, beta=0.01):
"""
:param load:
:param savestr:
:param lr:
:param curri:
:return:
"""
total_epochs = 1
iter_per_epoch = int(saturation / param_bs)
optim = None
starting_epoch = 0
starting_iteration = 0
logfile = "log/dnc_" + savestr + "_" + datetime_filename() + ".txt"
num_workers = 16
# ig=InputGenG(small_target=True)
ig = InputGenH(small_target=True)
trainds = ig.get_train()
validds = ig.get_valid()
traindl = DataLoader(dataset=trainds,
batch_size=param_bs,
num_workers=num_workers,
collate_fn=pad_collate,
pin_memory=True)
validdl = DataLoader(dataset=validds,
batch_size=param_bs,
num_workers=num_workers // 2,
collate_fn=pad_collate,
pin_memory=True)
print("Using", num_workers, "workers for training set")
computer = SeqDNC(x=param_x,
h=param_h,
L=param_L,
v_t=param_v_t,
W=param_W,
R=param_R,
N=param_N)
# load model:
if load:
print("loading model")
computer, optim, starting_epoch, starting_iteration = load_model(
computer, optim, starting_epoch, starting_iteration, savestr)
computer = computer.cuda()
if optim is None:
optimizer = torch.optim.Adam(computer.parameters(), lr=lr)
else:
# print('use Adadelta optimizer with learning rate ', lr)
# optimizer = torch.optim.Adadelta(computer.parameters(), lr=lr)
optimizer = optim
for group in optimizer.param_groups:
print("Currently using a learing rate of ", group["lr"])
# creating the positive_weights
with open("/infodev1/rep/projects/jason/pickle/dcc.pkl", "rb") as f:
# loaded here is a vector where v_i is the number of times death label i has occured
weights = pickle.load(f)
negs = 59652 - weights
weights[weights < 4] = 3
weights = negs / weights
weights = torch.from_numpy(weights).float().cuda()
weights = Variable(weights)
real_criterion = TOELoss()
# this parameter does not appear in PyTorch 0.3.1
binary_criterion = WeightedBCELLoss(pos_weight=weights)
# starting with the epoch after the loaded one
train(computer, optimizer, real_criterion, binary_criterion,
traindl, validdl, int(starting_epoch), total_epochs,
int(starting_iteration), iter_per_epoch, savestr, beta, logfile)
def main(load, savestr, lr=1e-3, beta=1e-3):
total_epochs = 10
iter_per_epoch = 2019
optim = None
starting_epoch = 0
starting_iteration = 0
logfile = "log/lstm_" + savestr + "_" + datetime_filename() + ".txt"
num_workers = 16
ig = InputGenJ()
trainds = ig.get_train()
validds = ig.get_valid()
testds = ig.get_test()
validdl = DataLoader(dataset=validds,
batch_size=8,
num_workers=num_workers // 2,
collate_fn=pad_collate,
shuffle=True)
traindl = DataLoader(dataset=trainds,
batch_size=8,
num_workers=num_workers,
collate_fn=pad_collate,
shuffle=True)
print("Using", num_workers, "workers for training set")
# testing whether this LSTM works is basically a question whether
lstm = lstmwrapperJ(input_size=ig.input_dim, output_size=ig.output_dim)
# load model:
if load:
print("loading model")
lstm, optim, starting_epoch, starting_iteration = load_model(
lstm, optim, starting_epoch, starting_iteration, savestr)
lstm = lstm.cuda()
if optim is None:
optimizer = torch.optim.Adam(lstm.parameters(), lr=lr)
else:
# print('use Adadelta optimizer with learning rate ', lr)
# optimizer = torch.optim.Adadelta(computer.parameters(), lr=lr)
optimizer = optim
for group in optimizer.param_groups:
print("Currently using a learing rate of ", group["lr"])
real_criterion = TOELoss()
# creating the positive_weights
# with open("/infodev1/rep/projects/jason/pickle/dcc.pkl","rb") as f:
# # loaded here is a vector where v_i is the number of times death label i has occured
# weights=pickle.load(f)
# negs=59652-weights
# weights[weights<4]=3
# weights=negs/weights
# weights=torch.from_numpy(weights).float().cuda()
# weights=Variable(weights)
# binary_criterion = WeightedBCELLoss(pos_weight=weights)
binary_criterion = nn.BCEWithLogitsLoss()
# starting with the epoch after the loaded one
train(lstm, optimizer, real_criterion, binary_criterion, traindl, validdl,
int(starting_epoch), total_epochs, int(starting_iteration),
iter_per_epoch, savestr, beta, ig.output_dim, logfile)
def main(load=False, lr=1e-3, beta=1e-3, savestr="", kill_time=True):
total_epochs = 1
iter_per_epoch = 10000
lr = lr
optim = None
starting_epoch = 0
starting_iteration = 0
logfile = "log/taco_" + savestr + "_" + datetime_filename() + ".txt"
num_workers = 4
ig = InputGenH(small_target=True)
validds = ig.get_valid()
trainds = ig.get_train()
validdl = DataLoader(dataset=validds,
batch_size=8,
num_workers=num_workers,
collate_fn=pad_collate,
pin_memory=True)
traindl = DataLoader(dataset=trainds,
batch_size=8,
num_workers=num_workers,
collate_fn=pad_collate,
pin_memory=True)
print("Using", num_workers, "workers for training set")
computer = Tacotron()
# load model:
if load:
print("loading model")
computer, optim, starting_epoch, starting_iteration = load_model(
computer, optim, starting_epoch, starting_iteration, savestr)
computer = computer.cuda()
if optim is None:
print("Using Adam with lr", lr)
optimizer = torch.optim.Adam(
[i for i in computer.parameters() if i.requires_grad], lr=lr)
else:
# print('use Adadelta optimizer with learning rate ', lr)
# optimizer = torch.optim.Adadelta(computer.parameters(), lr=lr)
optimizer = optim
for group in optimizer.param_groups:
print("Currently using a learing rate of ", group["lr"])
# creating the positive_weights
with open("/infodev1/rep/projects/jason/pickle/dcc.pkl", "rb") as f:
# loaded here is a vector where v_i is the number of times death label i has occured
weights = pickle.load(f)
negs = 59652 - weights
weights[weights < 4] = 3
weights = negs / weights
weights = torch.from_numpy(weights).float().cuda()
weights = Variable(weights)
binary_criterion = WeightedBCELLoss(pos_weight=weights)
# starting with the epoch after the loaded one
real_criterion = TOELoss()
train(computer, optimizer, real_criterion, binary_criterion,
traindl, validdl, int(starting_epoch), total_epochs,
int(starting_iteration), iter_per_epoch, savestr, beta, logfile,
kill_time)
:param afterpipe:
:return:
"""
beforep = torch.sigmoid(beforepipe)
afterp = torch.sigmoid(afterpipe)
beforep = beforep.clamp(1e-6, 1 - 1e-6)
afterp = afterp.clamp(1e-6, 1 - 1e-6)
pos = beforepipe * torch.log(beforep / afterp)
neg = (1 - beforep) * torch.log((1 - beforep) / (1 - afterp))
return pos + neg
if __name__ == '__main__':
from death.post.inputgen_planJ import InputGenJ, pad_collate
from death.final.losses import TOELoss
real_criterion = TOELoss()
binary_criterion = DiscreteCrossEntropy()
igj = InputGenJ()
trainig = igj.get_train()
d1 = trainig[10]
d2 = trainig[11]
# input=torch.empty(64,400,7298).uniform_()
# target=torch.empty(64,435).uniform_()
input, target, loss_type, time_length = [
t.cuda() for t in pad_collate((d1, d2))
]
model = TransformerMixedAttnSoftmax(binary_criterion=binary_criterion,
real_criterion=real_criterion,
input_size=7298,
output_size=435,
prior=None).cuda()