def run(lr=0.001):
x = np.random.random((1000, 5)).astype("float32")
y = np.random.randint(0, 5, (1000, )).astype("int64")
trainloader = q.dataload(x[:800], y[:800], batch_size=100)
validloader = q.dataload(x[800:], y[800:], batch_size=100)
m = torch.nn.Sequential(torch.nn.Linear(5, 100), torch.nn.Linear(100, 5))
m[1].weight.requires_grad = False
losses = q.lossarray(torch.nn.CrossEntropyLoss())
params = m.parameters()
for param in params:
print(param.requires_grad)
init_val = m[1].weight.detach().numpy()
optim = torch.optim.Adam(q.params_of(m), lr=lr)
trainer = q.trainer(m).on(trainloader).loss(losses).optimizer(
optim).epochs(100)
# for b, (i, e) in trainer.inf_batches():
# print(i, e)
validator = q.tester(m).on(validloader).loss(losses)
q.train(trainer, validator).run()
new_val = m[1].weight.detach().numpy()
print(np.linalg.norm(new_val - init_val))
def run(lr=20.,
dropout=0.2,
dropconnect=0.2,
gradnorm=0.25,
epochs=25,
embdim=200,
encdim=200,
numlayers=2,
seqlen=35,
batsize=20,
eval_batsize=10,
cuda=False,
gpu=0,
test=False):
tt = q.ticktock("script")
device = torch.device("cpu")
if cuda:
device = torch.device("cuda", gpu)
tt.tick("loading data")
train_batches, valid_batches, test_batches, D = \
load_data(batsize=batsize, eval_batsize=eval_batsize, seqlen=seqlen)
tt.tock("data loaded")
print("{} batches in train".format(len(train_batches)))
tt.tick("creating model")
dims = [embdim] + ([encdim] * numlayers)
m = RNNLayer_LM(*dims, worddic=D, dropout=dropout)
if test:
for i, batch in enumerate(train_batches):
y = m(batch[0])
if i > 5:
break
print(y.size())
loss = q.SeqKLLoss(time_average=True, size_average=True, mode="logits")
ppl_loss = q.SeqPPL_Loss(time_average=True,
size_average=True,
mode="logits")
optim = torch.optim.SGD(q.params_of(m), lr=lr)
gradclip = q.ClipGradNorm(gradnorm)
trainer = q.trainer(m).on(train_batches).loss(loss).optimizer(
optim).device(device).hook(m).hook(gradclip)
tester = q.tester(m).on(valid_batches).loss(
loss, ppl_loss).device(device).hook(m)
tt.tock("created model")
tt.tick("training")
q.train(trainer, tester).run(epochs=epochs)
tt.tock("trained")
def run_classify(lr=0.001,
seqlen=6,
numex=500,
epochs=25,
batsize=10,
test=True,
cuda=False,
gpu=0):
device = torch.device("cpu")
if cuda:
device = torch.device("cuda", gpu)
# region construct data
colors = "red blue green magenta cyan orange yellow grey salmon pink purple teal".split(
)
D = dict(zip(colors, range(len(colors))))
inpseqs = []
targets = []
for i in range(numex):
inpseq = list(np.random.choice(colors, seqlen, replace=False))
target = np.random.choice(range(len(inpseq)), 1)[0]
target_class = D[inpseq[target]]
inpseq[target] = "${}$".format(inpseq[target])
inpseqs.append("".join(inpseq))
targets.append(target_class)
sm = q.StringMatrix()
sm.tokenize = lambda x: list(x)
for inpseq in inpseqs:
sm.add(inpseq)
sm.finalize()
print(sm[0])
print(sm.D)
targets = np.asarray(targets)
data = q.dataload(sm.matrix[:-100], targets[:-100], batch_size=batsize)
valid_data = q.dataload(sm.matrix[-100:],
targets[-100:],
batch_size=batsize)
# endregion
# region model
embdim = 20
enc2inpdim = 45
encdim = 20
outdim = 20
emb = q.WordEmb(embdim, worddic=sm.D) # sm dictionary (characters)
out = q.WordLinout(outdim, worddic=D) # target dictionary
# encoders:
enc1 = q.RNNEncoder(embdim, encdim, bidir=True)
enc2 = q.RNNCellEncoder(enc2inpdim, outdim // 2, bidir=True)
# model
class Model(torch.nn.Module):
def __init__(self, dim, _emb, _out, _enc1, _enc2, **kw):
super(Model, self).__init__(**kw)
self.dim, self.emb, self.out, self.enc1, self.enc2 = dim, _emb, _out, _enc1, _enc2
self.score = torch.nn.Sequential(
torch.nn.Linear(dim, 1, bias=False), torch.nn.Sigmoid())
self.emb_expander = ExpandVecs(embdim, enc2inpdim, 2)
self.enc_expander = ExpandVecs(encdim * 2, enc2inpdim, 2)
def forward(self, x, with_att=False):
# embed and encode
xemb, xmask = self.emb(x)
xenc = self.enc1(xemb, mask=xmask)
# compute attention
xatt = self.score(xenc).squeeze(
2) * xmask.float()[:, :xenc.size(1)]
# encode again
_xemb = self.emb_expander(xemb[:, :xenc.size(1)])
_xenc = self.enc_expander(xenc)
_, xenc2 = self.enc2(_xemb,
gate=xatt,
mask=xmask[:, :xenc.size(1)],
ret_states=True)
scores = self.out(xenc2.view(xenc.size(0), -1))
if with_att:
return scores, xatt
else:
return scores
model = Model(40, emb, out, enc1, enc2)
# endregion
# region test
if test:
inps = torch.tensor(sm.matrix[0:2])
outs = model(inps)
# endregion
# region train
optimizer = torch.optim.Adam(q.params_of(model), lr=lr)
trainer = q.trainer(model).on(data).loss(torch.nn.CrossEntropyLoss(), q.Accuracy())\
.optimizer(optimizer).hook(q.ClipGradNorm(5.)).device(device)
validator = q.tester(model).on(valid_data).loss(
q.Accuracy()).device(device)
q.train(trainer, validator).run(epochs=epochs)
# endregion
# region check attention #TODO
# feed a batch
inpd = torch.tensor(sm.matrix[400:410])
outd, att = model(inpd, with_att=True)
outd = torch.max(outd, 1)[1].cpu().detach().numpy()
inpd = inpd.cpu().detach().numpy()
att = att.cpu().detach().numpy()
rD = {v: k for k, v in sm.D.items()}
roD = {v: k for k, v in D.items()}
for i in range(len(att)):
inpdi = " ".join([rD[x] for x in inpd[i]])
outdi = roD[outd[i]]
print("input: {}\nattention: {}\nprediction: {}".format(
inpdi, " ".join(["{:.1f}".format(x) for x in att[i]]), outdi))
def run(lr=OPT_LR,
batsize=100,
epochs=1000,
validinter=20,
wreg=0.00000000001,
dropout=0.1,
embdim=50,
encdim=50,
numlayers=1,
cuda=False,
gpu=0,
mode="flat",
test=False,
gendata=False):
if gendata:
loadret = load_jsons()
pickle.dump(loadret,
open("loadcache.flat.pkl", "w"),
protocol=pickle.HIGHEST_PROTOCOL)
else:
settings = locals().copy()
logger = q.Logger(prefix="rank_lstm")
logger.save_settings(**settings)
device = torch.device("cpu")
if cuda:
device = torch.device("cuda", gpu)
tt = q.ticktock("script")
# region DATA
tt.tick("loading data")
qsm, csm, goldchainids, badchainids = pickle.load(
open("loadcache.{}.pkl".format(mode)))
eids = np.arange(0, len(goldchainids))
data = [qsm.matrix, eids]
traindata, validdata = q.datasplit(data, splits=(7, 3), random=False)
validdata, testdata = q.datasplit(validdata,
splits=(1, 2),
random=False)
trainloader = q.dataload(*traindata, batch_size=batsize, shuffle=True)
input_feeder = FlatInpFeeder(csm.matrix, goldchainids, badchainids)
def inp_bt(_qsm_batch, _eids_batch):
golds_batch, bads_batch = input_feeder(_eids_batch)
return _qsm_batch, golds_batch, bads_batch
if test:
# test input feeder
eids = q.var(torch.arange(0, 10).long()).v
_test_golds_batch, _test_bads_batch = input_feeder(eids)
tt.tock("data loaded")
# endregion
# region MODEL
dims = [encdim // 2] * numlayers
question_encoder = FlatEncoder(embdim, dims, qsm.D, bidir=True)
query_encoder = FlatEncoder(embdim, dims, csm.D, bidir=True)
similarity = DotDistance()
rankmodel = RankModel(question_encoder, query_encoder, similarity)
scoremodel = ScoreModel(question_encoder, query_encoder, similarity)
# endregion
# region VALIDATION
rankcomp = RankingComputer(scoremodel, validdata[1], validdata[0],
csm.matrix, goldchainids, badchainids)
# endregion
# region TRAINING
optim = torch.optim.Adam(q.params_of(rankmodel),
lr=lr,
weight_decay=wreg)
trainer = q.trainer(rankmodel).on(trainloader).loss(1)\
.set_batch_transformer(inp_bt).optimizer(optim).device(device)
def validation_function():
rankmetrics = rankcomp.compute(RecallAt(1, totaltrue=1),
RecallAt(5, totaltrue=1), MRR())
ret = []
for rankmetric in rankmetrics:
rankmetric = np.asarray(rankmetric)
ret_i = rankmetric.mean()
ret.append(ret_i)
return "valid: " + " - ".join(["{:.4f}".format(x) for x in ret])
q.train(trainer, validation_function).run(epochs,
validinter=validinter)
def run(lr=0.01,
epochs=10,
batsize=64,
momentum=0.5,
cuda=False,
gpu=0,
seed=1):
settings = locals().copy()
logger = q.Logger(prefix="mnist")
logger.save_settings(**settings)
torch.manual_seed(seed)
if cuda:
torch.cuda.set_device(gpu)
torch.cuda.manual_seed(seed)
kwargs = {}
train_loader = torch.utils.data.DataLoader(datasets.MNIST(
'../../datasets/mnist',
train=True,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])),
batch_size=batsize,
shuffle=True,
**kwargs)
test_loader = torch.utils.data.DataLoader(datasets.MNIST(
'../../datasets/mnist',
train=False,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])),
batch_size=batsize,
shuffle=False,
**kwargs)
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
self.conv2_drop = nn.Dropout2d()
self.fc1 = nn.Linear(320, 50)
self.fc2 = nn.Linear(50, 10)
def forward(self, x):
x = F.relu(F.max_pool2d(self.conv1(x), 2))
x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
x = x.view(-1, 320)
x = F.relu(self.fc1(x))
x = F.dropout(x, training=self.training)
x = self.fc2(x)
return F.log_softmax(x, dim=1)
model = Net()
optim = torch.optim.SGD(model.parameters(), lr=lr, momentum=momentum)
trainer = q.trainer(model).on(train_loader)\
.loss(torch.nn.NLLLoss(), q.Accuracy())\
.optimizer(optim).cuda(cuda)
validator = q.tester(model).on(test_loader)\
.loss(torch.nn.NLLLoss(), q.Accuracy())\
.cuda(cuda)
logger.loglosses(trainer, "train.losses")
logger.loglosses(validator, "valid.losses")
q.train(trainer, validator).run(epochs)