Esempio n. 1
0
def run(lr=0.001):
    x = np.random.random((1000, 5)).astype("float32")
    y = np.random.randint(0, 5, (1000, )).astype("int64")

    trainloader = q.dataload(x[:800], y[:800], batch_size=100)
    validloader = q.dataload(x[800:], y[800:], batch_size=100)

    m = torch.nn.Sequential(torch.nn.Linear(5, 100), torch.nn.Linear(100, 5))

    m[1].weight.requires_grad = False

    losses = q.lossarray(torch.nn.CrossEntropyLoss())

    params = m.parameters()
    for param in params:
        print(param.requires_grad)

    init_val = m[1].weight.detach().numpy()

    optim = torch.optim.Adam(q.params_of(m), lr=lr)

    trainer = q.trainer(m).on(trainloader).loss(losses).optimizer(
        optim).epochs(100)

    # for b, (i, e) in trainer.inf_batches():
    #     print(i, e)

    validator = q.tester(m).on(validloader).loss(losses)

    q.train(trainer, validator).run()

    new_val = m[1].weight.detach().numpy()

    print(np.linalg.norm(new_val - init_val))
Esempio n. 2
0
def run(lr=20.,
        dropout=0.2,
        dropconnect=0.2,
        gradnorm=0.25,
        epochs=25,
        embdim=200,
        encdim=200,
        numlayers=2,
        seqlen=35,
        batsize=20,
        eval_batsize=10,
        cuda=False,
        gpu=0,
        test=False):
    tt = q.ticktock("script")
    device = torch.device("cpu")
    if cuda:
        device = torch.device("cuda", gpu)
    tt.tick("loading data")
    train_batches, valid_batches, test_batches, D = \
        load_data(batsize=batsize, eval_batsize=eval_batsize, seqlen=seqlen)
    tt.tock("data loaded")
    print("{} batches in train".format(len(train_batches)))

    tt.tick("creating model")
    dims = [embdim] + ([encdim] * numlayers)
    m = RNNLayer_LM(*dims, worddic=D, dropout=dropout)

    if test:
        for i, batch in enumerate(train_batches):
            y = m(batch[0])
            if i > 5:
                break
        print(y.size())

    loss = q.SeqKLLoss(time_average=True, size_average=True, mode="logits")
    ppl_loss = q.SeqPPL_Loss(time_average=True,
                             size_average=True,
                             mode="logits")

    optim = torch.optim.SGD(q.params_of(m), lr=lr)
    gradclip = q.ClipGradNorm(gradnorm)

    trainer = q.trainer(m).on(train_batches).loss(loss).optimizer(
        optim).device(device).hook(m).hook(gradclip)
    tester = q.tester(m).on(valid_batches).loss(
        loss, ppl_loss).device(device).hook(m)

    tt.tock("created model")
    tt.tick("training")
    q.train(trainer, tester).run(epochs=epochs)
    tt.tock("trained")
def run_classify(lr=0.001,
                 seqlen=6,
                 numex=500,
                 epochs=25,
                 batsize=10,
                 test=True,
                 cuda=False,
                 gpu=0):
    device = torch.device("cpu")
    if cuda:
        device = torch.device("cuda", gpu)
    # region construct data
    colors = "red blue green magenta cyan orange yellow grey salmon pink purple teal".split(
    )
    D = dict(zip(colors, range(len(colors))))
    inpseqs = []
    targets = []
    for i in range(numex):
        inpseq = list(np.random.choice(colors, seqlen, replace=False))
        target = np.random.choice(range(len(inpseq)), 1)[0]
        target_class = D[inpseq[target]]
        inpseq[target] = "${}$".format(inpseq[target])
        inpseqs.append("".join(inpseq))
        targets.append(target_class)

    sm = q.StringMatrix()
    sm.tokenize = lambda x: list(x)

    for inpseq in inpseqs:
        sm.add(inpseq)

    sm.finalize()
    print(sm[0])
    print(sm.D)
    targets = np.asarray(targets)

    data = q.dataload(sm.matrix[:-100], targets[:-100], batch_size=batsize)
    valid_data = q.dataload(sm.matrix[-100:],
                            targets[-100:],
                            batch_size=batsize)
    # endregion

    # region model
    embdim = 20
    enc2inpdim = 45
    encdim = 20
    outdim = 20
    emb = q.WordEmb(embdim, worddic=sm.D)  # sm dictionary (characters)
    out = q.WordLinout(outdim, worddic=D)  # target dictionary
    # encoders:
    enc1 = q.RNNEncoder(embdim, encdim, bidir=True)
    enc2 = q.RNNCellEncoder(enc2inpdim, outdim // 2, bidir=True)

    # model
    class Model(torch.nn.Module):
        def __init__(self, dim, _emb, _out, _enc1, _enc2, **kw):
            super(Model, self).__init__(**kw)
            self.dim, self.emb, self.out, self.enc1, self.enc2 = dim, _emb, _out, _enc1, _enc2
            self.score = torch.nn.Sequential(
                torch.nn.Linear(dim, 1, bias=False), torch.nn.Sigmoid())
            self.emb_expander = ExpandVecs(embdim, enc2inpdim, 2)
            self.enc_expander = ExpandVecs(encdim * 2, enc2inpdim, 2)

        def forward(self, x, with_att=False):
            # embed and encode
            xemb, xmask = self.emb(x)
            xenc = self.enc1(xemb, mask=xmask)
            # compute attention
            xatt = self.score(xenc).squeeze(
                2) * xmask.float()[:, :xenc.size(1)]
            # encode again
            _xemb = self.emb_expander(xemb[:, :xenc.size(1)])
            _xenc = self.enc_expander(xenc)
            _, xenc2 = self.enc2(_xemb,
                                 gate=xatt,
                                 mask=xmask[:, :xenc.size(1)],
                                 ret_states=True)
            scores = self.out(xenc2.view(xenc.size(0), -1))
            if with_att:
                return scores, xatt
            else:
                return scores

    model = Model(40, emb, out, enc1, enc2)
    # endregion

    # region test
    if test:
        inps = torch.tensor(sm.matrix[0:2])
        outs = model(inps)
    # endregion

    # region train
    optimizer = torch.optim.Adam(q.params_of(model), lr=lr)
    trainer = q.trainer(model).on(data).loss(torch.nn.CrossEntropyLoss(), q.Accuracy())\
        .optimizer(optimizer).hook(q.ClipGradNorm(5.)).device(device)
    validator = q.tester(model).on(valid_data).loss(
        q.Accuracy()).device(device)
    q.train(trainer, validator).run(epochs=epochs)
    # endregion

    # region check attention    #TODO
    # feed a batch
    inpd = torch.tensor(sm.matrix[400:410])
    outd, att = model(inpd, with_att=True)
    outd = torch.max(outd, 1)[1].cpu().detach().numpy()
    inpd = inpd.cpu().detach().numpy()
    att = att.cpu().detach().numpy()
    rD = {v: k for k, v in sm.D.items()}
    roD = {v: k for k, v in D.items()}
    for i in range(len(att)):
        inpdi = "   ".join([rD[x] for x in inpd[i]])
        outdi = roD[outd[i]]
        print("input:     {}\nattention: {}\nprediction: {}".format(
            inpdi, " ".join(["{:.1f}".format(x) for x in att[i]]), outdi))
Esempio n. 4
0
def run(lr=OPT_LR,
        batsize=100,
        epochs=1000,
        validinter=20,
        wreg=0.00000000001,
        dropout=0.1,
        embdim=50,
        encdim=50,
        numlayers=1,
        cuda=False,
        gpu=0,
        mode="flat",
        test=False,
        gendata=False):
    if gendata:
        loadret = load_jsons()
        pickle.dump(loadret,
                    open("loadcache.flat.pkl", "w"),
                    protocol=pickle.HIGHEST_PROTOCOL)
    else:
        settings = locals().copy()
        logger = q.Logger(prefix="rank_lstm")
        logger.save_settings(**settings)

        device = torch.device("cpu")
        if cuda:
            device = torch.device("cuda", gpu)

        tt = q.ticktock("script")

        # region DATA
        tt.tick("loading data")
        qsm, csm, goldchainids, badchainids = pickle.load(
            open("loadcache.{}.pkl".format(mode)))
        eids = np.arange(0, len(goldchainids))

        data = [qsm.matrix, eids]
        traindata, validdata = q.datasplit(data, splits=(7, 3), random=False)
        validdata, testdata = q.datasplit(validdata,
                                          splits=(1, 2),
                                          random=False)

        trainloader = q.dataload(*traindata, batch_size=batsize, shuffle=True)

        input_feeder = FlatInpFeeder(csm.matrix, goldchainids, badchainids)

        def inp_bt(_qsm_batch, _eids_batch):
            golds_batch, bads_batch = input_feeder(_eids_batch)
            return _qsm_batch, golds_batch, bads_batch

        if test:
            # test input feeder
            eids = q.var(torch.arange(0, 10).long()).v
            _test_golds_batch, _test_bads_batch = input_feeder(eids)
        tt.tock("data loaded")
        # endregion

        # region MODEL
        dims = [encdim // 2] * numlayers

        question_encoder = FlatEncoder(embdim, dims, qsm.D, bidir=True)
        query_encoder = FlatEncoder(embdim, dims, csm.D, bidir=True)
        similarity = DotDistance()

        rankmodel = RankModel(question_encoder, query_encoder, similarity)
        scoremodel = ScoreModel(question_encoder, query_encoder, similarity)
        # endregion

        # region VALIDATION
        rankcomp = RankingComputer(scoremodel, validdata[1], validdata[0],
                                   csm.matrix, goldchainids, badchainids)
        # endregion

        # region TRAINING
        optim = torch.optim.Adam(q.params_of(rankmodel),
                                 lr=lr,
                                 weight_decay=wreg)
        trainer = q.trainer(rankmodel).on(trainloader).loss(1)\
                   .set_batch_transformer(inp_bt).optimizer(optim).device(device)

        def validation_function():
            rankmetrics = rankcomp.compute(RecallAt(1, totaltrue=1),
                                           RecallAt(5, totaltrue=1), MRR())
            ret = []
            for rankmetric in rankmetrics:
                rankmetric = np.asarray(rankmetric)
                ret_i = rankmetric.mean()
                ret.append(ret_i)
            return "valid: " + " - ".join(["{:.4f}".format(x) for x in ret])

        q.train(trainer, validation_function).run(epochs,
                                                  validinter=validinter)
def run(lr=0.01,
        epochs=10,
        batsize=64,
        momentum=0.5,
        cuda=False,
        gpu=0,
        seed=1):
    settings = locals().copy()
    logger = q.Logger(prefix="mnist")
    logger.save_settings(**settings)

    torch.manual_seed(seed)
    if cuda:
        torch.cuda.set_device(gpu)
        torch.cuda.manual_seed(seed)

    kwargs = {}
    train_loader = torch.utils.data.DataLoader(datasets.MNIST(
        '../../datasets/mnist',
        train=True,
        download=True,
        transform=transforms.Compose([
            transforms.ToTensor(),
            transforms.Normalize((0.1307, ), (0.3081, ))
        ])),
                                               batch_size=batsize,
                                               shuffle=True,
                                               **kwargs)
    test_loader = torch.utils.data.DataLoader(datasets.MNIST(
        '../../datasets/mnist',
        train=False,
        transform=transforms.Compose([
            transforms.ToTensor(),
            transforms.Normalize((0.1307, ), (0.3081, ))
        ])),
                                              batch_size=batsize,
                                              shuffle=False,
                                              **kwargs)

    class Net(nn.Module):
        def __init__(self):
            super(Net, self).__init__()
            self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
            self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
            self.conv2_drop = nn.Dropout2d()
            self.fc1 = nn.Linear(320, 50)
            self.fc2 = nn.Linear(50, 10)

        def forward(self, x):
            x = F.relu(F.max_pool2d(self.conv1(x), 2))
            x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
            x = x.view(-1, 320)
            x = F.relu(self.fc1(x))
            x = F.dropout(x, training=self.training)
            x = self.fc2(x)
            return F.log_softmax(x, dim=1)

    model = Net()

    optim = torch.optim.SGD(model.parameters(), lr=lr, momentum=momentum)

    trainer = q.trainer(model).on(train_loader)\
        .loss(torch.nn.NLLLoss(), q.Accuracy())\
        .optimizer(optim).cuda(cuda)
    validator = q.tester(model).on(test_loader)\
        .loss(torch.nn.NLLLoss(), q.Accuracy())\
        .cuda(cuda)

    logger.loglosses(trainer, "train.losses")
    logger.loglosses(validator, "valid.losses")

    q.train(trainer, validator).run(epochs)