示例#1
0
文件: test.py 项目: srifai/senna
def score(jobman, path):
    hp = jobman.state
    nsenna = 30000

    PATH = "/scratch/rifaisal/msrtest/test/"
    delta = hp["wsize"] / 2
    rest = hp["wsize"] % 2
    sent = T.matrix()

    embedding = cae(i_size=nsenna, h_size=hp["embedsize"], e_act=identity)
    H = ae(i_size=hp["embedsize"] * hp["wsize"], h_size=hp["hsize"], e_act=T.tanh)
    L = logistic(i_size=hp["hsize"], h_size=1, act=identity)

    load(embedding, path + "/embedding.pkl")
    load(H, path + "/hidden.pkl")
    load(L, path + "/logistic.pkl")

    posit_embed = T.dot(sent, embedding.params["e_weights"]).reshape((1, hp["embedsize"] * hp["wsize"]))
    posit_score = H.encode(posit_embed)
    scoreit = theano.function([sent], posit_score)
    sentences = parse_data()
    scores = []
    esims = []
    msim = []
    hsim = []
    Em = embedding.params["e_weights"].get_value(borrow=True)
    for i, (sc, w1, w2, c1, c2) in enumerate(sentences):
        sys.stdout.flush()

        c1 = [29999] * 10 + c1 + [29999] * 10
        c2 = [29999] * 10 + c2 + [29999] * 10

        w1seqs = [c1[10 + idx - delta : 10 + idx + delta + rest] for idx in w1]
        w2seqs = [c2[10 + idx - delta : 10 + idx + delta + rest] for idx in w2]

        c = []

        w1em = Em[c1[10 + w1[0]]]
        w2em = Em[c2[10 + w2[0]]]

        w1sc = numpy.concatenate([scoreit(idx2mat(w1seqs[0], nsenna)).flatten(), Em[c1[10 + w1[0]]]])
        w2sc = numpy.concatenate([scoreit(idx2mat(w2seqs[0], nsenna)).flatten(), Em[c2[10 + w2[0]]]])

        metric = L.params["weights"].get_value(borrow=True).flatten()

        sim = -(((w1sc - w2sc)) ** 2).sum()
        esim = -((w1em - w2em) ** 2).sum()

        msim.append(sim)
        esims.append(esim)
        hsim.append(numpy.mean(sc))

    print "Model:", scipy.stats.spearmanr(numpy.array(hsim), numpy.array(msim))[
        0
    ], ", Embeddings:", scipy.stats.spearmanr(numpy.array(hsim), numpy.array(esims))[0]
示例#2
0
def run(jobman,debug = False):
    expstart = time.time()
    hp = jobman.state

    # Symbolic variables
    s_bow = T.matrix()
    s_posit = T.matrix()#theano.sparse.csr_matrix()
    s_negat = T.matrix()#theano.sparse.csr_matrix()

    sentences = cPickle.load(open('/scratch/rifaisal/data/guten/guten_subset_idx.pkl'))

    senna = cPickle.load(open('/scratch/rifaisal/data/guten/senna.pkl'))
    gsubset = cPickle.load(open('/scratch/rifaisal/data/guten/guten_vocab_subset.pkl')).flatten().tolist()
    hashtab = dict( zip( gsubset, range( len( gsubset))))    

    senna = numpy.array(senna)[gsubset].tolist()
    s_valid = theano.sparse.csr_matrix()

    validsentence = sentences[-10:]
    sentences = sentences[:-10]




    nsent = len(sentences)
    nsenna = len(senna)

    # Layers
    
    embedding = cae(i_size=nsenna, h_size=hp['embedsize'], e_act = T.nnet.sigmoid)
    H = ae(i_size = hp['embedsize']*hp['wsize'], h_size=hp['hsize'], e_act = rect, d_act = hardtanh)
    L = logistic(i_size = hp['hsize'],  h_size = 1)

    valid_embedding = sparse.supervised.logistic(i_size=nsenna, h_size=hp['embedsize'], act = T.nnet.sigmoid)
    valid_embedding.params['weights'] = embedding.params['e_weights']
    valid_embedding.params['bias'] = embedding.params['e_bias']

    lr = hp['lr']
    h_size = hp['hsize']
    bs = hp['bs']

    posit_embed = embedding.encode(s_posit).reshape((1,hp['embedsize']*hp['wsize']))
    negat_embed = embedding.encode(s_negat).reshape((hp['nneg'],hp['embedsize']*hp['wsize']))
    valid_embed = valid_embedding.encode(s_valid).reshape((nsenna,hp['embedsize']*hp['wsize']))

    posit_score = L.encode(H.encode(posit_embed))
    negat_score = L.encode(H.encode(negat_embed))
    valid_score = L.encode(H.encode(valid_embed))

    C = (negat_score - posit_score.flatten() + hp['margin'])

    rec = embedding.reconstruct(s_bow, loss='ce')
    CC = (rect(C)).mean() + hp['lambda'] * rec

    opt = theano.function([s_posit, s_negat, s_bow], 
                          [C.mean(),rec], 
                          updates = dict( L.update(CC,lr) + H.update(CC,lr) + embedding.update(CC,lr)) )

    validfct = theano.function([s_valid],valid_score)

    def saveexp():
        save(embedding,fname+'embedding.pkl')
        save(H,fname+'hidden.pkl')
        save(L,fname+'logistic.pkl')
        print 'Saved successfully'

    delta = hp['wsize']/2
    rest = hp['wsize']%2

    freq_idx = cPickle.load(open('/scratch/rifaisal/data/guten/gutten_sorted_vocab.pkl'))[:1000]
    freq_idx =  [ hashtab[idx] for idx in freq_idx ]

    fname = sys.argv[0]+'_'
    
    for e in range(hp['epoch']):
        c = []
        r = []
        for i in range(nsent):
            rsent = numpy.random.randint(nsent-1)
            nword = len(sentences[rsent])
            if nword < hp['wsize'] + 2:
                continue

            pidx = numpy.random.randint(low = delta, high = nword-delta)
            pchunk = sentences[rsent][pidx-delta:pidx+delta+rest]
            nchunk = []
            st = sentences[rsent][pidx-delta:pidx]
            en = sentences[rsent][pidx+1:pidx+delta+rest]
            rndidx = numpy.random.randint(nsenna, size = (hp['nneg'],))
            nchunk = []
            for j in range(hp['nneg']):
                nchunk += en + [rndidx[j]] + st


            assert len(nchunk) == len(pchunk)*hp['nneg']

            p, n, b = (idx2mat(pchunk,nsenna), idx2mat(nchunk,nsenna), idx2vec(sentences[rsent],nsenna))

            l,g = opt(p,n,b)
            c.append(l)
            r.append(g)
            
            if (time.time() - expstart) > ( 3600 * 24 * 6 + 3600*20) or (i+1)%(50*hp['freq']) == 0:
                mrk = evaluation.error(validsentence, validfct, nsenna, hp['wsize'])
                hp['mrk'] = mrk
                hp['e'] = e
                hp['i'] = i
                jobman.save()
                saveexp()
                print 'Random Valid Mean rank',mrk

            if i%hp['freq'] == 0:
                hp['score'] = numpy.array(c).mean()
                hp['rec'] = numpy.array(r).mean()
                print e,i,'NN Score:', hp['score'], 'Reconstruction:', hp['rec']

                ne = knn(freq_idx,embedding.params['e_weights'].get_value(borrow=True))
                open('files/'+fname+'nearest.txt','w').write(display(ne,senna))

                saveexp()
                sys.stdout.flush()
                jobman.save()
                
    save()
示例#3
0
def run(jobman, debug=False):
    expstart = time.time()
    hp = jobman.state

    # Symbolic variables
    s_bow = T.matrix()
    s_posit = T.matrix()  #theano.sparse.csr_matrix()
    s_negat = T.matrix()  #theano.sparse.csr_matrix()

    sentences = cPickle.load(
        open('/scratch/rifaisal/data/guten/guten_subset_idx.pkl'))

    senna = cPickle.load(open('/scratch/rifaisal/data/guten/senna.pkl'))
    gsubset = cPickle.load(
        open('/scratch/rifaisal/data/guten/guten_vocab_subset.pkl')).flatten(
        ).tolist()
    hashtab = dict(zip(gsubset, range(len(gsubset))))

    senna = numpy.array(senna)[gsubset].tolist()
    s_valid = theano.sparse.csr_matrix()

    validsentence = sentences[-10:]
    sentences = sentences[:-10]

    nsent = len(sentences)
    nsenna = len(senna)

    # Layers

    embedding = cae(i_size=nsenna,
                    h_size=hp['embedsize'],
                    e_act=T.nnet.sigmoid)
    H = ae(i_size=hp['embedsize'] * hp['wsize'],
           h_size=hp['hsize'],
           e_act=rect,
           d_act=hardtanh)
    L = logistic(i_size=hp['hsize'], h_size=1)

    valid_embedding = sparse.supervised.logistic(i_size=nsenna,
                                                 h_size=hp['embedsize'],
                                                 act=T.nnet.sigmoid)
    valid_embedding.params['weights'] = embedding.params['e_weights']
    valid_embedding.params['bias'] = embedding.params['e_bias']

    lr = hp['lr']
    h_size = hp['hsize']
    bs = hp['bs']

    posit_embed = embedding.encode(s_posit).reshape(
        (1, hp['embedsize'] * hp['wsize']))
    negat_embed = embedding.encode(s_negat).reshape(
        (hp['nneg'], hp['embedsize'] * hp['wsize']))
    valid_embed = valid_embedding.encode(s_valid).reshape(
        (nsenna, hp['embedsize'] * hp['wsize']))

    posit_score = L.encode(H.encode(posit_embed))
    negat_score = L.encode(H.encode(negat_embed))
    valid_score = L.encode(H.encode(valid_embed))

    C = (negat_score - posit_score.flatten() + hp['margin'])

    rec = embedding.reconstruct(s_bow, loss='ce')
    CC = (rect(C)).mean() + hp['lambda'] * rec

    opt = theano.function([s_posit, s_negat, s_bow], [C.mean(), rec],
                          updates=dict(
                              L.update(CC, lr) + H.update(CC, lr) +
                              embedding.update(CC, lr)))

    validfct = theano.function([s_valid], valid_score)

    def saveexp():
        save(embedding, fname + 'embedding.pkl')
        save(H, fname + 'hidden.pkl')
        save(L, fname + 'logistic.pkl')
        print 'Saved successfully'

    delta = hp['wsize'] / 2
    rest = hp['wsize'] % 2

    freq_idx = cPickle.load(
        open('/scratch/rifaisal/data/guten/gutten_sorted_vocab.pkl'))[:1000]
    freq_idx = [hashtab[idx] for idx in freq_idx]

    fname = sys.argv[0] + '_'

    for e in range(hp['epoch']):
        c = []
        r = []
        for i in range(nsent):
            rsent = numpy.random.randint(nsent - 1)
            nword = len(sentences[rsent])
            if nword < hp['wsize'] + 2:
                continue

            pidx = numpy.random.randint(low=delta, high=nword - delta)
            pchunk = sentences[rsent][pidx - delta:pidx + delta + rest]
            nchunk = []
            st = sentences[rsent][pidx - delta:pidx]
            en = sentences[rsent][pidx + 1:pidx + delta + rest]
            rndidx = numpy.random.randint(nsenna, size=(hp['nneg'], ))
            nchunk = []
            for j in range(hp['nneg']):
                nchunk += en + [rndidx[j]] + st

            assert len(nchunk) == len(pchunk) * hp['nneg']

            p, n, b = (idx2mat(pchunk, nsenna), idx2mat(nchunk, nsenna),
                       idx2vec(sentences[rsent], nsenna))

            l, g = opt(p, n, b)
            c.append(l)
            r.append(g)

            if (time.time() - expstart) > (3600 * 24 * 6 + 3600 * 20) or (
                    i + 1) % (50 * hp['freq']) == 0:
                mrk = evaluation.error(validsentence, validfct, nsenna,
                                       hp['wsize'])
                hp['mrk'] = mrk
                hp['e'] = e
                hp['i'] = i
                jobman.save()
                saveexp()
                print 'Random Valid Mean rank', mrk

            if i % hp['freq'] == 0:
                hp['score'] = numpy.array(c).mean()
                hp['rec'] = numpy.array(r).mean()
                print e, i, 'NN Score:', hp['score'], 'Reconstruction:', hp[
                    'rec']

                ne = knn(freq_idx,
                         embedding.params['e_weights'].get_value(borrow=True))
                open('files/' + fname + 'nearest.txt',
                     'w').write(display(ne, senna))

                saveexp()
                sys.stdout.flush()
                jobman.save()

    save()
示例#4
0
def run(jobman, debug=False):
    expstart = time.time()
    hp = jobman.state

    if not os.path.exists('files/'): os.mkdir('files/')

    # Symbolic variables
    s_bow = T.matrix()
    s_idx = T.iscalar()
    s_tf = T.scalar()
    s_posit = T.matrix()  #theano.sparse.csr_matrix()
    s_negat = T.matrix()  #theano.sparse.csr_matrix()

    sentences = cPickle.load(
        open('/scratch/rifaisal/data/guten/guten_subset_idx.pkl'))

    senna = cPickle.load(open('/scratch/rifaisal/data/guten/senna.pkl'))
    gsubset = cPickle.load(
        open('/scratch/rifaisal/data/guten/guten_vocab_subset.pkl')).flatten(
        ).tolist()
    hashtab = dict(zip(gsubset, range(len(gsubset))))

    tfidf_data = numpy.load('/scratch/rifaisal/data/guten/guten_tfidf.npy'
                            ).item().tocsr().astype('float32')

    #tfidf = cPickle.load(open('/scratch/rifaisal/repos/senna/gutentokenizer.pkl'))

    senna = numpy.array(senna)[gsubset].tolist()
    s_valid = theano.sparse.csr_matrix()

    validsentence = sentences[10000:10010]

    nsent = len(sentences)
    nsenna = len(senna)

    # Layers

    embedding = cae(i_size=nsenna, h_size=hp['embedsize'], e_act=identity)

    H = ae(i_size=hp['embedsize'] * hp['wsize'],
           h_size=hp['hsize'],
           e_act=T.tanh)
    L = logistic(i_size=hp['hsize'], h_size=1, act=identity)
    S = logistic(i_size=hp['embedsize'], h_size=nsenna, act=T.nnet.softmax)

    valid_embedding = sparse.supervised.logistic(i_size=nsenna,
                                                 h_size=hp['embedsize'],
                                                 act=identity)
    valid_embedding.params['weights'] = sp.shared(
        value=scipy.sparse.csr_matrix(embedding.params['e_weights'].get_value(
            borrow=True)))
    valid_embedding.params['bias'] = embedding.params['e_bias']

    lr = hp['lr']
    h_size = hp['hsize']
    bs = hp['bs']

    posit_embed = T.dot(s_posit, embedding.params['e_weights']).reshape(
        (1, hp['embedsize'] * hp['wsize']))
    negat_embed = T.dot(s_negat, embedding.params['e_weights']).reshape(
        (hp['nneg'], hp['embedsize'] * hp['wsize']))
    valid_embed = sp.dot(s_valid, valid_embedding.params['weights']).reshape(
        (nsenna, hp['embedsize'] * hp['wsize']))

    posit_score = L.encode(H.encode(posit_embed))
    negat_score = L.encode(H.encode(negat_embed))
    valid_score = L.encode(H.encode(valid_embed))

    C = (negat_score - posit_score.flatten() + hp['margin'])

    s_bow_pred = S.encode(embedding.encode(s_bow))

    pred = s_tf * nllsoft(s_bow_pred, s_idx)

    CC = (rect(C)).mean() + hp['lambda'] * pred

    opt = theano.function(
        [s_posit, s_negat, s_bow, s_idx, s_tf], [(rect(C)).mean(), pred],
        updates=dict(
            S.update(CC, lr) + L.update(CC, lr) + H.update(CC, lr) +
            embedding.update_norm(CC, lr)))

    #validfct = theano.function([s_valid],valid_score)

    def saveexp():
        save(embedding, fname + 'embedding.pkl')
        save(H, fname + 'hidden.pkl')
        save(L, fname + 'logistic.pkl')

    delta = hp['wsize'] / 2
    rest = hp['wsize'] % 2

    freq_idx = cPickle.load(
        open('/scratch/rifaisal/data/guten/gutten_sorted_vocab.pkl'))[:1000]
    freq_idx = [hashtab[idx] for idx in freq_idx]

    fname = ''

    for e in range(hp['epoch']):
        c = []
        r = []
        count = 1
        for i in range(nsent):
            rsent = numpy.random.randint(nsent - 1)
            nword = len(sentences[rsent])
            if nword < hp['wsize'] + 2:
                continue

            pidx = numpy.random.randint(low=delta, high=nword - delta)
            pchunk = sentences[rsent][pidx - delta:pidx + delta + rest]
            nchunk = []
            st = sentences[rsent][pidx - delta:pidx]
            en = sentences[rsent][pidx + 1:pidx + delta + rest]
            rndidx = numpy.random.randint(nsenna, size=(hp['nneg'], ))
            nchunk = []
            for j in range(hp['nneg']):
                nchunk += en + [rndidx[j]] + st

            assert len(nchunk) == len(pchunk) * hp['nneg']
            tfidf_chunk = tfidf_data[rsent:rsent + 1].toarray()
            #pdb.set_trace()
            tfidf_value = tfidf_chunk[0, sentences[rsent][pidx]]
            tfidf_chunk[0, sentences[rsent][pidx]] = 0.
            tfidx = sentences[rsent][
                pidx]  # numpy.zeros(tfidf_chunk.shape).astype('float32')
            #tfidx[0,sentences[rsent][pidx]] = 1.
            p, n, b, iidx, tfval = (idx2mat(pchunk,
                                            nsenna), idx2mat(nchunk, nsenna),
                                    tfidf_chunk, tfidx, tfidf_value)
            count += tfval != 0
            l, g = opt(p, n, b, iidx, tfval)
            c = c
            c.append(l)
            r.append(g)
            """
            if (time.time() - expstart) > ( 3600 * 24 * 6 + 3600*20) or (i+1)%(20*hp['freq']) == 0 and debug==False:
                valid_embedding.params['weights'] = sp.shared(value = scipy.sparse.csr_matrix(embedding.params['e_weights'].get_value(borrow=True)))
                mrk = evaluation.error(validsentence, validfct, nsenna, hp['wsize'])
                hp['mrk'] = mrk
                jobman.save()
                saveexp()
                print 'Random Valid Mean rank',mrk
            """

            if (i + 1) % hp['freq'] == 0 or debug:
                hp['score'] = numpy.array(c).sum() / (numpy.array(c) > 0).sum()
                hp['pred'] = numpy.array(r).sum() / float(count)
                hp['e'] = e
                hp['i'] = i
                print ''
                print e, i, 'NN Score:', hp['score'], 'Reconstruction:', hp[
                    'pred']

                if debug != True:
                    ne = knn(
                        freq_idx,
                        embedding.params['e_weights'].get_value(borrow=True))
                    open('files/' + fname + 'nearest.txt',
                         'w').write(display(ne, senna))
                    saveexp()
                sys.stdout.flush()
                jobman.save()

    saveexp()
示例#5
0
文件: parsetest.py 项目: srifai/senna
def msrerror(vocab, jobman):
    hp = jobman.state
    nsenna = 30000

    PATH = '/scratch/rifaisal/msrtest/test/'
    delta = hp['wsize'] / 2
    rest = hp['wsize'] % 2
    sent = T.matrix()

    embedding = cae(i_size=nsenna, h_size=hp['embedsize'], e_act=identity)
    H = ae(i_size=hp['embedsize'] * hp['wsize'],
           h_size=hp['hsize'],
           e_act=T.tanh)
    L = logistic(i_size=hp['hsize'], h_size=1, act=identity)

    path = hp['loadpath']

    load(embedding, path + '/embedding.pkl')
    load(H, path + '/hidden.pkl')
    load(L, path + '/logistic.pkl')

    posit_embed = T.dot(sent, embedding.params['e_weights']).reshape(
        (1, hp['embedsize'] * hp['wsize']))
    posit_score = L.encode(H.encode(posit_embed))
    fct = theano.function([sent], posit_score)
    sentences = idxdataset(vocab)
    scores = []
    for i, s in enumerate(sentences):
        print i,
        sys.stdout.flush()
        nword = len(s)
        if nword < hp['wsize'] + 2:
            #print i,'Failure'
            s += [29999] * 3
        c = []
        for j in range(delta, nword - delta):
            pchunk = s[j - delta:j + delta + rest]
            p = idx2mat(pchunk, nsenna)
            l = fct(p)
            c.append(l)
        if not len(c):
            print 'pas bim'
            scores.append(0)
        else:
            scores.append(numpy.mean(c))
        #if i%5 == 0:
        #    print scores[i-5:i]

    score_template = open(PATH + 'data/Holmes.lm_format.questions.txt')
    score_output = open('energy.lm_output.txt', 'w')
    sentencelist = score_template.readlines()
    for sc, sentence in zip(scores, sentencelist):
        score_output.write(sentence.split('\n')[0] + '\t' + str(sc) + '\n')
    score_output.close()

    pipebestof5 = subprocess.Popen(
        ['perl', PATH + 'bestof5.pl', './energy.lm_output.txt'],
        stdout=subprocess.PIPE)
    energyanswer = open('./energy.answers', 'w')

    for line in pipebestof5.stdout:
        energyanswer.write(line)

    energyanswer.close()

    pipescore = subprocess.Popen([
        'perl', PATH + 'score.pl', './energy.answers',
        PATH + 'data/Holmes.lm_format.answers.txt'
    ],
                                 stdout=subprocess.PIPE)
    legend = ['correct', '%correct', 'valid', 'test']
    out = zip(legend,
              [r.split('\n')[0] for r in pipescore.stdout.readlines()[-4:]])
    res = dict(out)
    res = dict((k, float(v)) for k, v in res.iteritems())
    print res
    print out
示例#6
0
def run(jobman, debug=False):
    hp = jobman.state

    # Symbolic variables

    s_posit = T.matrix()  #theano.sparse.csr_matrix()
    s_negat = T.matrix()  #theano.sparse.csr_matrix()

    s_valid = theano.sparse.csr_matrix()

    sentences = cPickle.load(
        open('/data/lisatmp2/rifaisal/guten_subset_idx.pkl'))

    validsentence = sentences[-10:]
    sentences = sentences[:-10]
    senna = cPickle.load(open('/data/lisatmp2/rifaisal/senna.pkl'))
    gsubset = cPickle.load(
        open('/data/lisatmp2/rifaisal/guten_vocab_subset.pkl')).flatten(
        ).tolist()
    hashtab = dict(zip(gsubset, range(len(gsubset))))

    senna = numpy.array(senna)[gsubset].tolist()

    nsent = len(sentences)
    nsenna = len(senna)

    # Layers
    embedding = logistic(i_size=nsenna, h_size=hp['embedsize'], act=identity)
    H = ae(i_size=hp['embedsize'] * hp['wsize'],
           h_size=hp['hsize'],
           e_act=rect,
           d_act=hardtanh)
    L = logistic(i_size=hp['hsize'], h_size=1)  #, act = identity)

    valid_embedding = sparse.supervised.logistic(i_size=nsenna,
                                                 h_size=hp['embedsize'],
                                                 act=identity)
    #valid_embedding.params['weights'].set_value(embedding.params['weights'].get_value(borrow=True))
    #valid_embedding.params['bias'].set_value(embedding.params['bias'].get_value(borrow=True))

    lr = hp['lr']
    h_size = hp['hsize']
    bs = hp['bs']

    posit_embed = embedding.encode(s_posit).reshape(
        (1, hp['embedsize'] * hp['wsize']))
    negat_embed = embedding.encode(s_negat).reshape(
        (hp['nneg'], hp['embedsize'] * hp['wsize']))
    #valid_embed = valid_embedding.encode(s_valid).reshape((nsenna,hp['embedsize']*hp['wsize']))

    posit_score = L.encode(H.encode(posit_embed))
    negat_score = L.encode(H.encode(negat_embed))
    #valid_score = L.encode(H.encode(valid_embed))

    C = (negat_score - posit_score.flatten() + hp['margin'])

    CC = (rect(C)).mean()

    opt = theano.function([s_posit, s_negat],
                          C.mean(),
                          updates=dict(
                              L.update(CC, lr) + H.update(CC, lr) +
                              embedding.update_norm(CC, lr)))

    #validfct = theano.function([s_valid],valid_score)

    #print 'Random Valid Mean rank',evaluation.error(validsentence, validfct, nsenna, hp['wsize'])

    #load(valid_embedding,'files/gutensubsetdense_exp.py_embedding.pkl')
    load(embedding, 'files/gutensubsetdense_exp.py_embedding.pkl')
    load(H, 'files/gutensubsetdense_exp.py_hidden.pkl')
    load(L, 'files/gutensubsetdense_exp.py_logistic.pkl')

    delta = hp['wsize'] / 2
    rest = hp['wsize'] % 2

    freq_idx = cPickle.load(
        open('/data/lisatmp2/rifaisal/gutten_sorted_vocab.pkl'))[:1000]
    freq_idx = [hashtab[idx] for idx in freq_idx]

    fname = sys.argv[0] + '_'

    for e in range(hp['epoch']):
        c = []
        for i in range(nsent):
            rsent = numpy.random.randint(nsent - 1)
            nword = len(sentences[rsent])
            if nword < hp['wsize'] + 2:
                continue

            pidx = numpy.random.randint(low=delta, high=nword - delta)
            pchunk = sentences[rsent][pidx - delta:pidx + delta + rest]
            nchunk = []
            st = sentences[rsent][pidx - delta:pidx]
            en = sentences[rsent][pidx + 1:pidx + delta + rest]
            rndidx = numpy.random.randint(nsenna, size=(hp['nneg'], ))
            nchunk = []
            for j in range(hp['nneg']):
                nchunk += en + [rndidx[j]] + st

            assert len(nchunk) == len(pchunk) * hp['nneg']
            #start = time.time()
            p, n = (idx2mat(pchunk, nsenna), idx2mat(nchunk, nsenna))
            #print 'Select row:',time.time()-start,
            #start = time.time()
            c.append(opt(p, n))
            #print 'grad up:',time.time()-start

            if i % hp['freq'] == 0:
                print e, i, numpy.array(c).mean(0)
                ne = knn(freq_idx,
                         embedding.params['weights'].get_value(borrow=True))
                save(embedding, fname + 'embedding.pkl')
                save(H, fname + 'hidden.pkl')
                save(L, fname + 'logistic.pkl')
                sys.stdout.flush()
                open('files/' + fname + 'nearest.txt',
                     'w').write(display(ne, senna))

    #print 'Valid Mean rank',evaluation.error(validsentence, validfct, nsenna, hp['wsize'])
    save(embedding, fname + 'embedding.pkl')
    save(H, fname + 'hidden.pkl')
    save(L, fname + 'logistic.pkl')
示例#7
0
def run(jobman,debug = False):
    expstart = time.time()
    hp = jobman.state

    if not os.path.exists('files/'): os.mkdir('files/')

    # Symbolic variables
    s_posit = T.matrix()
    s_negat = T.matrix()
    idx_start = T.lscalar()
    idx_stop = T.lscalar()
    s_valid = theano.sparse.csr_matrix()



    w2i = cPickle.load(open('/mnt/scratch/bengio/bengio_group/data/gutenberg/merged_word2idx.pkl'))
    i2w = dict( (v,k) for k,v in w2i.iteritems() )
    i2w[0] = 'UNK'
    senna = [ i2w[i] for i in range(len(i2w.keys())) ]


    nsenna = len(senna)
    
    embedding = cae(i_size=nsenna, h_size=hp['embedsize'], e_act = identity)
    H = ae(i_size = hp['embedsize']*hp['wsize'], h_size=hp['hsize'], e_act = T.tanh)
    L = logistic(i_size = hp['hsize'], h_size = 1, act = identity)

    del H.params['d_bias']
    del embedding.params['d_bias']
    del embedding.params['e_bias']
    minsize = hp['minsize']
    maxsize = hp['maxsize']

    dsize = maxsize - minsize +1

    H.params['e_bias'] = theano.shared( numpy.array(numpy.zeros((dsize,hp['hsize'])),dtype=theano.config.floatX),name='e_bias')

    path = hp['loadpath']
 
    if path:
        load(embedding,path+'/embedding.pkl')
        #load(H,path+'/hidden.pkl')
        #load(L,path+'/logistic.pkl')
        hp['embedsize'] = embedding.params['e_weights'].get_value(borrow=True).shape[1]
        #hp['hsize'] = H.params['e_weights'].get_value(borrow=True).shape[1]
        jobman.save()

    H.params['e_bias'] = theano.shared( numpy.array(numpy.zeros((dsize,hp['hsize'])),dtype=theano.config.floatX),name='e_bias')
    valid_embedding = sparse.supervised.logistic(i_size=nsenna, h_size=hp['embedsize'], act = identity)
    valid_embedding.params['weights'] = sp.shared(value = scipy.sparse.csr_matrix(embedding.params['e_weights'].get_value(borrow=True)))


    lr = hp['lr']
    h_size = hp['hsize']
    bs = hp['bs']

    posit_embed = T.dot(s_posit, embedding.params['e_weights']).reshape((1,hp['embedsize']*hp['wsize']))
    negat_embed = T.dot(s_negat, embedding.params['e_weights']).reshape((hp['nneg'],hp['embedsize']*hp['wsize']))
    valid_embed = sp.dot(s_valid,valid_embedding.params['weights']).reshape((nsenna,hp['embedsize']*hp['wsize']))

    posit_embed_left = T.concatenate([posit_embed[:,idx_start*hp['embedsize']:idx_stop*hp['embedsize']],
                                  T.zeros_like(posit_embed[:,idx_stop*hp['embedsize']:]) ],axis=1)

    negat_embed_left = T.concatenate([negat_embed[:,idx_start*hp['embedsize']:idx_stop*hp['embedsize']],
                                   T.zeros_like(negat_embed[:,idx_stop*hp['embedsize']:]) ],axis=1)

    posit_embed_right = T.concatenate([ T.zeros_like(posit_embed[:,:idx_start*hp['embedsize']]),
                                  posit_embed[:,idx_start*hp['embedsize']:idx_stop*hp['embedsize']]],axis=1)

    negat_embed_right = T.concatenate([ T.zeros_like(negat_embed[:,:idx_start*hp['embedsize']]),
                                   negat_embed[:,idx_start*hp['embedsize']:idx_stop*hp['embedsize']]],axis=1)



    posit_embed = T.concatenate([ T.zeros_like(posit_embed[:,:idx_start*hp['embedsize']]),
                                  posit_embed[:,idx_start*hp['embedsize']:idx_stop*hp['embedsize']],
                                  T.zeros_like(posit_embed[:,idx_stop*hp['embedsize']:]) ],axis=1)

    negat_embed = T.concatenate([ T.zeros_like(negat_embed[:,:idx_start*hp['embedsize']]),
                                   negat_embed[:,idx_start*hp['embedsize']:idx_stop*hp['embedsize']],
                                   T.zeros_like(negat_embed[:,idx_stop*hp['embedsize']:]) ],axis=1)

    
    #posit_embed = ifelse(T.eq(idx_start, 0), posit_embed_left, posit_embed)
    #posit_embed = ifelse(T.eq(idx_stop, hp['maxsize']), posit_embed_right, posit_embed)

    #negat_embed = ifelse(T.eq(idx_start, 0), negat_embed_left, negat_embed)
    #negat_embed = ifelse(T.eq(idx_stop, hp['maxsize']), negat_embed_right, negat_embed)

    Hposit = T.tanh(T.dot(posit_embed,H.params['e_weights']) + H.params['e_bias'][idx_stop-idx_start-minsize,:])
    Hnegat = T.tanh(T.dot(negat_embed,H.params['e_weights']) + H.params['e_bias'][idx_stop-idx_start-minsize,:])
    posit_score = L.encode(Hposit)
    negat_score = L.encode(Hnegat)
    valid_score = L.encode(H.encode(valid_embed))

    C = (negat_score - posit_score.flatten() + hp['margin'])

    CC = (rect(C)).mean()

    opt = theano.function([s_posit, s_negat, idx_start, idx_stop],
                          (rect(C)).mean(),
                          updates = dict( L.update(CC,lr) + H.update(CC,lr) + embedding.update_norm(CC,lr)) )

    validfct = theano.function([s_valid],valid_score)

    def saveexp():
        save(embedding,fname+'embedding.pkl')
        save(H,fname+'hidden.pkl')
        save(L,fname+'logistic.pkl')

    delta = hp['wsize']/2
    rest = hp['wsize']%2

    freq_idx = cPickle.load(open('/mnt/scratch/bengio/bengio_group/data/gutenberg/sorted_vocab.pkl'))[:2000]
    fname = ''
    validsentence = []# cPickle.load(open('/scratch/rifaisal/data/wiki_april_2010/valid_debug.pkl'))
    tseenwords = not debug
    for e in range(hp['epoch']):
        hp['split'] = numpy.random.randint(45)
        sentences = cPickle.load(open('/mnt/scratch/bengio/bengio_group/data/gutenberg/ints_50000/split'+str(hp['split'])+'.pkl'))
        nsent = len(sentences)
        bigc = []
        bigr = []

        seen_words = 0
        for i,s in enumerate(sentences):
            nword = len(s)
            seen_words += nword
            tseenwords += nword

            if nword < hp['maxsize'] + 2:
                continue
            rndsize = numpy.random.randint(low=hp['minsize']+1,high=hp['maxsize']-1)
            idxsta = numpy.random.randint(low=1, high=hp['maxsize']-rndsize)
            idxsto = idxsta+rndsize

            print 'r',rndsize,'b',idxsta,'e',idxsto,'shape',H.params['e_bias'].get_value().shape

            c =[]
            r =[]
            if debug:
                print ' *** Processing document',i,'with',nword,
                sys.stdout.flush()
            for j in range(delta,nword-delta):
                nd = rndsize/2
                rd = rndsize%2
                pchunk = s[j-delta:j+delta+rest]
                nchunk = []
                
                rndidx = numpy.random.randint(nsenna, size = (hp['nneg'],))
                nchunk = []
                for kk in range(hp['nneg']):
                    tmpchunk = copy.copy(pchunk)
                    tmpchunk[idxsta+nd] = rndidx[kk]
                    nchunk += tmpchunk
                assert len(nchunk) == len(pchunk)*hp['nneg']
                p, n  = (idx2mat(pchunk,nsenna), idx2mat(nchunk,nsenna))
                l = opt(p,n, idxsta, idxsto)
                c.append(l)

                if debug:
                    print '.',
                    break


            if debug:
                print ''

            bigc += [numpy.array(c).sum()]

            if 0:#(time.time() - expstart) > ( 3600 * 24 * 6 + 3600*20) or (tseenwords)>(10*hp['freq']):
                tseenwords = 0
                valid_embedding.params['weights'] = sp.shared(value = scipy.sparse.csr_matrix(embedding.params['e_weights'].get_value(borrow=True)))
                mrk = evaluation.error(validsentence, validfct, nsenna, hp['wsize'])
                hp['mrk'] = mrk
                jobman.save()
                saveexp()
                print 'Random Valid Mean rank',mrk


            if seen_words > hp['freq'] or debug:
                seen_words = 0
                hp['score'] = numpy.array(bigc).mean() 
                hp['e'] = e
                hp['i'] = i
                print ''
                print e,i,'NN Score:', hp['score']

                if not debug:
                    ne = knn(freq_idx,embedding.params['e_weights'].get_value(borrow=True))
                    open('files/'+fname+'nearest.txt','w').write(display(ne,senna))
                    saveexp()
                sys.stdout.flush()
                jobman.save()
                
    saveexp()
示例#8
0
def run(jobman, debug=False):
    expstart = time.time()
    hp = jobman.state

    if not os.path.exists('files/'): os.mkdir('files/')

    # Symbolic variables
    s_posit = T.matrix()
    s_negat = T.matrix()
    idx_start = T.lscalar()
    idx_stop = T.lscalar()
    s_valid = theano.sparse.csr_matrix()

    w2i = cPickle.load(
        open(
            '/mnt/scratch/bengio/bengio_group/data/gutenberg/merged_word2idx.pkl'
        ))
    i2w = dict((v, k) for k, v in w2i.iteritems())
    i2w[0] = 'UNK'
    senna = [i2w[i] for i in range(len(i2w.keys()))]

    nsenna = len(senna)

    embedding = cae(i_size=nsenna, h_size=hp['embedsize'], e_act=identity)
    H = ae(i_size=hp['embedsize'] * hp['wsize'],
           h_size=hp['hsize'],
           e_act=T.tanh)
    L = logistic(i_size=hp['hsize'], h_size=1, act=identity)

    del H.params['d_bias']
    del embedding.params['d_bias']
    del embedding.params['e_bias']
    minsize = hp['minsize']
    maxsize = hp['maxsize']

    dsize = maxsize - minsize + 1

    H.params['e_bias'] = theano.shared(numpy.array(numpy.zeros(
        (dsize, hp['hsize'])),
                                                   dtype=theano.config.floatX),
                                       name='e_bias')

    path = hp['loadpath']

    if path:
        load(embedding, path + '/embedding.pkl')
        #load(H,path+'/hidden.pkl')
        #load(L,path+'/logistic.pkl')
        hp['embedsize'] = embedding.params['e_weights'].get_value(
            borrow=True).shape[1]
        #hp['hsize'] = H.params['e_weights'].get_value(borrow=True).shape[1]
        jobman.save()

    H.params['e_bias'] = theano.shared(numpy.array(numpy.zeros(
        (dsize, hp['hsize'])),
                                                   dtype=theano.config.floatX),
                                       name='e_bias')
    valid_embedding = sparse.supervised.logistic(i_size=nsenna,
                                                 h_size=hp['embedsize'],
                                                 act=identity)
    valid_embedding.params['weights'] = sp.shared(
        value=scipy.sparse.csr_matrix(embedding.params['e_weights'].get_value(
            borrow=True)))

    lr = hp['lr']
    h_size = hp['hsize']
    bs = hp['bs']

    posit_embed = T.dot(s_posit, embedding.params['e_weights']).reshape(
        (1, hp['embedsize'] * hp['wsize']))
    negat_embed = T.dot(s_negat, embedding.params['e_weights']).reshape(
        (hp['nneg'], hp['embedsize'] * hp['wsize']))
    valid_embed = sp.dot(s_valid, valid_embedding.params['weights']).reshape(
        (nsenna, hp['embedsize'] * hp['wsize']))

    posit_embed_left = T.concatenate([
        posit_embed[:, idx_start * hp['embedsize']:idx_stop * hp['embedsize']],
        T.zeros_like(posit_embed[:, idx_stop * hp['embedsize']:])
    ],
                                     axis=1)

    negat_embed_left = T.concatenate([
        negat_embed[:, idx_start * hp['embedsize']:idx_stop * hp['embedsize']],
        T.zeros_like(negat_embed[:, idx_stop * hp['embedsize']:])
    ],
                                     axis=1)

    posit_embed_right = T.concatenate([
        T.zeros_like(posit_embed[:, :idx_start * hp['embedsize']]),
        posit_embed[:, idx_start * hp['embedsize']:idx_stop * hp['embedsize']]
    ],
                                      axis=1)

    negat_embed_right = T.concatenate([
        T.zeros_like(negat_embed[:, :idx_start * hp['embedsize']]),
        negat_embed[:, idx_start * hp['embedsize']:idx_stop * hp['embedsize']]
    ],
                                      axis=1)

    posit_embed = T.concatenate([
        T.zeros_like(posit_embed[:, :idx_start * hp['embedsize']]),
        posit_embed[:, idx_start * hp['embedsize']:idx_stop * hp['embedsize']],
        T.zeros_like(posit_embed[:, idx_stop * hp['embedsize']:])
    ],
                                axis=1)

    negat_embed = T.concatenate([
        T.zeros_like(negat_embed[:, :idx_start * hp['embedsize']]),
        negat_embed[:, idx_start * hp['embedsize']:idx_stop * hp['embedsize']],
        T.zeros_like(negat_embed[:, idx_stop * hp['embedsize']:])
    ],
                                axis=1)

    #posit_embed = ifelse(T.eq(idx_start, 0), posit_embed_left, posit_embed)
    #posit_embed = ifelse(T.eq(idx_stop, hp['maxsize']), posit_embed_right, posit_embed)

    #negat_embed = ifelse(T.eq(idx_start, 0), negat_embed_left, negat_embed)
    #negat_embed = ifelse(T.eq(idx_stop, hp['maxsize']), negat_embed_right, negat_embed)

    Hposit = T.tanh(
        T.dot(posit_embed, H.params['e_weights']) +
        H.params['e_bias'][idx_stop - idx_start - minsize, :])
    Hnegat = T.tanh(
        T.dot(negat_embed, H.params['e_weights']) +
        H.params['e_bias'][idx_stop - idx_start - minsize, :])
    posit_score = L.encode(Hposit)
    negat_score = L.encode(Hnegat)
    valid_score = L.encode(H.encode(valid_embed))

    C = (negat_score - posit_score.flatten() + hp['margin'])

    CC = (rect(C)).mean()

    opt = theano.function([s_posit, s_negat, idx_start, idx_stop],
                          (rect(C)).mean(),
                          updates=dict(
                              L.update(CC, lr) + H.update(CC, lr) +
                              embedding.update_norm(CC, lr)))

    validfct = theano.function([s_valid], valid_score)

    def saveexp():
        save(embedding, fname + 'embedding.pkl')
        save(H, fname + 'hidden.pkl')
        save(L, fname + 'logistic.pkl')

    delta = hp['wsize'] / 2
    rest = hp['wsize'] % 2

    freq_idx = cPickle.load(
        open('/mnt/scratch/bengio/bengio_group/data/gutenberg/sorted_vocab.pkl'
             ))[:2000]
    fname = ''
    validsentence = [
    ]  # cPickle.load(open('/scratch/rifaisal/data/wiki_april_2010/valid_debug.pkl'))
    tseenwords = not debug
    for e in range(hp['epoch']):
        hp['split'] = numpy.random.randint(45)
        sentences = cPickle.load(
            open(
                '/mnt/scratch/bengio/bengio_group/data/gutenberg/ints_50000/split'
                + str(hp['split']) + '.pkl'))
        nsent = len(sentences)
        bigc = []
        bigr = []

        seen_words = 0
        for i, s in enumerate(sentences):
            nword = len(s)
            seen_words += nword
            tseenwords += nword

            if nword < hp['maxsize'] + 2:
                continue
            rndsize = numpy.random.randint(low=hp['minsize'] + 1,
                                           high=hp['maxsize'] - 1)
            idxsta = numpy.random.randint(low=1, high=hp['maxsize'] - rndsize)
            idxsto = idxsta + rndsize

            print 'r', rndsize, 'b', idxsta, 'e', idxsto, 'shape', H.params[
                'e_bias'].get_value().shape

            c = []
            r = []
            if debug:
                print ' *** Processing document', i, 'with', nword,
                sys.stdout.flush()
            for j in range(delta, nword - delta):
                nd = rndsize / 2
                rd = rndsize % 2
                pchunk = s[j - delta:j + delta + rest]
                nchunk = []

                rndidx = numpy.random.randint(nsenna, size=(hp['nneg'], ))
                nchunk = []
                for kk in range(hp['nneg']):
                    tmpchunk = copy.copy(pchunk)
                    tmpchunk[idxsta + nd] = rndidx[kk]
                    nchunk += tmpchunk
                assert len(nchunk) == len(pchunk) * hp['nneg']
                p, n = (idx2mat(pchunk, nsenna), idx2mat(nchunk, nsenna))
                l = opt(p, n, idxsta, idxsto)
                c.append(l)

                if debug:
                    print '.',
                    break

            if debug:
                print ''

            bigc += [numpy.array(c).sum()]

            if 0:  #(time.time() - expstart) > ( 3600 * 24 * 6 + 3600*20) or (tseenwords)>(10*hp['freq']):
                tseenwords = 0
                valid_embedding.params['weights'] = sp.shared(
                    value=scipy.sparse.csr_matrix(
                        embedding.params['e_weights'].get_value(borrow=True)))
                mrk = evaluation.error(validsentence, validfct, nsenna,
                                       hp['wsize'])
                hp['mrk'] = mrk
                jobman.save()
                saveexp()
                print 'Random Valid Mean rank', mrk

            if seen_words > hp['freq'] or debug:
                seen_words = 0
                hp['score'] = numpy.array(bigc).mean()
                hp['e'] = e
                hp['i'] = i
                print ''
                print e, i, 'NN Score:', hp['score']

                if not debug:
                    ne = knn(
                        freq_idx,
                        embedding.params['e_weights'].get_value(borrow=True))
                    open('files/' + fname + 'nearest.txt',
                         'w').write(display(ne, senna))
                    saveexp()
                sys.stdout.flush()
                jobman.save()

    saveexp()
示例#9
0
文件: test.py 项目: srifai/senna
def score(jobman, path):
    hp = jobman.state
    nsenna = 30000

    PATH = '/scratch/rifaisal/msrtest/test/'
    delta = hp['wsize'] / 2
    rest = hp['wsize'] % 2
    sent = T.matrix()

    embedding = cae(i_size=nsenna, h_size=hp['embedsize'], e_act=identity)
    H = ae(i_size=hp['embedsize'] * hp['wsize'],
           h_size=hp['hsize'],
           e_act=T.tanh)
    L = logistic(i_size=hp['hsize'], h_size=1, act=identity)

    load(embedding, path + '/embedding.pkl')
    load(H, path + '/hidden.pkl')
    load(L, path + '/logistic.pkl')

    posit_embed = T.dot(sent, embedding.params['e_weights']).reshape(
        (1, hp['embedsize'] * hp['wsize']))
    posit_score = H.encode(posit_embed)
    scoreit = theano.function([sent], posit_score)
    sentences = parse_data()
    scores = []
    esims = []
    msim = []
    hsim = []
    Em = embedding.params['e_weights'].get_value(borrow=True)
    for i, (sc, w1, w2, c1, c2) in enumerate(sentences):
        sys.stdout.flush()

        c1 = [29999] * 10 + c1 + [29999] * 10
        c2 = [29999] * 10 + c2 + [29999] * 10

        w1seqs = [c1[10 + idx - delta:10 + idx + delta + rest] for idx in w1]
        w2seqs = [c2[10 + idx - delta:10 + idx + delta + rest] for idx in w2]

        c = []

        w1em = Em[c1[10 + w1[0]]]
        w2em = Em[c2[10 + w2[0]]]

        w1sc = numpy.concatenate([
            scoreit(idx2mat(w1seqs[0], nsenna)).flatten(), Em[c1[10 + w1[0]]]
        ])
        w2sc = numpy.concatenate([
            scoreit(idx2mat(w2seqs[0], nsenna)).flatten(), Em[c2[10 + w2[0]]]
        ])

        metric = L.params['weights'].get_value(borrow=True).flatten()

        sim = -(((w1sc - w2sc))**2).sum()
        esim = -((w1em - w2em)**2).sum()

        msim.append(sim)
        esims.append(esim)
        hsim.append(numpy.mean(sc))

    print 'Model:', scipy.stats.spearmanr(
        numpy.array(hsim),
        numpy.array(msim))[0], ', Embeddings:', scipy.stats.spearmanr(
            numpy.array(hsim), numpy.array(esims))[0]
示例#10
0
文件: parsetest.py 项目: srifai/senna
def msrerror(vocab,jobman):
    hp = jobman.state
    nsenna = 30000

    PATH = '/scratch/rifaisal/msrtest/test/'
    delta = hp['wsize']/2
    rest = hp['wsize']%2
    sent = T.matrix()

    embedding = cae(i_size=nsenna, h_size=hp['embedsize'], e_act = identity)
    H = ae(i_size = hp['embedsize']*hp['wsize'], h_size=hp['hsize'], e_act = T.tanh)
    L = logistic(i_size = hp['hsize'], h_size = 1, act = identity)

    path = hp['loadpath']

    load(embedding,path+'/embedding.pkl')
    load(H,path+'/hidden.pkl')
    load(L,path+'/logistic.pkl')

    posit_embed = T.dot(sent, embedding.params['e_weights']).reshape((1,hp['embedsize']*hp['wsize']))
    posit_score = L.encode(H.encode(posit_embed))
    fct = theano.function([sent],posit_score)
    sentences = idxdataset(vocab)
    scores = []
    for i,s in enumerate(sentences):
        print i,
        sys.stdout.flush()
        nword = len(s)
        if nword < hp['wsize'] + 2:
            #print i,'Failure'
            s += [29999]*3
        c =[]
        for j in range(delta,nword-delta):
            pchunk = s[j-delta:j+delta+rest]
            p = idx2mat(pchunk,nsenna)
            l = fct(p)
            c.append(l)
        if not len(c):
            print 'pas bim'
            scores.append(0)
        else:
            scores.append(numpy.mean(c))
        #if i%5 == 0:
        #    print scores[i-5:i]

    score_template = open(PATH+'data/Holmes.lm_format.questions.txt')
    score_output = open('energy.lm_output.txt','w')
    sentencelist = score_template.readlines()
    for sc,sentence in zip(scores, sentencelist):
        score_output.write(sentence.split('\n')[0]+'\t'+str(sc)+'\n')
    score_output.close()

    pipebestof5 = subprocess.Popen(['perl', PATH+'bestof5.pl','./energy.lm_output.txt'],stdout=subprocess.PIPE)
    energyanswer = open('./energy.answers','w')

    for line in pipebestof5.stdout: energyanswer.write(line)

    energyanswer.close()

    pipescore = subprocess.Popen(['perl', PATH+'score.pl','./energy.answers',PATH+'data/Holmes.lm_format.answers.txt'],stdout=subprocess.PIPE)
    legend = ['correct','%correct','valid','test']
    out = zip(legend,[ r.split('\n')[0] for r in  pipescore.stdout.readlines()[-4:] ])
    res = dict(out)
    res = dict( (k,float(v)) for k,v in res.iteritems())
    print res
    print out
示例#11
0
def run(jobman,debug = False):
    hp = jobman.state

    # Symbolic variables

    s_posit = T.matrix()#theano.sparse.csr_matrix()
    s_negat = T.matrix()#theano.sparse.csr_matrix()

    s_valid = theano.sparse.csr_matrix()

    sentences = cPickle.load(open('/data/lisatmp2/rifaisal/guten_subset_idx.pkl'))

    validsentence = sentences[-10:]
    sentences = sentences[:-10]
    senna = cPickle.load(open('/data/lisatmp2/rifaisal/senna.pkl'))
    gsubset = cPickle.load(open('/data/lisatmp2/rifaisal/guten_vocab_subset.pkl')).flatten().tolist()
    hashtab = dict( zip( gsubset, range( len( gsubset))))    

    senna = numpy.array(senna)[gsubset].tolist()

    nsent = len(sentences)
    nsenna = len(senna)

    # Layers
    embedding = logistic(i_size=nsenna, h_size=hp['embedsize'], act = identity)
    H = ae(i_size = hp['embedsize']*hp['wsize'], h_size=hp['hsize'], e_act = rect, d_act = hardtanh)
    L = logistic(i_size = hp['hsize'],  h_size = 1)#, act = identity)

    valid_embedding = sparse.supervised.logistic(i_size=nsenna, h_size=hp['embedsize'], act = identity)
    #valid_embedding.params['weights'].set_value(embedding.params['weights'].get_value(borrow=True))
    #valid_embedding.params['bias'].set_value(embedding.params['bias'].get_value(borrow=True))


    lr = hp['lr']
    h_size = hp['hsize']
    bs = hp['bs']

    posit_embed = embedding.encode(s_posit).reshape((1,hp['embedsize']*hp['wsize']))
    negat_embed = embedding.encode(s_negat).reshape((hp['nneg'],hp['embedsize']*hp['wsize']))
    #valid_embed = valid_embedding.encode(s_valid).reshape((nsenna,hp['embedsize']*hp['wsize']))


    posit_score = L.encode(H.encode(posit_embed))
    negat_score = L.encode(H.encode(negat_embed))
    #valid_score = L.encode(H.encode(valid_embed))

    C = (negat_score - posit_score.flatten() + hp['margin'])

    CC = (rect(C)).mean()

    opt = theano.function([s_posit, s_negat], 
                          C.mean(), 
                          updates = dict( L.update(CC,lr) + H.update(CC,lr) + embedding.update_norm(CC,lr)) )

    #validfct = theano.function([s_valid],valid_score)

    #print 'Random Valid Mean rank',evaluation.error(validsentence, validfct, nsenna, hp['wsize'])

    #load(valid_embedding,'files/gutensubsetdense_exp.py_embedding.pkl')
    load(embedding,'files/gutensubsetdense_exp.py_embedding.pkl')
    load(H,'files/gutensubsetdense_exp.py_hidden.pkl')
    load(L,'files/gutensubsetdense_exp.py_logistic.pkl')

    delta = hp['wsize']/2
    rest = hp['wsize']%2

    freq_idx = cPickle.load(open('/data/lisatmp2/rifaisal/gutten_sorted_vocab.pkl'))[:1000]
    freq_idx =  [ hashtab[idx] for idx in freq_idx ]

    fname = sys.argv[0]+'_'
    

    for e in range(hp['epoch']):
        c = []
        for i in range(nsent):
            rsent = numpy.random.randint(nsent-1)
            nword = len(sentences[rsent])
            if nword < hp['wsize'] + 2:
                continue

            pidx = numpy.random.randint(low = delta, high = nword-delta)
            pchunk = sentences[rsent][pidx-delta:pidx+delta+rest]
            nchunk = []
            st = sentences[rsent][pidx-delta:pidx]
            en = sentences[rsent][pidx+1:pidx+delta+rest]
            rndidx = numpy.random.randint(nsenna, size = (hp['nneg'],))
            nchunk = []
            for j in range(hp['nneg']):
                nchunk += en + [rndidx[j]] + st


            assert len(nchunk) == len(pchunk)*hp['nneg']
            #start = time.time()
            p, n = (idx2mat(pchunk,nsenna), idx2mat(nchunk,nsenna))
            #print 'Select row:',time.time()-start,
            #start = time.time()
            c.append(opt(p,n))
            #print 'grad up:',time.time()-start

            if i%hp['freq'] == 0:
                print e,i, numpy.array(c).mean(0)
                ne = knn(freq_idx,embedding.params['weights'].get_value(borrow=True))
                save(embedding,fname+'embedding.pkl')
                save(H,fname+'hidden.pkl')
                save(L,fname+'logistic.pkl')
                sys.stdout.flush()
                open('files/'+fname+'nearest.txt','w').write(display(ne,senna))

    #print 'Valid Mean rank',evaluation.error(validsentence, validfct, nsenna, hp['wsize'])
    save(embedding,fname+'embedding.pkl')
    save(H,fname+'hidden.pkl')
    save(L,fname+'logistic.pkl')
示例#12
0
def run(jobman,debug = False):
    expstart = time.time()
    hp = jobman.state

    if not os.path.exists('files/'): os.mkdir('files/')

    # Symbolic variables
    s_bow = T.matrix()
    s_idx = T.iscalar()
    s_tf = T.scalar()
    s_posit = T.matrix()#theano.sparse.csr_matrix()
    s_negat = T.matrix()#theano.sparse.csr_matrix()

    sentences = cPickle.load(open('/scratch/rifaisal/data/guten/guten_subset_idx.pkl'))

    senna = cPickle.load(open('/scratch/rifaisal/data/guten/senna.pkl'))
    gsubset = cPickle.load(open('/scratch/rifaisal/data/guten/guten_vocab_subset.pkl')).flatten().tolist()
    hashtab = dict( zip( gsubset, range( len( gsubset))))    

    tfidf_data = numpy.load('/scratch/rifaisal/data/guten/guten_tfidf.npy').item().tocsr().astype('float32')

    #tfidf = cPickle.load(open('/scratch/rifaisal/repos/senna/gutentokenizer.pkl'))

    senna = numpy.array(senna)[gsubset].tolist()
    s_valid = theano.sparse.csr_matrix()

    validsentence = sentences[10000:10010]


    nsent = len(sentences)
    nsenna = len(senna)

    # Layers
    
    embedding = cae(i_size=nsenna, h_size=hp['embedsize'], e_act = identity)

    H = ae(i_size = hp['embedsize']*hp['wsize'], h_size=hp['hsize'], e_act = T.tanh)
    L = logistic(i_size = hp['hsize'], h_size = 1, act = identity)
    S = logistic(i_size = hp['embedsize'], h_size = nsenna, act= T.nnet.softmax)


    valid_embedding = sparse.supervised.logistic(i_size=nsenna, h_size=hp['embedsize'], act = identity)
    valid_embedding.params['weights'] = sp.shared(value = scipy.sparse.csr_matrix(embedding.params['e_weights'].get_value(borrow=True)))
    valid_embedding.params['bias'] = embedding.params['e_bias']

    lr = hp['lr']
    h_size = hp['hsize']
    bs = hp['bs']

    posit_embed = T.dot(s_posit, embedding.params['e_weights']).reshape((1,hp['embedsize']*hp['wsize']))
    negat_embed = T.dot(s_negat, embedding.params['e_weights']).reshape((hp['nneg'],hp['embedsize']*hp['wsize']))
    valid_embed = sp.dot(s_valid,valid_embedding.params['weights']).reshape((nsenna,hp['embedsize']*hp['wsize']))

    posit_score = L.encode(H.encode(posit_embed))
    negat_score = L.encode(H.encode(negat_embed))
    valid_score = L.encode(H.encode(valid_embed))

    C = (negat_score - posit_score.flatten() + hp['margin'])

    s_bow_pred = S.encode(embedding.encode(s_bow))


    pred = s_tf * nllsoft(s_bow_pred,s_idx)
    
    CC = (rect(C)).mean() + hp['lambda'] * pred

    opt = theano.function([s_posit, s_negat, s_bow, s_idx, s_tf], 
                          [(rect(C)).mean(),pred], 
                          updates = dict( S.update(CC,lr) + L.update(CC,lr) + H.update(CC,lr) + embedding.update_norm(CC,lr)) )

    #validfct = theano.function([s_valid],valid_score)

    def saveexp():
        save(embedding,fname+'embedding.pkl')
        save(H,fname+'hidden.pkl')
        save(L,fname+'logistic.pkl')

    delta = hp['wsize']/2
    rest = hp['wsize']%2

    freq_idx = cPickle.load(open('/scratch/rifaisal/data/guten/gutten_sorted_vocab.pkl'))[:1000]
    freq_idx =  [ hashtab[idx] for idx in freq_idx ]

    fname = ''
    
    for e in range(hp['epoch']):
        c = []
        r = []
        count = 1
        for i in range(nsent):
            rsent = numpy.random.randint(nsent-1)
            nword = len(sentences[rsent])
            if nword < hp['wsize'] + 2:
                continue

            pidx = numpy.random.randint(low = delta, high = nword-delta)
            pchunk = sentences[rsent][pidx-delta:pidx+delta+rest]
            nchunk = []
            st = sentences[rsent][pidx-delta:pidx]
            en = sentences[rsent][pidx+1:pidx+delta+rest]
            rndidx = numpy.random.randint(nsenna, size = (hp['nneg'],))
            nchunk = []
            for j in range(hp['nneg']):
                nchunk += en + [rndidx[j]] + st


            assert len(nchunk) == len(pchunk)*hp['nneg']
            tfidf_chunk = tfidf_data[rsent:rsent+1].toarray()
            #pdb.set_trace()
            tfidf_value = tfidf_chunk[0,sentences[rsent][pidx]]
            tfidf_chunk[0,sentences[rsent][pidx]] = 0.
            tfidx = sentences[rsent][pidx] # numpy.zeros(tfidf_chunk.shape).astype('float32')
            #tfidx[0,sentences[rsent][pidx]] = 1.
            p, n, b, iidx, tfval = (idx2mat(pchunk,nsenna), idx2mat(nchunk,nsenna), tfidf_chunk, tfidx, tfidf_value )
            count += tfval!=0
            l,g = opt(p,n,b, iidx, tfval)
            c = c
            c.append(l)
            r.append(g)

            """
            if (time.time() - expstart) > ( 3600 * 24 * 6 + 3600*20) or (i+1)%(20*hp['freq']) == 0 and debug==False:
                valid_embedding.params['weights'] = sp.shared(value = scipy.sparse.csr_matrix(embedding.params['e_weights'].get_value(borrow=True)))
                mrk = evaluation.error(validsentence, validfct, nsenna, hp['wsize'])
                hp['mrk'] = mrk
                jobman.save()
                saveexp()
                print 'Random Valid Mean rank',mrk
            """

            if (i+1)%hp['freq'] == 0 or debug:
                hp['score'] = numpy.array(c).sum() / (numpy.array(c)>0).sum()
                hp['pred'] = numpy.array(r).sum()/float(count)
                hp['e'] = e
                hp['i'] = i
                print ''
                print e,i,'NN Score:', hp['score'], 'Reconstruction:', hp['pred']

                if debug != True:
                    ne = knn(freq_idx,embedding.params['e_weights'].get_value(borrow=True))
                    open('files/'+fname+'nearest.txt','w').write(display(ne,senna))
                    saveexp()
                sys.stdout.flush()
                jobman.save()
                
    saveexp()
示例#13
0
def run(jobman,debug = False):
    expstart = time.time()
    hp = jobman.state

    if not os.path.exists('files/'): os.mkdir('files/')

    # Symbolic variables
    s_posit = T.matrix()
    s_negat = T.matrix()
    s_valid = theano.sparse.csr_matrix()

    #vocab = cPickle.load(open('/scratch/rifaisal/data/guten/senna.pkl'))
    #senna = cPickle.load(open('/scratch/rifaisal/data/wiki_april_2010/WestburyLab.wikicorp.201004_vocab30k.pkl'))
    w2i = cPickle.load(open('/scratch/rifaisal/data/gutenberg_aistats/merged_word2idx.pkl'))
    i2w = dict( (v,k) for k,v in w2i.iteritems() )
    i2w[0] = 'UNK'
    senna = [ i2w[i] for i in range(len(i2w.keys())) ]

    nsenna = len(senna)
    
    embedding = cae(i_size=nsenna, h_size=hp['embedsize'], e_act = identity)
    H = ae(i_size = hp['embedsize']*hp['wsize'], h_size=hp['hsize'], e_act = T.tanh)
    L = logistic(i_size = hp['hsize'], h_size = 1, act = identity)
 
    path = hp['loadpath']
 
    if path:
        load(embedding,path+'/embedding.pkl')
        load(H,path+'/hidden.pkl')
        load(L,path+'/logistic.pkl')
        hp['embedsize'] = embedding.params['e_weights'].get_value(borrow=True).shape[1]
        hp['hsize'] = H.params['e_weights'].get_value(borrow=True).shape[1]
        jobman.save()

    valid_embedding = sparse.supervised.logistic(i_size=nsenna, h_size=hp['embedsize'], act = identity)
    valid_embedding.params['weights'] = sp.shared(value = scipy.sparse.csr_matrix(embedding.params['e_weights'].get_value(borrow=True)))
    valid_embedding.params['bias'] = embedding.params['e_bias']

    lr = hp['lr']
    h_size = hp['hsize']
    bs = hp['bs']

    posit_embed = T.dot(s_posit, embedding.params['e_weights']).reshape((1,hp['embedsize']*hp['wsize']))
    negat_embed = T.dot(s_negat, embedding.params['e_weights']).reshape((hp['nneg'],hp['embedsize']*hp['wsize']))
    valid_embed = sp.dot(s_valid,valid_embedding.params['weights']).reshape((nsenna,hp['embedsize']*hp['wsize']))

    posit_score = L.encode(H.encode(posit_embed))
    negat_score = L.encode(H.encode(negat_embed))
    valid_score = L.encode(H.encode(valid_embed))

    C = (negat_score - posit_score.flatten() + hp['margin'])

    CC = (rect(C)).mean()

    opt = theano.function([s_posit, s_negat],
                          (rect(C)).mean(),
                          updates = dict( L.update(CC,lr) + H.update(CC,lr) + embedding.update_norm(CC,lr)) )

    #validfct = theano.function([s_valid],valid_score)

    def saveexp():
        save(embedding,fname+'embedding.pkl')
        save(H,fname+'hidden.pkl')
        save(L,fname+'logistic.pkl')


    delta = hp['wsize']/2
    rest = hp['wsize']%2
    #freq_idx = range(29000,30000)
    freq_idx = cPickle.load(open('/scratch/rifaisal/data/gutenberg_aistats/sorted_vocab.pkl'))[:2000]
    fname = ''
    #validsentence = cPickle.load(open('/scratch/rifaisal/data/gutenberg_aistats/valid.pkl'))
    tseenwords = not debug
    for e in range(hp['epoch']):
        hp['split'] = numpy.random.randint(45)
        sentences = cPickle.load(open('/scratch/rifaisal/data/gutenberg_aistats/split'+str(hp['split'])+'.pkl'))
        nsent = len(sentences)
        bigc = []
        bigr = []

        seen_words = 0
        for i,s in enumerate(sentences):
            nword = len(s)
            seen_words += nword
            tseenwords += nword

            if nword < hp['wsize'] + 2:
                continue
            c =[]
            r =[]
            if debug:
                print ' *** Processing document',i,'with',nword,
                sys.stdout.flush()
            for j in range(delta,nword-delta):
                pchunk = s[j-delta:j+delta+rest]
                nchunk = []
                st = s[j-delta:j]
                en = s[j+1:j+delta+rest]
                rndidx = numpy.random.randint(nsenna, size = (hp['nneg'],))
                nchunk = []
                for kk in range(hp['nneg']):
                    nchunk += st + [rndidx[kk]] + en

                assert len(nchunk) == len(pchunk)*hp['nneg']
                p, n  = (idx2mat(pchunk,nsenna), idx2mat(nchunk,nsenna))
                l = opt(p,n)
                c.append(l)

                if debug:
                    print '.',
                    break


            if debug:
                print ''

            bigc += [numpy.array(c).sum()]

            if 0:#(time.time() - expstart) > ( 3600 * 24 * 6 + 3600*20) or (tseenwords)>(10*hp['freq']):
                tseenwords = 0
                valid_embedding.params['weights'] = sp.shared(value = scipy.sparse.csr_matrix(embedding.params['e_weights'].get_value(borrow=True)))
                mrk = evaluation.error(validsentence, validfct, nsenna, hp['wsize'])
                hp['mrk'] = mrk
                jobman.save()
                saveexp()
                print 'Random Valid Mean rank',mrk


            if seen_words > hp['freq'] or debug:
                seen_words = 0
                hp['score'] = numpy.array(bigc).mean() 
                hp['e'] = e
                hp['i'] = i
                print ''
                print e,i,'NN Score:', hp['score']

                if not debug:
                    ne = knn(freq_idx,embedding.params['e_weights'].get_value(borrow=True))
                    open('files/'+fname+'nearest.txt','w').write(display(ne,senna))
                    saveexp()
                sys.stdout.flush()
                jobman.save()
                
    saveexp()