def buildDocDoc(arts, type, language):
    ipyutils.loadDicts(prefix="gensim_" + language)
    arts = [art for art in arts if art.id_int in ipyutils.rdocids]
    assert len(arts) == len(ipyutils.rdocids)

    logging.info("loading msc<->id mapping")
    cats, rcats = loadMsc2Id(language)

    mscsFile = common.matrixFile("mscs_%s.mm" % type)
    matMsc = matutils.loadMatrix(mscsFile)
    mscDict = {}
    for art in arts:
        artId = ipyutils.rdocids[art.id_int]
        mscIds = [rcats[mscs.niceMSC(msc)[0]] for msc in art.msc]
        mscDict[artId] = mscIds

    logging.info("computing doc*doc similarity matrix based on %s" % mscsFile)
    docdoc = numpy.zeros((len(arts), len(arts)), numpy.float32)

    for i in xrange(len(arts)):
        if i % 100 == 0:
            logging.info("PROGRESS: %i/%i" % (i, len(arts)))
        art1Id = ipyutils.rdocids[arts[i].id_int]
        for j in xrange(i, len(arts)):
            art2Id = ipyutils.rdocids[arts[j].id_int]
            bestScore = 0.0
            for msc1Id in mscDict[art1Id]:
                for msc2Id in mscDict[art2Id]:
                    bestScore = max(bestScore, matMsc[msc1Id, msc2Id])
            docdoc[art1Id, art2Id] = docdoc[art2Id, art1Id] = bestScore

    matutils.saveMatrix(docdoc, common.matrixFile("docdoc_" + language + "_%s.mm" % type), sparse=False)
    return docdoc
def loadVSMatrices(prefix = 'cmj', loadTFIDF = True, loadLSI = True):
    """load and return vector space matrices of the documents"""
    global vecTFIDF, vecLSI
    vecTFIDF, vecLSI = None, None
    if loadTFIDF:
        vecTFIDF = matutils.loadMatrix(common.matrixFile(prefix + 'TFIDF.mm'))
    if loadLSI:
        vecLSI = matutils.loadMatrix(common.matrixFile(prefix + 'LSI_VT.mm'))
    return vecTFIDF, vecLSI
def loadVSMatrices(prefix='cmj', loadTFIDF=True, loadLSI=True):
    """load and return vector space matrices of the documents"""
    global vecTFIDF, vecLSI
    vecTFIDF, vecLSI = None, None
    if loadTFIDF:
        vecTFIDF = matutils.loadMatrix(common.matrixFile(prefix + 'TFIDF.mm'))
    if loadLSI:
        vecLSI = matutils.loadMatrix(common.matrixFile(prefix + 'LSI_VT.mm'))
    return vecTFIDF, vecLSI
Exemple #4
0
def buildMscPureMatrix(lang):
    if os.path.exists(ARTS_FILE):
        logging.warning(
            "SKIPPING creating MSC database from meta.xml files (using old %s); is this what you want?"
            % ARTS_FILE)
    else:
        logging.info("creating MSC database from meta.xml files")
        createMscsDb(
        )  # only run this when collection changes (test for file existence / delete file explicitly?)
    arts = [
        art for art in docsim.getArts(
            ARTS_FILE, acceptNoBody=True, acceptNoMsc=False)
        if art.language == lang or lang == 'any'
    ]

    cats = dict(enumerate(mscs.getFreqMSCs(arts, minFreq=1, useMains=False)))
    rcats = utils_dml.reverseMap(cats)
    saveMsc2Id(cats, lang)

    logging.info("building MSC binary matrix")
    resultBin = numpy.zeros(
        (len(cats), len(cats)), dtype=numpy.float32
    )  # binary msc similarity (with fixed msc hierarchy = identity)
    for idi, cati in cats.iteritems():
        for idj, catj in cats.iteritems():
            #            print idi, cati, idj, catj
            if idi == idj:
                resultBin[idi, idj] = 1.0
            else:
                resultBin[idi, idj] = 0.0
    matutils.saveMatrix(resultBin,
                        common.matrixFile("mscs_bin_%s.mm" % lang),
                        sparse=False)
    return resultBin
Exemple #5
0
def buildMscOverlapMatrix(lang):
    logging.info("building MSC overlap matrix")

    arts = [
        art for art in docsim.getArts(
            ARTS_FILE, acceptNoBody=True, acceptNoMsc=False)
        if art.language == lang or lang == 'any'
    ]
    cats, rcats = loadMsc2Id(lang)  # from buildPure

    logging.info("computing MSC matrices")

    overlap = numpy.zeros(
        (len(cats), len(cats)), dtype=int
    )  # binary msc similarity (with fixed msc hierarchy = identity)
    for art in arts:
        for msc1 in art.msc:
            for msc2 in art.msc:
                overlap[rcats[mscs.niceMSC(msc1)[0]],
                        rcats[mscs.niceMSC(msc2)[0]]] += 1

    resultOverlap = numpy.zeros((len(cats), len(cats)), dtype=numpy.float32)
    for i in xrange(resultOverlap.shape[0]):
        max = numpy.max(overlap[i])
        for j in xrange(resultOverlap.shape[1]):
            resultOverlap[i, j] = math.log(1.0 + 100.0 * overlap[i, j] /
                                           max) / math.log(101)

    matutils.saveMatrix(resultOverlap,
                        common.matrixFile("mscs_overlap_%s.mm" % lang),
                        sparse=False)
    return resultOverlap
def buildMscPureMatrix(lang):
    if os.path.exists(ARTS_FILE):
        logging.warning(
            "SKIPPING creating MSC database from meta.xml files (using old %s); is this what you want?" % ARTS_FILE
        )
    else:
        logging.info("creating MSC database from meta.xml files")
        createMscsDb()  # only run this when collection changes (test for file existence / delete file explicitly?)
    arts = [
        art
        for art in docsim.getArts(ARTS_FILE, acceptNoBody=True, acceptNoMsc=False)
        if art.language == lang or lang == "any"
    ]

    cats = dict(enumerate(mscs.getFreqMSCs(arts, minFreq=1, useMains=False)))
    rcats = utils_dml.reverseMap(cats)
    saveMsc2Id(cats, lang)

    logging.info("building MSC binary matrix")
    resultBin = numpy.zeros(
        (len(cats), len(cats)), dtype=numpy.float32
    )  # binary msc similarity (with fixed msc hierarchy = identity)
    for idi, cati in cats.iteritems():
        for idj, catj in cats.iteritems():
            #            print idi, cati, idj, catj
            if idi == idj:
                resultBin[idi, idj] = 1.0
            else:
                resultBin[idi, idj] = 0.0
    matutils.saveMatrix(resultBin, common.matrixFile("mscs_bin_%s.mm" % lang), sparse=False)
    return resultBin
def buildMscOverlapMatrix(lang):
    logging.info("building MSC overlap matrix")

    arts = [
        art
        for art in docsim.getArts(ARTS_FILE, acceptNoBody=True, acceptNoMsc=False)
        if art.language == lang or lang == "any"
    ]
    cats, rcats = loadMsc2Id(lang)  # from buildPure

    logging.info("computing MSC matrices")

    overlap = numpy.zeros(
        (len(cats), len(cats)), dtype=int
    )  # binary msc similarity (with fixed msc hierarchy = identity)
    for art in arts:
        for msc1 in art.msc:
            for msc2 in art.msc:
                overlap[rcats[mscs.niceMSC(msc1)[0]], rcats[mscs.niceMSC(msc2)[0]]] += 1

    resultOverlap = numpy.zeros((len(cats), len(cats)), dtype=numpy.float32)
    for i in xrange(resultOverlap.shape[0]):
        max = numpy.max(overlap[i])
        for j in xrange(resultOverlap.shape[1]):
            resultOverlap[i, j] = math.log(1.0 + 100.0 * overlap[i, j] / max) / math.log(101)

    matutils.saveMatrix(resultOverlap, common.matrixFile("mscs_overlap_%s.mm" % lang), sparse=False)
    return resultOverlap
Exemple #8
0
def buildDocDoc(arts, type, language):
    ipyutils.loadDicts(prefix='gensim_' + language)
    arts = [art for art in arts if art.id_int in ipyutils.rdocids]
    assert (len(arts) == len(ipyutils.rdocids))

    logging.info("loading msc<->id mapping")
    cats, rcats = loadMsc2Id(language)

    mscsFile = common.matrixFile("mscs_%s.mm" % type)
    matMsc = matutils.loadMatrix(mscsFile)
    mscDict = {}
    for art in arts:
        artId = ipyutils.rdocids[art.id_int]
        mscIds = [rcats[mscs.niceMSC(msc)[0]] for msc in art.msc]
        mscDict[artId] = mscIds

    logging.info("computing doc*doc similarity matrix based on %s" % mscsFile)
    docdoc = numpy.zeros((len(arts), len(arts)), numpy.float32)

    for i in xrange(len(arts)):
        if i % 100 == 0:
            logging.info("PROGRESS: %i/%i" % (i, len(arts)))
        art1Id = ipyutils.rdocids[arts[i].id_int]
        for j in xrange(i, len(arts)):
            art2Id = ipyutils.rdocids[arts[j].id_int]
            bestScore = 0.0
            for msc1Id in mscDict[art1Id]:
                for msc2Id in mscDict[art2Id]:
                    bestScore = max(bestScore, matMsc[msc1Id, msc2Id])
            docdoc[art1Id, art2Id] = docdoc[art2Id, art1Id] = bestScore

    matutils.saveMatrix(docdoc,
                        common.matrixFile("docdoc_" + language +
                                          "_%s.mm" % type),
                        sparse=False)
    return docdoc
Exemple #9
0
def rmseFile(matfile1, matfile2):
    data1 = matutils.loadMatrix(common.matrixFile(matfile1))
    data2 = matutils.loadMatrix(common.matrixFile(matfile2))
    return matutils.rmse(data1, data2)
Exemple #10
0
def buildMscCentroidMatrix(language):
    logging.info("building MSC centroid matrix from %s" % ARTS_FILE)
    arts = [
        art for art in docsim.getArts(
            ARTS_FILE, acceptNoBody=False, acceptNoMsc=False)
        if art.language == language or language == 'any'
    ]
    prefix = 'mscs_serial_%s_' % language
    matFile = common.matrixFile(prefix + 'TFIDF_T.mm')
    if os.path.exists(matFile):
        logging.warning(
            'SKIPPING creating TFIDF matrix for %s (file %s present). Is this what you wanted?'
            % (language, matFile))
        tfidf = matutils.loadMatrix(matFile).tocsr()
    else:
        logging.info('creating TFIDF matrix for %s to %s' %
                     (language, matFile))
        tfidf = docsim.buildTFIDFMatrices(arts,
                                          prefix=prefix,
                                          saveMatrices=False).tocsr()

    ipyutils.loadDicts(prefix=prefix)
    arts = [
        art for art in arts if art.id_int in ipyutils.rdocids
    ]  # remove articles that had empty body (according to their tfidf vector)
    if len(ipyutils.rdocids) != len(arts):
        logging.error(
            "no. of TFIDF document = %i, but there are %i documents in the database (mismatch)"
            % (len(ipyutils.rdocids), len(arts)))
        raise Exception(
            "different size of database/dictionary; version mismatch?")

    cats, rcats = loadMsc2Id(language)  # from buildPure
    #    print "mscs:", cats

    logging.info("loading tfidf collection matrix (for centroids)")
    tfidf = matutils.loadMatrix(
        common.matrixFile('gensim_' + language + 'TFIDF_T.mm')).tocsr()
    logging.debug("loaded %ix%i matrix" % tfidf.shape)

    logging.info("computing centroids")
    centroids = numpy.zeros((len(cats), tfidf.shape[1]), numpy.float)
    #    print "centroids.shape =", centroids.shape
    num = numpy.zeros((len(cats), ), numpy.int)
    artCnt = 0
    for art in arts:
        if not art.id_int in ipyutils.rdocids:
            logging.warning("article not found among docids: %s" % art)
            continue
        artCnt += 1
        artId = ipyutils.rdocids[art.id_int]
        tops = [mscs.niceMSC(msc)[0] for msc in art.msc]
        tops = set(
            tops
        )  # only count each top-level once (comment out this line to count e.g. 30H55 and 30.13 twice for this article, as cat. 30)
        for top in tops:
            mscId = rcats[top]
            vec = tfidf[artId].toarray()
            vec.shape = (vec.size, )
            #            print "vec.shape = ", vec.shape
            centroids[mscId] += vec
            num[mscId] += 1
        if artCnt < 10 or artCnt % 1000 == 0:
            logging.debug(
                "sanity check - article %s has id %i and has mscs=%s, mscsIds=%s"
                % (art.id_int, artId, art.msc,
                   [rcats[mscs.niceMSC(msc)[0]] for msc in art.msc]))
    if not artCnt == tfidf.shape[0]:
        raise Exception("not all articles used; database/matrix mismatch?")
    for i, vec in enumerate(centroids):
        logging.info(
            "centroid for msc %s (id %i) is an average of %i vectors" %
            (cats[i], i, num[i]))
        if numpy.sum(numpy.abs(vec)) == 0:
            logging.warning("empty centroid for msc %s (msc int id %i)" %
                            (cats[i], i))
    for mscId in cats.iterkeys():
        centroids[mscId] /= num[mscId]
    logging.info(
        "used %i articles for %i vectors (articles may have more than one msc and so can be counted more than once)"
        % (artCnt, sum(num)))

    logging.info("computing MSC centroid matrix")
    resultCentroid = numpy.zeros((len(cats), len(cats)), dtype=numpy.float32)
    for idi, cati in cats.iteritems():
        for idj, catj in cats.iteritems():
            #            print idi, cati, idj, catj
            sim = matutils.cossim(centroids[idi], centroids[idj])
            if numpy.isfinite(sim):
                resultCentroid[idi, idj] = sim
            else:
                resultCentroid[idi, idj] = 0.0

    matutils.saveMatrix(resultCentroid,
                        common.matrixFile("mscs_centroid_%s.mm" % language),
                        sparse=False)
def loadConcepts(prefix='cmj'):
    """load and return the LSI_U matrix of concepts"""
    global concepts
    concepts = matutils.loadMatrix(common.matrixFile(prefix + 'LSI_U.mm'))
    return concepts
def rmseFile(matfile1, matfile2):
    data1 = matutils.loadMatrix(common.matrixFile(matfile1))
    data2 = matutils.loadMatrix(common.matrixFile(matfile2))
    return matutils.rmse(data1, data2)
def buildMscCentroidMatrix(language):
    logging.info("building MSC centroid matrix from %s" % ARTS_FILE)
    arts = [
        art
        for art in docsim.getArts(ARTS_FILE, acceptNoBody=False, acceptNoMsc=False)
        if art.language == language or language == "any"
    ]
    prefix = "mscs_serial_%s_" % language
    matFile = common.matrixFile(prefix + "TFIDF_T.mm")
    if os.path.exists(matFile):
        logging.warning(
            "SKIPPING creating TFIDF matrix for %s (file %s present). Is this what you wanted?" % (language, matFile)
        )
        tfidf = matutils.loadMatrix(matFile).tocsr()
    else:
        logging.info("creating TFIDF matrix for %s to %s" % (language, matFile))
        tfidf = docsim.buildTFIDFMatrices(arts, prefix=prefix, saveMatrices=False).tocsr()

    ipyutils.loadDicts(prefix=prefix)
    arts = [
        art for art in arts if art.id_int in ipyutils.rdocids
    ]  # remove articles that had empty body (according to their tfidf vector)
    if len(ipyutils.rdocids) != len(arts):
        logging.error(
            "no. of TFIDF document = %i, but there are %i documents in the database (mismatch)"
            % (len(ipyutils.rdocids), len(arts))
        )
        raise Exception("different size of database/dictionary; version mismatch?")

    cats, rcats = loadMsc2Id(language)  # from buildPure
    #    print "mscs:", cats

    logging.info("loading tfidf collection matrix (for centroids)")
    tfidf = matutils.loadMatrix(common.matrixFile("gensim_" + language + "TFIDF_T.mm")).tocsr()
    logging.debug("loaded %ix%i matrix" % tfidf.shape)

    logging.info("computing centroids")
    centroids = numpy.zeros((len(cats), tfidf.shape[1]), numpy.float)
    #    print "centroids.shape =", centroids.shape
    num = numpy.zeros((len(cats),), numpy.int)
    artCnt = 0
    for art in arts:
        if not art.id_int in ipyutils.rdocids:
            logging.warning("article not found among docids: %s" % art)
            continue
        artCnt += 1
        artId = ipyutils.rdocids[art.id_int]
        tops = [mscs.niceMSC(msc)[0] for msc in art.msc]
        tops = set(
            tops
        )  # only count each top-level once (comment out this line to count e.g. 30H55 and 30.13 twice for this article, as cat. 30)
        for top in tops:
            mscId = rcats[top]
            vec = tfidf[artId].toarray()
            vec.shape = (vec.size,)
            #            print "vec.shape = ", vec.shape
            centroids[mscId] += vec
            num[mscId] += 1
        if artCnt < 10 or artCnt % 1000 == 0:
            logging.debug(
                "sanity check - article %s has id %i and has mscs=%s, mscsIds=%s"
                % (art.id_int, artId, art.msc, [rcats[mscs.niceMSC(msc)[0]] for msc in art.msc])
            )
    if not artCnt == tfidf.shape[0]:
        raise Exception("not all articles used; database/matrix mismatch?")
    for i, vec in enumerate(centroids):
        logging.info("centroid for msc %s (id %i) is an average of %i vectors" % (cats[i], i, num[i]))
        if numpy.sum(numpy.abs(vec)) == 0:
            logging.warning("empty centroid for msc %s (msc int id %i)" % (cats[i], i))
    for mscId in cats.iterkeys():
        centroids[mscId] /= num[mscId]
    logging.info(
        "used %i articles for %i vectors (articles may have more than one msc and so can be counted more than once)"
        % (artCnt, sum(num))
    )

    logging.info("computing MSC centroid matrix")
    resultCentroid = numpy.zeros((len(cats), len(cats)), dtype=numpy.float32)
    for idi, cati in cats.iteritems():
        for idj, catj in cats.iteritems():
            #            print idi, cati, idj, catj
            sim = matutils.cossim(centroids[idi], centroids[idj])
            if numpy.isfinite(sim):
                resultCentroid[idi, idj] = sim
            else:
                resultCentroid[idi, idj] = 0.0

    matutils.saveMatrix(resultCentroid, common.matrixFile("mscs_centroid_%s.mm" % language), sparse=False)
def loadConcepts(prefix = 'cmj'):
    """load and return the LSI_U matrix of concepts"""
    global concepts
    concepts = matutils.loadMatrix(common.matrixFile(prefix + 'LSI_U.mm'))
    return concepts
def loadSimMatrices(prefix='cmj'):
    global m, l
    m = matutils.loadMatrix(common.matrixFile(prefix + 'TFIDFsim.mm'))
    l = matutils.loadMatrix(common.matrixFile(prefix + 'LSIsim.mm'))
    return m, l
Exemple #16
0
#!/usr/bin/env python2.5

import logging

logging.root.level = 10

import common
import pylab
import numpy
import matutils

mat = matutils.loadMatrix(common.matrixFile("gensim_engTFIDFsim.mm"))
#mat = numpy.zeros((10, 20), float)
#mat[2] = 0.2
#mat[8] = 0.9
logging.info("%ix%i matrix loaded" % mat.shape)
pylab.figure()
pylab.imshow(mat, cmap = pylab.cm.gray, interpolation = "nearest")
pylab.savefig("tfidf_sim")
		oldMsc = msc
logging.info("%i breaks" % len(breaks))
print "breaks:", breaks

# print individual category matrices
breaks.append((i, msc)) # append the last category
print "breaks with end:", breaks

new2old = {}
for i, (msc, id) in enumerate(art2msc):
    new2old[i] = ipyutils.rdocids[id]
print '==first 100 new2old:', new2old.items()[:100]
	

logging.info("loading cossim matrix")
mat = matutils.loadMatrix(common.matrixFile("gensim_eng%ssim.mm" % (SIM_TYPE)))
for i in xrange(len(breaks) - 1):
	numArts = breaks[i + 1][0] - breaks[i][0]
	logging.debug("%i articles in category %s" % (numArts, breaks[i][1]))
	matId = numpy.zeros((numArts, numArts), numpy.float)
	for i1 in xrange(numArts):
		print breaks[i][0] + i1, i1, art2mscOld[breaks[i][0] + i1]
		for i2 in xrange(numArts):
			pos1, pos2 = breaks[i][0] + i1, breaks[i][0] + i2
			matId[i1, i2] = mat[new2old[pos1], new2old[pos2]]
	processMsc(matId, breaks[i][1][0])

sys.exit(0)

new2old = {}
for i, (msc, id) in enumerate(art2msc):
        breaks.append((i, msc))
        oldMsc = msc
logging.info("%i breaks" % len(breaks))
print "breaks:", breaks

# print individual category matrices
breaks.append((i, msc))  # append the last category
print "breaks with end:", breaks

new2old = {}
for i, (msc, id) in enumerate(art2msc):
    new2old[i] = ipyutils.rdocids[id]
print '==first 100 new2old:', new2old.items()[:100]

logging.info("loading cossim matrix")
mat = matutils.loadMatrix(common.matrixFile("gensim_eng%ssim.mm" % (SIM_TYPE)))
for i in xrange(len(breaks) - 1):
    numArts = breaks[i + 1][0] - breaks[i][0]
    logging.debug("%i articles in category %s" % (numArts, breaks[i][1]))
    matId = numpy.zeros((numArts, numArts), numpy.float)
    for i1 in xrange(numArts):
        print breaks[i][0] + i1, i1, art2mscOld[breaks[i][0] + i1]
        for i2 in xrange(numArts):
            pos1, pos2 = breaks[i][0] + i1, breaks[i][0] + i2
            matId[i1, i2] = mat[new2old[pos1], new2old[pos2]]
    processMsc(matId, breaks[i][1][0])

sys.exit(0)

new2old = {}
for i, (msc, id) in enumerate(art2msc):
def loadSimMatrices(prefix = 'cmj'):
    global m, l
    m = matutils.loadMatrix(common.matrixFile(prefix + 'TFIDFsim.mm'))
    l = matutils.loadMatrix(common.matrixFile(prefix + 'LSIsim.mm'))
    return m, l