def buildMscOverlapMatrix(lang): logging.info("building MSC overlap matrix") arts = [ art for art in docsim.getArts( ARTS_FILE, acceptNoBody=True, acceptNoMsc=False) if art.language == lang or lang == 'any' ] cats, rcats = loadMsc2Id(lang) # from buildPure logging.info("computing MSC matrices") overlap = numpy.zeros( (len(cats), len(cats)), dtype=int ) # binary msc similarity (with fixed msc hierarchy = identity) for art in arts: for msc1 in art.msc: for msc2 in art.msc: overlap[rcats[mscs.niceMSC(msc1)[0]], rcats[mscs.niceMSC(msc2)[0]]] += 1 resultOverlap = numpy.zeros((len(cats), len(cats)), dtype=numpy.float32) for i in xrange(resultOverlap.shape[0]): max = numpy.max(overlap[i]) for j in xrange(resultOverlap.shape[1]): resultOverlap[i, j] = math.log(1.0 + 100.0 * overlap[i, j] / max) / math.log(101) matutils.saveMatrix(resultOverlap, common.matrixFile("mscs_overlap_%s.mm" % lang), sparse=False) return resultOverlap
def buildMscPureMatrix(lang): if os.path.exists(ARTS_FILE): logging.warning( "SKIPPING creating MSC database from meta.xml files (using old %s); is this what you want?" % ARTS_FILE) else: logging.info("creating MSC database from meta.xml files") createMscsDb( ) # only run this when collection changes (test for file existence / delete file explicitly?) arts = [ art for art in docsim.getArts( ARTS_FILE, acceptNoBody=True, acceptNoMsc=False) if art.language == lang or lang == 'any' ] cats = dict(enumerate(mscs.getFreqMSCs(arts, minFreq=1, useMains=False))) rcats = utils_dml.reverseMap(cats) saveMsc2Id(cats, lang) logging.info("building MSC binary matrix") resultBin = numpy.zeros( (len(cats), len(cats)), dtype=numpy.float32 ) # binary msc similarity (with fixed msc hierarchy = identity) for idi, cati in cats.iteritems(): for idj, catj in cats.iteritems(): # print idi, cati, idj, catj if idi == idj: resultBin[idi, idj] = 1.0 else: resultBin[idi, idj] = 0.0 matutils.saveMatrix(resultBin, common.matrixFile("mscs_bin_%s.mm" % lang), sparse=False) return resultBin
def buildMscPureMatrix(lang): if os.path.exists(ARTS_FILE): logging.warning( "SKIPPING creating MSC database from meta.xml files (using old %s); is this what you want?" % ARTS_FILE ) else: logging.info("creating MSC database from meta.xml files") createMscsDb() # only run this when collection changes (test for file existence / delete file explicitly?) arts = [ art for art in docsim.getArts(ARTS_FILE, acceptNoBody=True, acceptNoMsc=False) if art.language == lang or lang == "any" ] cats = dict(enumerate(mscs.getFreqMSCs(arts, minFreq=1, useMains=False))) rcats = utils_dml.reverseMap(cats) saveMsc2Id(cats, lang) logging.info("building MSC binary matrix") resultBin = numpy.zeros( (len(cats), len(cats)), dtype=numpy.float32 ) # binary msc similarity (with fixed msc hierarchy = identity) for idi, cati in cats.iteritems(): for idj, catj in cats.iteritems(): # print idi, cati, idj, catj if idi == idj: resultBin[idi, idj] = 1.0 else: resultBin[idi, idj] = 0.0 matutils.saveMatrix(resultBin, common.matrixFile("mscs_bin_%s.mm" % lang), sparse=False) return resultBin
def buildMscOverlapMatrix(lang): logging.info("building MSC overlap matrix") arts = [ art for art in docsim.getArts(ARTS_FILE, acceptNoBody=True, acceptNoMsc=False) if art.language == lang or lang == "any" ] cats, rcats = loadMsc2Id(lang) # from buildPure logging.info("computing MSC matrices") overlap = numpy.zeros( (len(cats), len(cats)), dtype=int ) # binary msc similarity (with fixed msc hierarchy = identity) for art in arts: for msc1 in art.msc: for msc2 in art.msc: overlap[rcats[mscs.niceMSC(msc1)[0]], rcats[mscs.niceMSC(msc2)[0]]] += 1 resultOverlap = numpy.zeros((len(cats), len(cats)), dtype=numpy.float32) for i in xrange(resultOverlap.shape[0]): max = numpy.max(overlap[i]) for j in xrange(resultOverlap.shape[1]): resultOverlap[i, j] = math.log(1.0 + 100.0 * overlap[i, j] / max) / math.log(101) matutils.saveMatrix(resultOverlap, common.matrixFile("mscs_overlap_%s.mm" % lang), sparse=False) return resultOverlap
logging.info("avg similarity within %s: %f" % (id, numpy.mean(mat))) logging.info("stddev similarity within %s: %f" % (id, numpy.std(mat))) logging.debug("creating msc=%s luminescence matrix" % (id)) print mat matLum = getLum(mat).astype(numpy.uint8) fname = "%s_sim_lum%i_%s.png" % (SIM_TYPE, LUM_SCALE, id) logging.debug("saving msc=%s similarity luminescence matrix to %s" % (id, fname)) logging.info("saving to %s" % fname) i = Image.fromarray(matLum, 'L') i.save(fname) logging.info("loading articles") #arts = docsim.getArts(common.dbFile('serial_eng', '1msc')) arts = docsim.getArts('/home/radim/workspace/data/dml/results/serial_msc.pdl') for art in arts: art.fullmsc = art.msc[:] art.msc = tuple(removeDup([mscs.niceMSC(msc)[0] for msc in art.msc])) arts = [ art for art in arts if art.id_int in ipyutils.rdocids and art.language == "eng" and len(art.msc) == 1 ] art2msc = [(art.fullmsc, art.id_int, art.msc) for art in arts] art2msc.sort() del arts logging.info("len(art2msc)=%i" % len(art2msc))
def buildMscCentroidMatrix(language): logging.info("building MSC centroid matrix from %s" % ARTS_FILE) arts = [ art for art in docsim.getArts( ARTS_FILE, acceptNoBody=False, acceptNoMsc=False) if art.language == language or language == 'any' ] prefix = 'mscs_serial_%s_' % language matFile = common.matrixFile(prefix + 'TFIDF_T.mm') if os.path.exists(matFile): logging.warning( 'SKIPPING creating TFIDF matrix for %s (file %s present). Is this what you wanted?' % (language, matFile)) tfidf = matutils.loadMatrix(matFile).tocsr() else: logging.info('creating TFIDF matrix for %s to %s' % (language, matFile)) tfidf = docsim.buildTFIDFMatrices(arts, prefix=prefix, saveMatrices=False).tocsr() ipyutils.loadDicts(prefix=prefix) arts = [ art for art in arts if art.id_int in ipyutils.rdocids ] # remove articles that had empty body (according to their tfidf vector) if len(ipyutils.rdocids) != len(arts): logging.error( "no. of TFIDF document = %i, but there are %i documents in the database (mismatch)" % (len(ipyutils.rdocids), len(arts))) raise Exception( "different size of database/dictionary; version mismatch?") cats, rcats = loadMsc2Id(language) # from buildPure # print "mscs:", cats logging.info("loading tfidf collection matrix (for centroids)") tfidf = matutils.loadMatrix( common.matrixFile('gensim_' + language + 'TFIDF_T.mm')).tocsr() logging.debug("loaded %ix%i matrix" % tfidf.shape) logging.info("computing centroids") centroids = numpy.zeros((len(cats), tfidf.shape[1]), numpy.float) # print "centroids.shape =", centroids.shape num = numpy.zeros((len(cats), ), numpy.int) artCnt = 0 for art in arts: if not art.id_int in ipyutils.rdocids: logging.warning("article not found among docids: %s" % art) continue artCnt += 1 artId = ipyutils.rdocids[art.id_int] tops = [mscs.niceMSC(msc)[0] for msc in art.msc] tops = set( tops ) # only count each top-level once (comment out this line to count e.g. 30H55 and 30.13 twice for this article, as cat. 30) for top in tops: mscId = rcats[top] vec = tfidf[artId].toarray() vec.shape = (vec.size, ) # print "vec.shape = ", vec.shape centroids[mscId] += vec num[mscId] += 1 if artCnt < 10 or artCnt % 1000 == 0: logging.debug( "sanity check - article %s has id %i and has mscs=%s, mscsIds=%s" % (art.id_int, artId, art.msc, [rcats[mscs.niceMSC(msc)[0]] for msc in art.msc])) if not artCnt == tfidf.shape[0]: raise Exception("not all articles used; database/matrix mismatch?") for i, vec in enumerate(centroids): logging.info( "centroid for msc %s (id %i) is an average of %i vectors" % (cats[i], i, num[i])) if numpy.sum(numpy.abs(vec)) == 0: logging.warning("empty centroid for msc %s (msc int id %i)" % (cats[i], i)) for mscId in cats.iterkeys(): centroids[mscId] /= num[mscId] logging.info( "used %i articles for %i vectors (articles may have more than one msc and so can be counted more than once)" % (artCnt, sum(num))) logging.info("computing MSC centroid matrix") resultCentroid = numpy.zeros((len(cats), len(cats)), dtype=numpy.float32) for idi, cati in cats.iteritems(): for idj, catj in cats.iteritems(): # print idi, cati, idj, catj sim = matutils.cossim(centroids[idi], centroids[idj]) if numpy.isfinite(sim): resultCentroid[idi, idj] = sim else: resultCentroid[idi, idj] = 0.0 matutils.saveMatrix(resultCentroid, common.matrixFile("mscs_centroid_%s.mm" % language), sparse=False)
def processMsc(mat, id): logging.info("avg similarity within %s: %f" % (id, numpy.mean(mat))) logging.info("stddev similarity within %s: %f" % (id, numpy.std(mat))) logging.debug("creating msc=%s luminescence matrix" % (id)) print mat matLum = getLum(mat).astype(numpy.uint8) fname = "%s_sim_lum%i_%s.png" % (SIM_TYPE, LUM_SCALE, id) logging.debug("saving msc=%s similarity luminescence matrix to %s" % (id, fname)) logging.info("saving to %s" % fname) i = Image.fromarray(matLum, 'L') i.save(fname) logging.info("loading articles") #arts = docsim.getArts(common.dbFile('serial_eng', '1msc')) arts = docsim.getArts('/home/radim/workspace/data/dml/results/serial_msc.pdl') for art in arts: art.fullmsc = art.msc[:] art.msc = tuple(removeDup([mscs.niceMSC(msc)[0] for msc in art.msc])) arts = [art for art in arts if art.id_int in ipyutils.rdocids and art.language == "eng" and len(art.msc) == 1] art2msc = [(art.fullmsc, art.id_int, art.msc) for art in arts] art2msc.sort() del arts logging.info("len(art2msc)=%i" % len(art2msc)) print "first ten art2msc:", art2msc[:10] print "last ten art2msc:", art2msc[-10:]
def buildMscCentroidMatrix(language): logging.info("building MSC centroid matrix from %s" % ARTS_FILE) arts = [ art for art in docsim.getArts(ARTS_FILE, acceptNoBody=False, acceptNoMsc=False) if art.language == language or language == "any" ] prefix = "mscs_serial_%s_" % language matFile = common.matrixFile(prefix + "TFIDF_T.mm") if os.path.exists(matFile): logging.warning( "SKIPPING creating TFIDF matrix for %s (file %s present). Is this what you wanted?" % (language, matFile) ) tfidf = matutils.loadMatrix(matFile).tocsr() else: logging.info("creating TFIDF matrix for %s to %s" % (language, matFile)) tfidf = docsim.buildTFIDFMatrices(arts, prefix=prefix, saveMatrices=False).tocsr() ipyutils.loadDicts(prefix=prefix) arts = [ art for art in arts if art.id_int in ipyutils.rdocids ] # remove articles that had empty body (according to their tfidf vector) if len(ipyutils.rdocids) != len(arts): logging.error( "no. of TFIDF document = %i, but there are %i documents in the database (mismatch)" % (len(ipyutils.rdocids), len(arts)) ) raise Exception("different size of database/dictionary; version mismatch?") cats, rcats = loadMsc2Id(language) # from buildPure # print "mscs:", cats logging.info("loading tfidf collection matrix (for centroids)") tfidf = matutils.loadMatrix(common.matrixFile("gensim_" + language + "TFIDF_T.mm")).tocsr() logging.debug("loaded %ix%i matrix" % tfidf.shape) logging.info("computing centroids") centroids = numpy.zeros((len(cats), tfidf.shape[1]), numpy.float) # print "centroids.shape =", centroids.shape num = numpy.zeros((len(cats),), numpy.int) artCnt = 0 for art in arts: if not art.id_int in ipyutils.rdocids: logging.warning("article not found among docids: %s" % art) continue artCnt += 1 artId = ipyutils.rdocids[art.id_int] tops = [mscs.niceMSC(msc)[0] for msc in art.msc] tops = set( tops ) # only count each top-level once (comment out this line to count e.g. 30H55 and 30.13 twice for this article, as cat. 30) for top in tops: mscId = rcats[top] vec = tfidf[artId].toarray() vec.shape = (vec.size,) # print "vec.shape = ", vec.shape centroids[mscId] += vec num[mscId] += 1 if artCnt < 10 or artCnt % 1000 == 0: logging.debug( "sanity check - article %s has id %i and has mscs=%s, mscsIds=%s" % (art.id_int, artId, art.msc, [rcats[mscs.niceMSC(msc)[0]] for msc in art.msc]) ) if not artCnt == tfidf.shape[0]: raise Exception("not all articles used; database/matrix mismatch?") for i, vec in enumerate(centroids): logging.info("centroid for msc %s (id %i) is an average of %i vectors" % (cats[i], i, num[i])) if numpy.sum(numpy.abs(vec)) == 0: logging.warning("empty centroid for msc %s (msc int id %i)" % (cats[i], i)) for mscId in cats.iterkeys(): centroids[mscId] /= num[mscId] logging.info( "used %i articles for %i vectors (articles may have more than one msc and so can be counted more than once)" % (artCnt, sum(num)) ) logging.info("computing MSC centroid matrix") resultCentroid = numpy.zeros((len(cats), len(cats)), dtype=numpy.float32) for idi, cati in cats.iteritems(): for idj, catj in cats.iteritems(): # print idi, cati, idj, catj sim = matutils.cossim(centroids[idi], centroids[idj]) if numpy.isfinite(sim): resultCentroid[idi, idj] = sim else: resultCentroid[idi, idj] = 0.0 matutils.saveMatrix(resultCentroid, common.matrixFile("mscs_centroid_%s.mm" % language), sparse=False)