def cooccurence(): logger.info('vocabrary load...') vocab = nltkwrapper.Vocab(context) ranking = vocab.vocab(300) stems = vocab.stemmer.getdict().items() logger.info('load words coocurence...') cooc = vocab.cooccurence(dict(ranking).keys()).items() cooc.sort(lambda x,y:cmp(y[1],x[1])) cooc = [(k0,k1,c) for ((k0,k1),c) in cooc] logger.info('save...') savecsv(os.path.join(context.dbdir(), 'ranking.csv'), ranking) savecsv(os.path.join(context.dbdir(), 'cooccurence.csv'), cooc) savecsv(os.path.join(context.dbdir(), 'stem.csv'), stems)
def cooccurence(): logger.info('vocabrary load...') vocab = nltkwrapper.Vocab(context) ranking = vocab.vocab(300) stems = vocab.stemmer.getdict().items() logger.info('load words coocurence...') cooc = vocab.cooccurence(dict(ranking).keys()).items() cooc.sort(lambda x, y: cmp(y[1], x[1])) cooc = [(k0, k1, c) for ((k0, k1), c) in cooc] logger.info('save...') savecsv(os.path.join(context.dbdir(), 'ranking.csv'), ranking) savecsv(os.path.join(context.dbdir(), 'cooccurence.csv'), cooc) savecsv(os.path.join(context.dbdir(), 'stem.csv'), stems)
def saveranking(ranking): dbdir = context.dbdir() with io.open(os.path.join(dbdir, 'ranking.csv'), 'w') as wio: for word, freq in ranking: wio.write(u"\t".join([word, unicode(freq)]) + u"\n") return ranking
def bibliocount(): counter = biblio.count() items = counter.items() items.sort(lambda x, y: cmp(y[1], x[1])) items = [[author, year, title, count] for ((author, year, title), count) in items] savecsv(os.path.join(context.dbdir(), 'biblio.csv'), items)
def saveranking(ranking): dbdir = context.dbdir() with io.open(os.path.join(dbdir, 'ranking.csv'), 'w') as wio: for word, freq in ranking: wio.write(u"\t".join([word, unicode(freq)]) + u"\n") return ranking
def loadranking(): dirname = context.dbdir() ranking = [] with io.open(os.path.join(dirname, 'ranking.csv')) as fio: for line in fio: word,count = line.rstrip().split("\t") ranking.append((word, int(count))) return ranking
def loadranking(): dirname = context.dbdir() ranking = [] with io.open(os.path.join(dirname, 'ranking.csv')) as fio: for line in fio: word, count = line.rstrip().split("\t") ranking.append((word, int(count))) return ranking
def preparedir(): dirs = [ context.datadir(), context.graphdir(), context.dbdir(), context.textdatadir() ] for dirname in dirs: if not os.path.exists(dirname): os.mkdir(dirname)
def loadorgword(): dirname = context.dbdir() dic = dict() with io.open(os.path.join(dirname, 'stem.csv')) as fio: for line in fio: stem,org = line.rstrip().split("\t") dic[stem] = org stemmer = nltkwrapper.Stem() stemmer.loaddata(dic) return stemmer
def loadorgword(): dirname = context.dbdir() dic = dict() with io.open(os.path.join(dirname, 'stem.csv')) as fio: for line in fio: stem, org = line.rstrip().split("\t") dic[stem] = org stemmer = nltkwrapper.Stem() stemmer.loaddata(dic) return stemmer
def load(limit=1300): stemmer = ranking.loadorgword() ranks = ranking.loadtaggedranking(stemmer) ranks = [(w, cnt) for w,tag,cnt in ranks] rankdict = dict(ranks) words = set(w[0] for w in ranks) dbdir = context.dbdir() edges = [] with io.open(os.path.join(dbdir,'cooccurence.csv')) as fio: for line in fio: word0,word1,count = line.rstrip().split("\t") word0 = stemmer.orgword(word0) word1 = stemmer.orgword(word1) if word0 in words and word1 in words: edges.append((word0,word1,int(count))) edges.sort(lambda x,y:cmp(y[2],x[2])) edges = [(w0,w1,k) for (w0,w1,k) in edges if k>limit] return (edges, rankdict)
def load(limit=1300): stemmer = ranking.loadorgword() ranks = ranking.loadtaggedranking(stemmer) ranks = [(w, cnt) for w, tag, cnt in ranks] rankdict = dict(ranks) words = set(w[0] for w in ranks) dbdir = context.dbdir() edges = [] with io.open(os.path.join(dbdir, 'cooccurence.csv')) as fio: for line in fio: word0, word1, count = line.rstrip().split("\t") word0 = stemmer.orgword(word0) word1 = stemmer.orgword(word1) if word0 in words and word1 in words: edges.append((word0, word1, int(count))) edges.sort(lambda x, y: cmp(y[2], x[2])) edges = [(w0, w1, k) for (w0, w1, k) in edges if k > limit] return (edges, rankdict)
def ranking(): vocab = nltkwrapper.Vocab(context) ranking = vocab.vocab(300) stems = vocab.stemmer.getdict().items() savecsv(os.path.join(context.dbdir(), 'ranking.csv'), ranking) savecsv(os.path.join(context.dbdir(), 'stem.csv'), stems)
def graphdata(): data = graph.load() savecsv(os.path.join(context.dbdir(), 'graphdata.csv'), data)
def nnranking(): ranks = rankmod.loadnnranking() savecsv(os.path.join(context.dbdir(), 'nnranking.csv'), ranks)
def taggedranking(): ranks = rankmod.loadtaggedranking() savecsv(os.path.join(context.dbdir(), 'taggedranking.csv'), ranks)
def graphdata(): data = graph.load() savecsv(os.path.join(context.dbdir(), 'graphdata.csv'), data)
def bibliocount(): counter = biblio.count() items = counter.items() items.sort(lambda x,y: cmp(y[1],x[1])) items = [[author, year, title, count] for ((author, year, title), count) in items] savecsv(os.path.join(context.dbdir(), 'biblio.csv'), items)
def preparedir(): dirs = [context.datadir(), context.graphdir(), context.dbdir(), context.textdatadir()] for dirname in dirs: if not os.path.exists(dirname): os.mkdir(dirname)
def taggedranking(): ranks = rankmod.loadtaggedranking() savecsv(os.path.join(context.dbdir(), 'taggedranking.csv'), ranks)
def nnranking(): ranks = rankmod.loadnnranking() savecsv(os.path.join(context.dbdir(), 'nnranking.csv'), ranks)
def ranking(): vocab = nltkwrapper.Vocab(context) ranking = vocab.vocab(300) stems = vocab.stemmer.getdict().items() savecsv(os.path.join(context.dbdir(), 'ranking.csv'), ranking) savecsv(os.path.join(context.dbdir(), 'stem.csv'), stems)