def make_nltk_text(directory,
collapse_dirs = True,
tagged = False,
lemmatise = False,
just_content_words = False):
"""
Turn a lot of trees into an nltk style text"""
import nltk
import os
from other import tregex_engine
if type(directory) == str:
dirs = [os.path.join(directory, d) for d in os.listdir(directory) if os.path.isdir(os.path.join(directory, d))]
if len(dirs) == 0:
dirs = [directory]
elif type(directory) == list:
dirs = directory
return_tuples = False
if tagged:
return_tuples = True
if just_content_words:
lemmatise = True
query = r'__ < (/.?[A-Za-z0-9].?/ !< __)'
if not return_tuples and not lemmatise:
options = ['-o', '-t']
else:
options = ['-o']
# filthy code.
all_out = []
for d in dirs:
print "Flattening %s ... " % str(d)
res = tregex_engine(corpus = d,
query = query,
options = options,
lemmatise = lemmatise,
just_content_words = just_content_words,
return_tuples = return_tuples)
all_out.append(res)
if collapse_dirs:
tmp = []
for res in all_out:
for w in res:
tmp.append(w)
all_out = tmp
textx = nltk.Text(all_out)
else:
textx = {}
for name, text in zip(dirs, all_out):
t = nltk.Text(all_out)
textx[os.path.basename(name)] = t
return textx
def searchtree(tree, query, options = ['-t', '-o']):
import corpkit
"Searches a tree with Tregex and returns matching terminals"
import os
from other import tregex_engine
from tests import check_dit
try:
get_ipython().getoutput()
except TypeError:
have_ipython = True
except NameError:
import subprocess
have_ipython = False
fo = open('tree.tmp',"w")
fo.write(tree + '\n')
fo.close()
result = tregex_engine(query = query, check_query = True)
result = tregex_engine(query = query, options = options, corpus = "tree.tmp")
os.remove("tree.tmp")
return result
def eugener(path,
query,
depth = 5,
top = 20,
lemmatise = False,
just_content_words = False,
remove_query_from_output = False,
remove_zero_depth = False,
return_tags = False):
"""
***This is probably broken now, can fix if there's a use for it.***
get most frequent words in corpus path to left and right of query regex
path: path to corpus containing subcorpora
query: regex to match word to be zero depth
depth: number of places left and right to look
top: number of most frequent entries to return
lemmatise: wordnet lemmatisation
just_content_words: keep only n, v, a, r tagged words
remove_query_from_output: remove o
"""
import os
import nltk
import re
from collections import Counter
import pandas as pd
from textprogressbar import TextProgressBar
from other import tregex_engine
# manual lemmatisation here:
from dictionaries.word_transforms import wordlist
try:
get_ipython().getoutput()
except TypeError:
have_ipython = True
from IPython.display import display, clear_output
except NameError:
import subprocess
have_ipython = False
from tests import check_dit # probably never needed
if lemmatise:
from nltk.stem.wordnet import WordNetLemmatizer
lmtzr=WordNetLemmatizer()
regex = re.compile(query)
wordregex = re.compile('[A-Za-z0-9]')
print ''
# get list of subcorpora
dirs = [d for d in os.listdir(path) if os.path.isdir(os.path.join(path, d))]
sorted_dirs = sorted(dirs)
# define risk word
# place for our output
dfs = {}
p = TextProgressBar(len(sorted_dirs))
for index, corpus in enumerate(sorted_dirs):
p.animate(index)
# search the corpus for whole sents containing risk word
subcorpus = os.path.join(path, corpus)
if lemmatise:
query = r'__ <# (__ !< __)'
else:
query = r'__ !> __'
results = tregex_engine(query, ['-o'], subcorpus,
lemmatise = lemmatise,
just_content_words = just_content_words)
# lowercase
processed = [(r.lower(), tag) for r, tag in processed]
# remove punct
processed = [w for w in processed if re.search(wordregex, w[0])]
# a place for info about each corpus
# word list to use later
all_words = []
dicts = []
# go left and right depth times (for 2, makes [-2, -1, 0, 1, 2])
for i in range(-depth, (depth + 1)):
newdict = Counter()
matching = []
# go through each token
for index, (token, tag) in enumerate(processed):
# if token matches risk expression
if re.search(regex, token):
# get the word at depth index
# try statement for cases where the target word index isn't there
try:
if i < 0:
num = index - abs(i)
if return_tags:
matching.append(processed[num][1])
else:
matching.append(processed[num][0])
else:
if return_tags:
matching.append(processed[index + i][1])
else:
matching.append(processed[index + i][0])
except:
pass
# tally results
counted = Counter(matching)
# remove punctuation etc
for key in counted:
# commented because this stuff was moved earlier.
#if key.isalnum():
#if key not in stopwords:
#if remove_stopwords:
#if key not in stopwords:
#newdict[key] = counted[key]
#else:
#newdict[key] = counted[key]
newdict[key] = counted[key]
for w in counted.keys():
all_words.append(w)
#top_tokens = newdict.most_common(top)
dicts.append(newdict)
# make pandas series
sers = []
# for each unique word
for word in list(set(all_words)):
#get counts for each depth
series = [dct[word] for dct in dicts]
# add a total
series.append(sum([dct[word] for dct in dicts]))
#make index names for depths plus total
index_names = range(-depth, (depth + 1))
index_names.append('Total')
# turn into pandas data, and name the series the word
ser = pd.Series(series, index = index_names)
ser.name = word
sers.append(ser)
# concatenate series into dataframe
df = pd.concat(sers, axis=1)
# sort by total
tot = df.ix['Total']
df = df[tot.argsort()[::-1]]
# remove words matching the regex if need be
if remove_query_from_output:
cols = [c for c in list(df.columns) if not re.search(regex, c)]
df = pd.DataFrame(df[cols])
# remove zero depth if need be
if remove_zero_depth:
df = df.drop(0, axis = 0)
# just top entries
df = pd.DataFrame(df[list(df.columns)[:top]])
#transpose
dfs[corpus] = df.T
# complete animation, then clear
p.animate(len(sorted_dirs))
if have_ipython:
clear_output()
# some settings for good display
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
pd.set_option('expand_frame_repr', False)
pd.set_option('colheader_justify', 'right')
# print the start of each frame, then return them all
for item in sorted(dfs):
print item, '\n', dfs[item].head(), '\n'
return dfs
def dictmaker(path,
dictname,
query = 'any',
dictpath = 'data/dictionaries',
lemmatise = False,
just_content_words = False,
use_dependencies = False):
"""makes a pickle wordlist named dictname in dictpath"""
import corpkit
import os
import pickle
import re
import nltk
from time import localtime, strftime
from StringIO import StringIO
import shutil
from collections import Counter
from textprogressbar import TextProgressBar
from other import tregex_engine
try:
from IPython.display import display, clear_output
except ImportError:
pass
try:
get_ipython().getoutput()
except TypeError:
have_ipython = True
except NameError:
import subprocess
have_ipython = False
if lemmatise:
dictname = dictname + '-lemmatised'
if not dictname.endswith('.p'):
dictname = dictname + '.p'
# allow direct passing of dirs
path_is_list = False
one_big_corpus = False
if type(path) == str:
sorted_dirs = [d for d in os.listdir(path) if os.path.isdir(os.path.join(path,d))]
# if no subcorpora, just do the dir passed in
if len(sorted_dirs) == 0:
one_big_corpus = True
sorted_dirs = [path]
elif type(path) == list:
path_is_list = True
sorted_dirs = sorted(path)
if type(sorted_dirs[0]) == int:
sorted_dirs = [str(d) for d in sorted_dirs]
try:
sorted_dirs.sort(key=int)
except:
pass
try:
if not os.path.exists(dictpath):
os.makedirs(dictpath)
except IOError:
print "Error making " + dictpath + "/ directory."
while os.path.isfile(os.path.join(dictpath, dictname)):
time = strftime("%H:%M:%S", localtime())
selection = raw_input('\n%s: %s already exists in %s.\n' \
' You have the following options:\n\n' \
' a) save with a new name\n' \
' b) delete %s\n' \
' c) exit\n\nYour selection: ' % (time, dictname, dictpath, os.path.join(dictpath, dictname)))
if 'a' in selection:
sel = raw_input('\nNew save name: ')
dictname = sel
if lemmatise:
dictname = dictname.replace('-lemmatised.p', '')
dictname = dictname + '-lemmatised'
if not dictname.endswith('.p'):
dictname = dictname + '.p'
elif 'b' in selection:
os.remove(os.path.join(dictpath, dictname))
elif 'c' in selection:
print ''
return
else:
as_str = str(selection)
print ' Choice "%s" not recognised.' % selection
time = strftime("%H:%M:%S", localtime())
print '\n%s: Extracting words from files ... \n' % time
# all this just to get a list of files and make a better progress bar
if use_dependencies:
counts = []
for d in sorted_dirs:
if not one_big_corpus:
subcorpus = os.path.join(path, d)
else:
subcorpus = path
if use_dependencies:
files = [f for f in os.listdir(subcorpus) if f.endswith('.xml')]
else:
files = [f for f in os.listdir(subcorpus)]
counts.append(len(files))
num_files = sum(counts)
c = 0
p = TextProgressBar(num_files)
else:
p = TextProgressBar(len(sorted_dirs))
def tokener(xmldata):
import corpkit
"""print word, using good lemmatisation"""
from bs4 import BeautifulSoup
import gc
open_classes = ['N', 'V', 'R', 'J']
result = []
just_good_deps = SoupStrainer('tokens')
soup = BeautifulSoup(xmldata, parse_only=just_good_deps)
for token in soup.find_all('token'):
word = token.word.text
query = re.compile(r'.*')
if re.search(query, word):
if lemmatise:
word = token.lemma.text
if just_content_words:
if not token.pos.text[0] in open_classes:
continue
result.append(word)
# attempt to stop memory problems.
# not sure if this helps, though:
soup.decompose()
soup = None
data = None
gc.collect()
return result
# translate 'any' query
if query == 'any':
if lemmatise:
query = r'__ <# (__ !< __)'
else:
query = r'__ !< __'
if lemmatise:
options = ['-o']
else:
options = ['-t', '-o']
if use_dependencies:
from bs4 import BeautifulSoup, SoupStrainer
if query == 'any':
query = r'.*'
query = re.compile(query)
allwords = []
for index, d in enumerate(sorted_dirs):
if not use_dependencies:
p.animate(index)
if not path_is_list:
if len(sorted_dirs) == 1:
subcorp = d
else:
subcorp = os.path.join(path, d)
else:
subcorp = d
# check query first time through
if not use_dependencies:
if index == 0:
trees_found = tregex_engine(corpus = subcorp, check_for_trees = True)
if not trees_found:
lemmatise = False
dictname = dictname.replace('-lemmatised', '')
if trees_found:
results = tregex_engine(corpus = subcorp, options = options, query = query,
lemmatise = lemmatise,
just_content_words = just_content_words)
for result in results:
allwords.append(result)
elif use_dependencies:
regex_nonword_filter = re.compile("[A-Za-z]")
results = []
fs = [os.path.join(subcorp, f) for f in os.listdir(subcorp)]
for f in fs:
p.animate(c, str(c) + '/' + str(num_files))
c += 1
data = open(f).read()
result_from_a_file = tokener(data)
for w in result_from_a_file:
if re.search(regex_nonword_filter, w):
allwords.append(w.lower())
if not use_dependencies:
if not trees_found:
for f in os.listdir(subcorp):
raw = unicode(open(os.path.join(subcorp, f)).read(), 'utf-8', errors = 'ignore')
sent_tokenizer=nltk.data.load('tokenizers/punkt/english.pickle')
sents = sent_tokenizer.tokenize(raw)
tokenized_sents = [nltk.word_tokenize(i) for i in sents]
for sent in tokenized_sents:
for w in sent:
allwords.append(w.lower())
#100%
p.animate(len(sorted_dirs))
# make a dict
dictionary = Counter(allwords)
with open(os.path.join(dictpath, dictname), 'wb') as handle:
pickle.dump(dictionary, handle)
time = strftime("%H:%M:%S", localtime())
print '\n\n' + time + ': Done! ' + dictname + ' created in ' + dictpath + '/'
def datareader(data, plaintext = False, **kwargs):
import corpkit
"""
Returns a string of plain text from a number of kinds of data.
The kinds of data currently accepted are:
path to corpus : all trees are flattened
path to subcorpus : all trees are flattened
conc() output (list of concordance lines)
csv file generated with conc()
a string of text
"""
import os
import pandas
from other import tregex_engine
from tests import check_dit
try:
get_ipython().getoutput()
except TypeError:
have_ipython = True
except NameError:
import subprocess
have_ipython = False
tregex_engine_used = False
# if unicode, make it a string
if type(data) == unicode:
if not os.path.isdir(data):
if not os.path.isfile(data):
return good
if type(data) == str:
# if it's a file, read it
if os.path.isfile(data):
good = open(data).read()
# if it's a dir, flatten all trees
elif os.path.isdir(data):
# get all sentences newline separated
query = r'__ !< __'
options = ['-o', '-t']
# if lemmatise, we get each word on a newline
if 'lemmatise' in kwargs:
if kwargs['lemmatise'] is True:
query = r'__ <# (__ !< __)'
options = ['-o']
# check for trees ...
#while plaintext is False:
#for f in first_twenty:
#plaintext = tregex_engine(corpus = f, check_for_trees = True)
if not plaintext:
tregex_engine_used = True
results = tregex_engine(corpus = data,
options = options,
query = query,
**kwargs)
else:
results = []
fs = [os.path.join(data, f) for f in os.listdir(data)]
# do recursive if need
if any(os.path.isdir(f) for f in fs):
recursive_files = []
for dirname, dirnames, filenames in os.walk(data):
for filename in filenames:
recursive_files.append(os.path.join(dirname, filename))
fs = recursive_files
import nltk
sent_tokenizer=nltk.data.load('tokenizers/punkt/english.pickle')
for f in fs:
raw = unicode(open(f).read(), 'utf-8', errors = 'ignore')
sents = sent_tokenizer.tokenize(raw)
tokenized_sents = [nltk.word_tokenize(i) for i in sents]
for sent in tokenized_sents:
for w in sent:
results.append(w.lower())
return results
#good = '\n'.join(results)
# if a string of text,
else:
good = data
# if conc results, turn into string...
elif type(data) == pandas.core.frame.DataFrame:
# if conc lines:
try:
if list(data.columns) == ['l', 'm', 'r']:
conc_lines = True
else:
conc_lines = False
except:
conc_lines = False
if conc_lines:
# may not be unicode!?
good = [' '.join(list(data.ix[l])) for l in list(data.index)]
else:
good = data
# make unicode
if not tregex_engine_used:
try:
good = unicode(good, 'utf-8', errors = 'ignore')
except TypeError:
pass
return good
