def test_smooth_prob():
p = PreshCounter()
# 1 10
# 2 6
# 3 4
# 5 2
# 8 1
for i in range(10):
p.inc(100-i, 1) # 10 items of freq 1
for i in range(6):
p.inc(90 - i, 2) # 6 items of freq 2
for i in range(4):
p.inc(80 - i, 3) # 4 items of freq 3
for i in range(2):
p.inc(70 - i, 5) # 2 items of freq 5
for i in range(1):
p.inc(60 - i, 8) # 1 item of freq 8
assert p.total == (10 * 1) + (6 * 2) + (4 * 3) + (2 * 5) + (1 * 8)
assert p.prob(100) == 1.0 / p.total
assert p.prob(200) == 0.0
assert p.prob(60) == 8.0 / p.total
p.smooth()
assert p.smoother(1) < 1.0
assert p.smoother(8) < 8.0
assert p.prob(1000) < p.prob(100)
for event, count in reversed(sorted(p, key=lambda it: it[1])):
assert p.smoother(count) < count
def count_frequencies(lang_id, corpus_dir):
"""
Counts word count en document count for each word in the corpus.
:param lang_id:
:param corpus_dir: directory with text files
:return:
"""
nlp = spacy.load(lang_id)
vocab = nlp.vocab
tokenizer = nlp.tokenizer
counts = PreshCounter()
doccounts = PreshCounter()
for filename in os.listdir(corpus_dir):
with codecs.open(os.path.join(corpus_dir, filename),
encoding='utf-8') as f:
data = f.read()
doc = tokenizer(data)
doc.count_by(ORTH, counts=counts)
doccount = doc.count_by(ORTH)
for k, v in doccount.iteritems():
doccounts.inc(k, 1)
return counts, doccounts, tokenizer
def test_smooth_prob():
p = PreshCounter()
# 1 10
# 2 6
# 3 4
# 5 2
# 8 1
for i in range(10):
p.inc(100 - i, 1) # 10 items of freq 1
for i in range(6):
p.inc(90 - i, 2) # 6 items of freq 2
for i in range(4):
p.inc(80 - i, 3) # 4 items of freq 3
for i in range(2):
p.inc(70 - i, 5) # 2 items of freq 5
for i in range(1):
p.inc(60 - i, 8) # 1 item of freq 8
assert p.total == (10 * 1) + (6 * 2) + (4 * 3) + (2 * 5) + (1 * 8)
assert p.prob(100) == 1.0 / p.total
assert p.prob(200) == 0.0
assert p.prob(60) == 8.0 / p.total
p.smooth()
assert p.smoother(1) < 1.0
assert p.smoother(8) < 8.0
assert p.prob(1000) < p.prob(100)
for event, count in reversed(sorted(p, key=lambda it: it[1])):
assert p.smoother(count) < count
class Corpus(object):
def __init__(self, directory, min_freq=10):
self.directory = directory
self.counts = PreshCounter()
self.strings = {}
self.min_freq = min_freq
def count_doc(self, words):
# Get counts for this document
doc_counts = PreshCounter()
doc_strings = {}
for word in words:
key = hash_string(word)
doc_counts.inc(key, 1)
doc_strings[key] = word
n = 0
for key, count in doc_counts:
self.counts.inc(key, count)
# TODO: Why doesn't inc return this? =/
corpus_count = self.counts[key]
# Remember the string when we exceed min count
if corpus_count >= self.min_freq and (corpus_count - count) < self.min_freq:
self.strings[key] = doc_strings[key]
n += count
return n
def __iter__(self):
for text_loc in iter_dir(self.directory):
with io.open(text_loc, 'r', encoding='utf8') as file_:
sent_strs = list(file_)
random.shuffle(sent_strs)
for sent_str in sent_strs:
yield sent_str.split()
def _read_freqs(loc, max_length=100, min_doc_freq=5, min_freq=200):
if not loc.exists():
print("Warning: Frequencies file not found")
return {}, 0.0
counts = PreshCounter()
total = 0
if str(loc).endswith('gz'):
file_ = gzip.open(str(loc))
else:
file_ = loc.open()
for i, line in enumerate(file_):
freq, doc_freq, key = line.split('\t', 2)
freq = int(freq)
counts.inc(i+1, freq)
total += freq
counts.smooth()
log_total = math.log(total)
probs = {}
for line in loc.open():
freq, doc_freq, key = line.split('\t', 2)
doc_freq = int(doc_freq)
freq = int(freq)
if doc_freq >= min_doc_freq and freq >= min_freq and len(key) < max_length:
word = literal_eval(key)
smooth_count = counts.smoother(int(freq))
log_smooth_count = math.log(smooth_count)
probs[word] = math.log(smooth_count) - log_total
oov_prob = math.log(counts.smoother(0)) - log_total
return probs, oov_prob
class Corpus(object):
def __init__(self, directory, min_freq=10):
self.directory = directory
self.counts = PreshCounter()
self.strings = {}
self.min_freq = min_freq
def count_doc(self, words):
# Get counts for this document
doc_counts = PreshCounter()
doc_strings = {}
for word in words:
key = hash_string(word)
doc_counts.inc(key, 1)
doc_strings[key] = word
n = 0
for key, count in doc_counts:
self.counts.inc(key, count)
# TODO: Why doesn't inc return this? =/
corpus_count = self.counts[key]
# Remember the string when we exceed min count
if corpus_count >= self.min_freq and (corpus_count -
count) < self.min_freq:
self.strings[key] = doc_strings[key]
n += count
return n
def __iter__(self):
for text_loc in iter_dir(self.directory):
with io.open(text_loc, 'r', encoding='utf8') as file_:
sent_strs = list(file_)
random.shuffle(sent_strs)
for sent_str in sent_strs:
yield sent_str.split()
def read_freqs(freqs_loc, max_length=100, min_doc_freq=5, min_freq=50):
counts = PreshCounter()
total = 0
with freqs_loc.open() as f:
for i, line in enumerate(f):
freq, doc_freq, key = line.rstrip().split("\t", 2)
freq = int(freq)
counts.inc(i + 1, freq)
total += freq
counts.smooth()
log_total = math.log(total)
probs = {}
with freqs_loc.open() as f:
for line in tqdm(f):
freq, doc_freq, key = line.rstrip().split("\t", 2)
doc_freq = int(doc_freq)
freq = int(freq)
if doc_freq >= min_doc_freq and freq >= min_freq and len(key) < max_length:
try:
word = literal_eval(key)
except SyntaxError:
# Take odd strings literally.
word = literal_eval("'%s'" % key)
smooth_count = counts.smoother(int(freq))
probs[word] = math.log(smooth_count) - log_total
oov_prob = math.log(counts.smoother(0)) - log_total
return probs, oov_prob
def read_freqs(freqs_loc, max_length=100, min_doc_freq=5, min_freq=50):
counts = PreshCounter()
total = 0
with freqs_loc.open() as f:
for i, line in enumerate(f):
freq, doc_freq, key = line.rstrip().split("\t", 2)
freq = int(freq)
counts.inc(i + 1, freq)
total += freq
counts.smooth()
log_total = math.log(total)
probs = {}
with freqs_loc.open() as f:
for line in tqdm(f):
freq, doc_freq, key = line.rstrip().split("\t", 2)
doc_freq = int(doc_freq)
freq = int(freq)
if doc_freq >= min_doc_freq and freq >= min_freq and len(
key) < max_length:
try:
word = literal_eval(key)
except SyntaxError:
# Take odd strings literally.
word = literal_eval("'%s'" % key)
smooth_count = counts.smoother(int(freq))
probs[word] = math.log(smooth_count) - log_total
oov_prob = math.log(counts.smoother(0)) - log_total
return probs, oov_prob
def read_freqs(freqs_loc, max_length=100, min_doc_freq=5, min_freq=50):
print("Counting frequencies...")
counts = PreshCounter()
total = 0
with freqs_loc.open() as f:
for i, line in enumerate(f):
freq, doc_freq, key = line.rstrip().split('\t', 2)
freq = int(freq)
counts.inc(i + 1, freq)
total += freq
counts.smooth()
log_total = math.log(total)
probs = {}
with freqs_loc.open() as f:
for line in tqdm(f):
freq, doc_freq, key = line.rstrip().split('\t', 2)
doc_freq = int(doc_freq)
freq = int(freq)
if doc_freq >= min_doc_freq and freq >= min_freq and len(
key) < max_length:
word = literal_eval(key)
smooth_count = counts.smoother(int(freq))
probs[word] = math.log(smooth_count) - log_total
oov_prob = math.log(counts.smoother(0)) - log_total
return probs, oov_prob
def _read_probs_from_freqs(loc, max_length=100, min_doc_freq=5, min_freq=200):
if not loc.exists():
print("Warning: Frequencies file not found")
return {}, 0.0
counts = PreshCounter()
total = 0
if str(loc).endswith('gz'):
file_ = gzip.open(str(loc))
else:
file_ = loc.open()
for i, line in enumerate(file_):
freq, doc_freq, key = line.rstrip().split('\t', 2)
freq = int(freq)
counts.inc(i + 1, freq)
total += freq
counts.smooth()
log_total = math.log(total)
if str(loc).endswith('gz'):
file_ = gzip.open(str(loc))
else:
file_ = loc.open()
probs = {}
for line in file_:
freq, doc_freq, key = line.rstrip().split('\t', 2)
doc_freq = int(doc_freq)
freq = int(freq)
if doc_freq >= min_doc_freq and freq >= min_freq and len(
key) < max_length:
word = literal_eval(key)
smooth_count = counts.smoother(int(freq))
probs[word] = math.log(smooth_count) - log_total
oov_prob = math.log(counts.smoother(0)) - log_total
return probs, oov_prob
def countTheWords(doc):
counts = PreshCounter()
for word in doc:
counts.inc(word.orth, 1)
for (word_id, count) in counts:
print(count, nlp.vocab.strings[word_id])
return counts
def test_unsmooth_prob():
counter = PreshCounter()
assert counter.prob(12) == 0.0
counter.inc(12, 1)
assert counter.prob(12) == 1.0
counter.inc(14, 10)
assert counter.prob(14) == 10 / 11
assert counter.prob(12) == 1.0 / 11
def test_unsmooth_prob():
counter = PreshCounter()
assert counter.prob(12) == 0.0
counter.inc(12, 1)
assert counter.prob(12) == 1.0
counter.inc(14, 10)
assert counter.prob(14) == 10 / 11
assert counter.prob(12) == 1.0 / 11
def get_features(docs, n_docs, max_length=100):
Xs = np.zeros((n_docs, max_length), dtype=np.int32)
counts = PreshCounter()
for i, doc in enumerate(docs):
doc.count_by
for j, token in enumerate(doc[:max_length]):
if token.has_vector:
Xs[i, j] = token.rank
counts.inc(token.rank, 1)
else:
Xs[i, j] = 0
return Xs, counts
def merge_counts(locs, out_loc):
string_map = StringStore()
counts = PreshCounter()
for loc in locs:
with io.open(loc, 'r', encoding='utf8') as file_:
for line in file_:
freq, word = line.strip().split('\t', 1)
orth = string_map[word]
counts.inc(orth, int(freq))
with io.open(out_loc, 'w', encoding='utf8') as file_:
for orth, count in counts:
string = string_map[orth]
file_.write('%d\t%s\n' % (count, string))
def merge_counts(locs, out_loc):
string_map = StringStore()
counts = PreshCounter()
for loc in locs:
with codecs.open(loc, 'r', 'utf8') as file_:
for line in file_:
freq, word = line.strip().split('\t', 1)
orth = string_map[word]
counts.inc(orth, int(freq))
with codecs.open(out_loc, 'w', 'utf8') as file_:
for orth, count in counts:
string = string_map[orth]
file_.write('%d\t%s\n' % (count, string))
def process(batch_id, inputs, output_dir, lang, n_threads, batch_size,
min_ngram, max_ngram):
logging.info('Processing batch_id: {}'.format(batch_id))
subtrees = PreshCounter()
subtrees_string_map = StringStore()
noun_chunks = PreshCounter()
noun_chunks_string_map = StringStore()
if lang.lower() == "en":
from spacy.en import English
NLU = English()
NLU.matcher = None
elif lang.lower() == "id":
from spacy.id import Indonesian
NLU = Indonesian()
NLU.matcher = None
for i, doc in enumerate(
NLU.pipe(inputs, batch_size=batch_size, n_threads=n_threads)):
phrases = set()
for tok in doc:
st_len = len(list(tok.subtree))
if min_ngram <= st_len <= max_ngram:
st = ''.join([rep_text(t.text_with_ws)
for t in tok.subtree]).strip()
orth = subtrees_string_map[st]
subtrees.inc(orth, 1)
for np in doc.noun_chunks:
if min_ngram <= len(np) <= max_ngram:
st = ''.join([rep_text(t.text_with_ws) for t in np]).strip()
orth = noun_chunks_string_map[st]
noun_chunks.inc(orth, 1)
if i % batch_size == 0:
logging.info('Processing batch_id: {}, doc: {}'.format(
batch_id, i))
output_fname = path.join(output_dir, 'batch{}.st.freq'.format(batch_id))
with io.open(output_fname, 'w', encoding='utf-8') as out:
for orth, count in subtrees:
st = subtrees_string_map[orth]
if count >= 5 and '!LONGWORD!' not in st:
out.write('{}\t{}\n'.format(count, st))
output_fname = path.join(output_dir, 'batch{}.np.freq'.format(batch_id))
with io.open(output_fname, 'w', encoding='utf-8') as out:
for orth, count in noun_chunks:
if count >= 5:
st = noun_chunks_string_map[orth]
out.write('{}\t{}\n'.format(count, st))
class Corpus(object):
def __init__(self, directory):
self.directory = directory
self.counts = PreshCounter()
def count_doc(self, doc):
for word in doc:
self.counts.inc(word.orth, 1)
def __iter__(self):
for text_loc in self.directory.rglob("*.txt"):
with io.open(text_loc, "r", encoding="utf8") as file_:
text = file_.read()
yield text
class Corpus(object):
def __init__(self, directory, min_freq=10):
self.directory = directory
self.counts = PreshCounter()
self.strings = {}
self.min_freq = min_freq
def count_doc(self, doc):
# Get counts for this document
for word in doc:
self.counts.inc(word.orth, 1)
return len(doc)
def __iter__(self):
for text_loc in iter_dir(self.directory):
with io.open(text_loc, 'r', encoding='utf8') as file_:
text = file_.read()
yield text
def main(patterns_loc, text_loc, counts_loc, n=10000000):
nlp = English(parser=False, tagger=False, entity=False)
print("Make matcher")
phrases = read_gazetteer(nlp.tokenizer, patterns_loc, n=n)
counts = PreshCounter()
t1 = time.time()
for mwe in get_matches(nlp.tokenizer, phrases, read_text(text_loc)):
counts.inc(hash_string(mwe.text), 1)
t2 = time.time()
print("10m tokens in %d s" % (t2 - t1))
with codecs.open(counts_loc, 'w', 'utf8') as file_:
for phrase in read_gazetteer(nlp.tokenizer, patterns_loc, n=n):
text = phrase.string
key = hash_string(text)
count = counts[key]
if count != 0:
file_.write('%d\t%s\n' % (count, text))
def test_large_freqs():
if 'TEST_FILE_LOC' in os.environ:
loc = os.environ['TEST_FILE_LOC']
else:
return None
counts = PreshCounter()
for i, line in enumerate(open(loc)):
line = line.strip()
if not line:
continue
freq = int(line.split()[0])
counts.inc(i + 1, freq)
oov = i + 2
assert counts.prob(oov) == 0.0
assert counts.prob(1) < 0.1
counts.smooth()
assert counts.prob(oov) > 0
assert counts.prob(oov) < counts.prob(i)
def test_large_freqs():
if 'TEST_FILE_LOC' in os.environ:
loc = os.environ['TEST_FILE_LOC']
else:
return None
counts = PreshCounter()
for i, line in enumerate(open(loc)):
line = line.strip()
if not line:
continue
freq = int(line.split()[0])
counts.inc(i+1, freq)
oov = i+2
assert counts.prob(oov) == 0.0
assert counts.prob(1) < 0.1
counts.smooth()
assert counts.prob(oov) > 0
assert counts.prob(oov) < counts.prob(i)
def main(patterns_loc, text_loc, counts_loc, n=10000000):
nlp = English(parser=False, tagger=False, entity=False)
print("Make matcher")
phrases = read_gazetteer(nlp.tokenizer, patterns_loc, n=n)
counts = PreshCounter()
t1 = time.time()
for mwe in get_matches(nlp.tokenizer, phrases, read_text(text_loc)):
counts.inc(hash_string(mwe.text), 1)
t2 = time.time()
print("10m tokens in %d s" % (t2 - t1))
with codecs.open(counts_loc, 'w', 'utf8') as file_:
for phrase in read_gazetteer(nlp.tokenizer, patterns_loc, n=n):
text = phrase.string
key = hash_string(text)
count = counts[key]
if count != 0:
file_.write('%d\t%s\n' % (count, text))
class Corpus(object):
def __init__(self, directory, min_freq=10):
self.directory = directory
self.counts = PreshCounter()
self.strings = {}
self.min_freq = min_freq
def count_doc(self, doc):
# Get counts for this document
for word in doc:
self.counts.inc(word.orth, 1)
return len(doc)
def __iter__(self):
for text_loc in iter_dir(self.directory):
with text_loc.open("r", encoding="utf-8") as file_:
text = file_.read()
yield text
def count_doc(self, words):
# Get counts for this document
doc_counts = PreshCounter()
doc_strings = {}
for word in words:
key = hash_string(word)
doc_counts.inc(key, 1)
doc_strings[key] = word
n = 0
for key, count in doc_counts:
self.counts.inc(key, count)
# TODO: Why doesn't inc return this? =/
corpus_count = self.counts[key]
# Remember the string when we exceed min count
if corpus_count >= self.min_freq and (corpus_count - count) < self.min_freq:
self.strings[key] = doc_strings[key]
n += count
return n
def count_doc(self, words):
# Get counts for this document
doc_counts = PreshCounter()
doc_strings = {}
for word in words:
key = hash_string(word)
doc_counts.inc(key, 1)
doc_strings[key] = word
n = 0
for key, count in doc_counts:
self.counts.inc(key, count)
# TODO: Why doesn't inc return this? =/
corpus_count = self.counts[key]
# Remember the string when we exceed min count
if corpus_count >= self.min_freq and (corpus_count -
count) < self.min_freq:
self.strings[key] = doc_strings[key]
n += count
return n
def count_doc(self, words):
doc_counts = PreshCounter()
doc_strings = {}
for word in words:
key = hash_string(word)
doc_counts.inc(key, 1)
doc_strings[key] = word
n = 0
for key, count in doc_counts:
self.counts.inc(key, count)
corpus_count = self.counts[key]
if corpus_count >= self.min_freq and (corpus_count -
count) < self.min_freq:
self.strings[key] = doc_strings[key]
n += count
return n
class Corpus(object):
def __init__(self, directory, min_freq=10):
self.directory = directory
self.counts = PreshCounter()
self.strings = {}
self.min_freq = min_freq
def count_doc(self, words):
doc_counts = PreshCounter()
doc_strings = {}
for word in words:
key = hash_string(word)
doc_counts.inc(key, 1)
doc_strings[key] = word
n = 0
for key, count in doc_counts:
self.counts.inc(key, count)
corpus_count = self.counts[key]
if corpus_count >= self.min_freq and (corpus_count -
count) < self.min_freq:
self.strings[key] = doc_strings[key]
n += count
return n
def __iter__(self):
for text_loc in trainModel.iterDir(self.directory):
with io.open(text_loc, 'r', encoding='utf8') as file_:
sent_strs = list(file_)
random.shuffle(sent_strs)
for sent_str in sent_strs:
yield sent_str.split()
def read_probs(freqs_path, max_length=100, min_doc_freq=5, min_freq=200):
counts = PreshCounter()
total = 0
freqs_file = check_unzip(freqs_path)
for i, line in enumerate(freqs_file):
freq, doc_freq, key = line.rstrip().split('\t', 2)
freq = int(freq)
counts.inc(i+1, freq)
total += freq
counts.smooth()
log_total = math.log(total)
freqs_file = check_unzip(freqs_path)
probs = {}
for line in freqs_file:
freq, doc_freq, key = line.rstrip().split('\t', 2)
doc_freq = int(doc_freq)
freq = int(freq)
if doc_freq >= min_doc_freq and freq >= min_freq and len(key) < max_length:
word = literal_eval(key)
smooth_count = counts.smoother(int(freq))
probs[word] = math.log(smooth_count) - log_total
oov_prob = math.log(counts.smoother(0)) - log_total
return probs, oov_prob
def read_probs(freqs_path, max_length=100, min_doc_freq=5, min_freq=200):
counts = PreshCounter()
total = 0
freqs_file = check_unzip(freqs_path)
for i, line in enumerate(freqs_file):
freq, doc_freq, key = line.rstrip().split('\t', 2)
freq = int(freq)
counts.inc(i+1, freq)
total += freq
counts.smooth()
log_total = math.log(total)
freqs_file = check_unzip(freqs_path)
probs = {}
for line in freqs_file:
freq, doc_freq, key = line.rstrip().split('\t', 2)
doc_freq = int(doc_freq)
freq = int(freq)
if doc_freq >= min_doc_freq and freq >= min_freq and len(key) < max_length:
word = literal_eval(key)
smooth_count = counts.smoother(int(freq))
probs[word] = math.log(smooth_count) - log_total
oov_prob = math.log(counts.smoother(0)) - log_total
return probs, oov_prob
def read_freqs(freqs_loc, max_length=100, min_doc_freq=5, min_freq=50):
print("Counting frequencies...")
counts = PreshCounter()
total = 0
with freqs_loc.open() as f:
for i, line in enumerate(f):
freq, doc_freq, key = line.rstrip().split('\t', 2)
freq = int(freq)
counts.inc(i + 1, freq)
total += freq
counts.smooth()
log_total = math.log(total)
probs = {}
with freqs_loc.open() as f:
for line in tqdm(f):
freq, doc_freq, key = line.rstrip().split('\t', 2)
doc_freq = int(doc_freq)
freq = int(freq)
if doc_freq >= min_doc_freq and freq >= min_freq and len(key) < max_length:
word = literal_eval(key)
smooth_count = counts.smoother(int(freq))
probs[word] = math.log(smooth_count) - log_total
oov_prob = math.log(counts.smoother(0)) - log_total
return probs, oov_prob
def test_count():
counter = PreshCounter()
assert counter[12] == 0
counter.inc(12, 1)
assert counter[12] == 1
counter.inc(14, 10)
counter.inc(9, 10)
counter.inc(12, 4)
assert counter[12] == 5
assert counter[14] == 10
assert counter[9] == 10
def test_count():
counter = PreshCounter()
assert counter[12] == 0
counter.inc(12, 1)
assert counter[12] == 1
counter.inc(14, 10)
counter.inc(9, 10)
counter.inc(12, 4)
assert counter[12] == 5
assert counter[14] == 10
assert counter[9] == 10
