#porno_vocabulary = [ l.strip() for l in open(BAD_WORD_FILE, "r") ]
#G.restrict_vocabulary("w1 w2 w3", porno_vocabulary, invert=True)
# draw a subsample
#if SUBSAMPLE_N is not None:
#G.subsample_lines(N=SUBSAMPLE_N)
# we need to resort this so that we can have w1 and w3 equal and then all the n-grams matched
G.sort("w1 w3 unigram bigram trigram", lines=1000000)
G.head()
item_number = 0
line_stack = []
for l in G.lines(tmp=False, parts=False):
# extract the columns from line
w1, w3, unigram, bigram, trigram = G.extract_columns(l, keys="w1 w3 unigram bigram trigram", dtype=[str, str, float, float, float])
# now remove things which cannot possibly match anymore
while len(line_stack) > 0:
w1_, w3_, unigram_, bigram_, trigram = G.extract_columns(line_stack[0], keys="w1 w3 unigram bigram trigram", dtype=[str, str, float, float, float])
if not (w1_ == w1 and w3_ == w3 and check_tolerance(unigram, unigram_)):
del line_stack[0]
# now go through the line_stack and try out each
# it must already be within tolerance on unigram, or it would have been removed
for x in line_stack:
w1_, w3_, unigram_, bigram_, trigram = G.extract_columns(x, keys="w1 w3 unigram bigram trigram", dtype=[str, str, float, float, float])
# it must have already been within tolerance on unigram or it would be removed
assert( check_tolerance(unigram, unigram_) )
porno_vocabulary = [l.strip() for l in open(BAD_WORD_FILE, "r")]
G.restrict_vocabulary("w1 w2 w3", porno_vocabulary, invert=True)
# and then subsample
G.subsample_lines(N=SUBSAMPLE_N)
# and make sure we are sorted for the below
G.sort("unigram bigram trigram", dtype=float)
G.head() # just a peek
item_number = 0
line_stack = []
for l in G.lines(tmp=False, parts=False):
# extrac the columns from line
unigram, bigram, trigram = G.extract_columns(l,
keys="unigram bigram trigram",
dtype=float)
# now remove things which cannot possibly match anymore
while len(line_stack) > 0 and not check_tolerance(
unigram,
G.extract_columns(line_stack[0], keys="unigram", dtype=float)[0]):
del line_stack[0]
# now go through the line_stack and try out each
# it must already be within tolerance on unigram, or it would have been removed
for x in line_stack:
#print "Checking ", x
x_unigram, x_bigram, x_trigram = G.extract_columns(
x, keys="unigram bigram trigram", dtype=float)
# Now throw out the porno words
porno_vocabulary = [ l.strip() for l in open(BAD_WORD_FILE, "r") ]
G.restrict_vocabulary("w1 w2 w3", porno_vocabulary, invert=True)
# and then subsample
G.subsample_lines(N=SUBSAMPLE_N)
# and make sure we are sorted for the below
G.sort("unigram bigram trigram", dtype=float)
G.head() # just a peek
item_number = 0
line_stack = []
for l in G.lines(tmp=False, parts=False):
# extrac the columns from line
unigram, bigram, trigram = G.extract_columns(l, keys="unigram bigram trigram", dtype=float)
# now remove things which cannot possibly match anymore
while len(line_stack) > 0 and not check_tolerance(unigram, G.extract_columns(line_stack[0], keys="unigram", dtype=float)[0]):
del line_stack[0]
# now go through the line_stack and try out each
# it must already be within tolerance on unigram, or it would have been removed
for x in line_stack:
#print "Checking ", x
x_unigram, x_bigram, x_trigram = G.extract_columns(x, keys="unigram bigram trigram", dtype=float)
# it must have already been within tolerance on unigram or it would be removed
assert( check_tolerance(unigram, x_unigram) )
# and check the bigrams
