def _get_window_size_analysis(window_size):
try:
document = __document
log.info("Checking window_size = %s" % window_size)
#check if result already exists
if InformationValueResult.query.find({
"window_size": window_size,
"document_id": document._id
}).count() > 0:
log.warning('Result already found on database')
return
iv_words = __information_value_calculator.information_value(
window_size)
try:
InformationValueResult(window_size=window_size,
document_id=document._id,
iv_words=iv_words)
log.info("Storing results for document %s, window_size %s" %
(document.name, window_size))
odm_session.flush()
except DuplicateKeyError:
log.warning('Result already found')
return (window_size,
WindowAnalysis(window_size,
iv_words,
number_of_words=__number_of_words))
except WindowSizeTooLarge:
return None
def load_moby_dick_analysis():
tokens = get_moby_dick_tokens()
text = gutenberg.raw('melville-moby_dick.txt')
try:
moby_dick_doc = Document(
url='gutenberg',
name='moby dick',
text=text,
month='Jan',
year='1851'
)
odm_session.flush()
except DuplicateKeyError:
moby_dick_doc = Document.query.get(name='moby dick')
for sum_threshold in sum_thresholds:
log.info("Trying analysis for threshold = %s" % sum_threshold)
analysis = get_optimal_window_size(tokens, window_sizes, 20, sum_threshold=sum_threshold)[1]
anal_dict = analysis.encode()
window_size = anal_dict['window_size']
log.debug("Best result = %s" % window_size)
InformationValueResult(
window_size = window_size,
threshold = sum_threshold,
document = moby_dick_doc,
iv_words = anal_dict['top_words'],
max_iv = anal_dict['max_iv'],
sum_iv = anal_dict['sum_iv']
)
odm_session.flush()
def wget(url):
log.info('Trying to fetch url: %s' % url)
response = requests.get(url)
Document(
url=url,
name=url,
text=response.text,
)
odm_session.flush()
def wget(url):
log.info('Trying to fetch url: %s' % url)
response = requests.get(url)
Document(
url=url,
name=url,
text=response.text,
)
odm_session.flush()
def _delete_non_best_analysis(document, window_sizes):
best_res = document.get_information_value_result()
log.info("Best window size for %s: %s" % (document.name, best_res.window_size))
log.info("Removing other results from mongo")
for one_res in document.results:
if one_res.window_size!=best_res.window_size:
log.info("Removing window size %s" % one_res.window_size)
one_res.delete()
log.info("Flushing")
odm_session.flush()
def _delete_non_best_analysis(document, window_sizes):
best_res = document.get_information_value_result()
log.info("Best window size for %s: %s" %
(document.name, best_res.window_size))
log.info("Removing other results from mongo")
for one_res in document.results:
if one_res.window_size != best_res.window_size:
log.info("Removing window size %s" % one_res.window_size)
one_res.delete()
log.info("Flushing")
odm_session.flush()
def process_item(self, item, spider):
Document(
name=item['name'],
url=item['url'],
text=item['text'],
month=item['month'],
year=int(item['year']),
)
try:
odm_session.flush()
except DuplicateKeyError:
log.msg('Duplicate found', level=log.WARNING)
return
log.msg('Document saved', level=log.INFO)
return item
def populate_database():
log.info("Populating database...")
with file('lenin_work.json') as f:
raw_works = json.load(f)
log.info("Inserting %s works to database..." % (len(raw_works)))
for raw_work in raw_works:
try:
Document(url=raw_work['url'],
text=raw_work['text'],
name=raw_work['name'],
month=raw_work['month'],
year=raw_work['year'])
odm_session.flush()
except DuplicateKeyError:
log.info("Duplicate found skipping...")
log.info("Done")
def get_analysis_of_the_mind_document():
with file("tests/analysis_of_the_mind.txt") as f:
document = Document(url='tests/analysis_of_the_mind.txt',
name='Analysis of the mind',
text=f.read(),
month='Dec',
year='1921')
raw_text = f.read()
def tokenizer_wrapper(raw_text):
return map(
str.lower,
tokenize(raw_text, only_alphanum=True, clean_punctuation=True))
document.tokenizer = tokenizer_wrapper
odm_session.flush()
return document
def get_origin_of_species_document():
with file("tests/origin.txt") as f:
document = Document(url='tests/origin.txt',
name='Origin of species',
text=f.read(),
month='Nov',
year='1859')
raw_text = f.read()
def tokenizer_wrapper(raw_text):
return map(
str.lower,
tokenize(raw_text, only_alphanum=True, clean_punctuation=True))
document.tokenizer = tokenizer_wrapper
odm_session.flush()
return document
def populate_database():
log.info("Populating database...")
with file('lenin_work.json') as f:
raw_works = json.load(f)
log.info("Inserting %s works to database..." % (len(raw_works)))
for raw_work in raw_works:
try:
Document(
url=raw_work['url'],
text=raw_work['text'],
name=raw_work['name'],
month=raw_work['month'],
year=raw_work['year']
)
odm_session.flush()
except DuplicateKeyError:
log.info("Duplicate found skipping...")
log.info("Done")
def get_moby_dick_document():
moby_dick = gutenberg.raw('melville-moby_dick.txt')
document = Document(
url = 'melville-moby_dick.txt',
name = 'Moby dick',
text = moby_dick,
month = 'Oct',
year = 1851
)
# document uses tokenizer func for create tokens, since we need to enforce
# only_alphanum and clean_punct we need a wrapper
def tokenizer_wrapper(raw_text):
return map(str.lower, map(str, tokenize(raw_text, only_alphanum=True, clean_punctuation=True)))
document.tokenizer = tokenizer_wrapper
odm_session.flush()
return document
def get_moby_dick_document():
moby_dick = gutenberg.raw('melville-moby_dick.txt')
document = Document(url='melville-moby_dick.txt',
name='Moby dick',
text=moby_dick,
month='Oct',
year=1851)
# document uses tokenizer func for create tokens, since we need to enforce
# only_alphanum and clean_punct we need a wrapper
def tokenizer_wrapper(raw_text):
return map(
str.lower,
map(str,
tokenize(raw_text, only_alphanum=True,
clean_punctuation=True)))
document.tokenizer = tokenizer_wrapper
odm_session.flush()
return document
def reset_senses():
log.info("Resetting all senses")
for doc in Document.query.find():
doc.related_sense = {}
odm_session.flush()
def reset_senses():
log.info("Resetting all senses")
for doc in Document.query.find():
doc.related_sense = {}
odm_session.flush()