def build_index(in_dir, out_dict, out_postings):
"""
Build index from documents stored in the input directory,
then output the dictionary file and postings file
"""
print('Indexing...')
stemmer = PorterStemmer()
dictionaries = Dictionaries(out_dict)
postings = Postings(out_postings)
offset = 1
for docID in os.listdir(in_dir):
f = open(f'{in_dir}/{docID}', 'r')
content_tokens = word_tokenize(f.read())
for word in content_tokens:
term = stemmer.stem(word=word).lower()
if dictionaries.has_term(term):
old_offset = dictionaries.get_offset(term)
postings.add_docId_to_offset(old_offset, docID)
else:
dictionaries.add_term(term, offset)
postings.add_doc_id(offset)
postings.add_docId_to_offset(offset, docID)
offset += 1
dictionaries.increment_frequency(term)
postings.save_to_file(dictionaries)
dictionaries.save_to_file()
def build_index(in_dir, out_dict, out_postings):
"""
Build index from documents stored in the input directory,
then output the dictionary file and postings file
"""
print('Indexing...')
stemmer = PorterStemmer()
dictionaries = Dictionaries(out_dict)
postings = Postings(out_postings)
offset = 1
count = len(os.listdir(in_dir))
for docID in os.listdir(in_dir):
f = open(f'{in_dir}/{docID}', 'r')
content = f.read()
sentences = sent_tokenize(content)
doc_terms = []
for sentence in sentences:
for word in word_tokenize(sentence):
term = stemmer.stem(word=word.lower())
doc_terms.append(term)
# Calculate weighted term frequencies for each term
weighted_term_freqs = [(x[0], get_term_frequency_weight(x[1]))
for x in Counter(doc_terms).most_common()]
# Calculate document vector length
doc_length = math.sqrt(
sum(map(lambda x: x[1] * x[1], weighted_term_freqs)))
for term, normalised_tf in weighted_term_freqs:
if dictionaries.has_term(term):
old_offset = dictionaries.get_offset(term)
postings.add_docId_tf_to_offset(old_offset, docID,
normalised_tf / doc_length)
else:
dictionaries.add_term(term, offset)
postings.add_doc_id(offset)
postings.add_docId_tf_to_offset(offset, docID,
normalised_tf / doc_length)
offset += 1
postings.save_to_file(dictionaries, count)
dictionaries.save_to_file()
