def compute_lda_model(input_dir,
output_file,
n_topics=500,
extension="xml",
language="en",
normalization="stemming"):
"""Compute a LDA model from a collection of documents. Latent Dirichlet
Allocation is computed using sklearn module.
Args:
input_dir (str): the input directory.
output_file (str): the output file.
n_topics (int): number of topics for the LDA model, defaults to 500.
extension (str): file extension for input documents, defaults to xml.
language (str): language of the input documents, used for stop_words
in sklearn CountVectorizer, defaults to 'en'.
normalization (str): word normalization method, defaults to 'stemming'.
Other possible values are 'lemmatization' or 'None' for using word
surface forms instead of stems/lemmas.
"""
# texts container
texts = []
# loop throught the documents
for input_file in glob.iglob(input_dir + '/*.' + extension):
logging.info('reading file {}'.format(input_file))
# initialize load file object
doc = LoadFile()
# read the input file
doc.load_document(input=input_file,
language=language,
normalization=normalization)
# container for current document
text = []
# loop through sentences
for sentence in doc.sentences:
# get the tokens (stems) from the sentence if they are not
# punctuation marks
text.extend([
sentence.stems[i] for i in range(sentence.length)
if sentence.pos[i] != 'PUNCT' and sentence.pos[i].isalpha()
])
# add the document to the texts container
texts.append(' '.join(text))
# vectorize dataset
# get the stoplist from nltk because CountVectorizer only contains english
# stopwords atm
tf_vectorizer = CountVectorizer(
stop_words=stopwords.words(ISO_to_language[language]))
tf = tf_vectorizer.fit_transform(texts)
# extract vocabulary
vocabulary = tf_vectorizer.get_feature_names()
# create LDA model and train
lda_model = LatentDirichletAllocation(n_components=n_topics,
random_state=0,
learning_method='batch')
lda_model.fit(tf)
# save all data necessary for later prediction
saved_model = (vocabulary, lda_model.components_,
lda_model.exp_dirichlet_component_,
lda_model.doc_topic_prior_)
# Dump the df container
logging.info('writing LDA model to {}'.format(output_file))
# create directories from path if not exists
if os.path.dirname(output_file):
os.makedirs(os.path.dirname(output_file), exist_ok=True)
# dump the LDA model
with gzip.open(output_file, 'wb') as fp:
pickle.dump(saved_model, fp)
def compute_document_frequency(input_dir,
output_file,
extension='xml',
language='en',
normalization="stemming",
stoplist=None,
delimiter='\t',
n=3):
"""Compute the n-gram document frequencies from a set of input documents. An
extra row is added to the output file for specifying the number of
documents from which the document frequencies were computed
(--NB_DOC-- tab XXX). The output file is compressed using gzip.
Args:
input_dir (str): the input directory.
output_file (str): the output file.
extension (str): file extension for input documents, defaults to xml.
language (str): language of the input documents (used for computing the
n-stem or n-lemma forms), defaults to 'en' (english).
normalization (str): word normalization method, defaults to 'stemming'.
Other possible values are 'lemmatization' or 'None' for using word
surface forms instead of stems/lemmas.
stoplist (list): the stop words for filtering n-grams, default to None.
delimiter (str): the delimiter between n-grams and document frequencies,
defaults to tabulation (\t).
n (int): the size of the n-grams, defaults to 3.
"""
# document frequency container
frequencies = defaultdict(set)
# initialize number of documents
nb_documents = 0
# loop through the documents
for input_file in glob.glob(input_dir + '/*.' + extension):
logging.info('reading file ' + input_file)
# initialize load file object
doc = LoadFile()
# read the input file
doc.load_document(input=input_file,
language=language,
normalization=normalization)
# candidate selection
doc.ngram_selection(n=n)
# filter candidates containing punctuation marks
doc.candidate_filtering(stoplist=stoplist)
# loop through candidates
for lexical_form in doc.candidates:
frequencies[lexical_form].add(input_file)
nb_documents += 1
# create directories from path if not exists
if os.path.dirname(output_file):
os.makedirs(os.path.dirname(output_file), exist_ok=True)
# dump the df container
with gzip.open(output_file, 'wb') as f:
# add the number of documents as special token
first_line = '--NB_DOC--' + delimiter + str(nb_documents)
f.write(first_line.encode('utf-8') + b'\n')
for ngram in frequencies:
line = ngram + delimiter + str(len(frequencies[ngram]))
f.write(line.encode('utf-8') + b'\n')
def compute_lda_model(input_dir,
output_file,
n_topics=500,
format="corenlp",
extension="xml",
use_lemmas=False,
stemmer="porter",
language="english"):
""" Compute a LDA model from a collection of documents. Latent Dirichlet
Allocation is computed using sklearn module.
Args:
input_dir (str): the input directory.
output_file (str): the output file.
n_topics (int): number of topics for the LDA model, defaults to 500.
format (str): the input files format, defaults to corenlp.
extension (str): file extension for input documents, defaults to
xml.
use_lemmas (bool): whether lemmas from stanford corenlp are used
instead of stems (computed by nltk), defaults to False.
stemmer (str): the stemmer in nltk to used (if used), defaults
to porter.
language (str): the language of the documents, used for stop_words
in sklearn CountVectorizer, defaults to 'english'.
"""
# texts container
texts = []
# loop throught the documents
for input_file in glob.glob(input_dir + '/*.' + extension):
# initialize load file object
doc = LoadFile(input_file)
# read the input file
doc.read_document(format=format,
use_lemmas=use_lemmas,
stemmer=stemmer,
sep='/')
# container for current document
text = []
# loop through sentences
for sentence in doc.sentences:
# get the tokens (stems) from the sentence if they are not
# punctuation marks
text.extend([ sentence.stems[i] for i in range(sentence.length) \
if not re.search('[^A-Z$]', sentence.pos[i]) ])
# add the document to the texts container
texts.append(' '.join(text))
# vectorize dataset
# get the stoplist from nltk because CountVectorizer only contains english
# stopwords atm
tf_vectorizer = CountVectorizer(stop_words=stopwords.words(language))
tf = tf_vectorizer.fit_transform(texts)
# extract vocabulary
vocabulary = tf_vectorizer.get_feature_names()
# create LDA model and train
lda_model = LatentDirichletAllocation(n_components=n_topics,
random_state=0,
learning_method='batch')
lda_model.fit(tf)
# save all data necessary for later prediction
saved_model = (vocabulary, lda_model.components_,
lda_model.exp_dirichlet_component_,
lda_model.doc_topic_prior_)
# Dump the df container
logging.info('writing LDA model to ' + output_file)
with gzip.open(output_file, 'wb') as fp:
pickle.dump(saved_model, fp)
def compute_document_frequency(input_dir,
output_file,
format="corenlp",
extension="xml",
use_lemmas=False,
stemmer="porter",
stoplist=None,
delimiter='\t',
n=3):
""" Compute n-gram document frequencies from a set of input documents. An
extra row is added to the output file for specifying the number of
documents from which the frequencies were computed (--NB_DOC-- tab XX).
Args:
input_dir (str): the input directory.
output_file (str): the output file.
format (str): the input files format, defaults to corenlp.
extension (str): file extension for input documents, defaults to
xml.
use_lemmas (bool): whether lemmas from stanford corenlp are used
instead of stems (computed by nltk), defaults to False.
stemmer (str): the stemmer in nltk to used (if used), defaults
to porter.
stoplist (list): the stop words for filtering n-grams, default to
None.
delimiter (str): the delimiter between n-grams and document
frequencies, default to tabulation.
n (int): the length for ngrams, defaults to 3.
"""
# document frequency container
frequencies = defaultdict(set)
# initialize number of documents
nb_documents = 0
# loop throught the documents
for input_file in glob.glob(input_dir + '/*.' + extension):
logging.info('reading file ' + input_file)
# initialize load file object
doc = LoadFile(input_file)
# read the input file
doc.read_document(format=format,
use_lemmas=use_lemmas,
stemmer=stemmer,
sep='/')
# candidate selection
doc.ngram_selection(n=n)
# filter candidates containing punctuation marks
doc.candidate_filtering(stoplist=stoplist)
# loop through candidates
for lexical_form in doc.candidates:
frequencies[lexical_form].add(input_file)
nb_documents += 1
# Dump the df container
with gzip.open(output_file, 'wb') as f:
# add the number of documents as special token
first_line = '--NB_DOC--' + delimiter + str(nb_documents)
f.write(first_line.encode('utf-8') + b'\n')
for ngram in frequencies:
line = ngram + delimiter + str(len(frequencies[ngram]))
f.write(line.encode('utf-8') + b'\n')
def compute_document_frequency(input_dir,
output_file,
extension='xml',
language='en',
normalization="stemming",
stoplist=None,
delimiter='\t',
n=3,
max_length=10**6,
encoding=None):
"""Compute the n-gram document frequencies from a set of input documents. An
extra row is added to the output file for specifying the number of
documents from which the document frequencies were computed
(--NB_DOC-- tab XXX). The output file is compressed using gzip.
Args:
input_dir (str): the input directory.
output_file (str): the output file.
extension (str): file extension for input documents, defaults to xml.
language (str): language of the input documents (used for computing the
n-stem or n-lemma forms), defaults to 'en' (english).
normalization (str): word normalization method, defaults to 'stemming'.
Other possible values are 'lemmatization' or 'None' for using word
surface forms instead of stems/lemmas.
stoplist (list): the stop words for filtering n-grams, default to None.
delimiter (str): the delimiter between n-grams and document frequencies,
defaults to tabulation (\t).
n (int): the size of the n-grams, defaults to 3.
encoding (str): encoding of files in input_dir, default to None.
"""
# document frequency container
frequencies = defaultdict(int)
# initialize number of documents
nb_documents = 0
count=0
# loop through the documents
for input_file in glob.iglob(input_dir + os.sep + '*.' + extension):
print(nb_documents)
# if count>10:
# break
#logging.info('reading file {}'.format(input_file))
try:
# initialize load file object
doc = LoadFile()
# read the input file
doc.load_document(input=input_file,
language=language,
normalization=normalization,
max_length=max_length,
encoding=encoding, spacy_model=spacy_pipelines)
# candidate selection
doc.ngram_selection(n=n)
# filter candidates containing punctuation marks
doc.candidate_filtering(stoplist=stoplist)
# loop through candidates
for lexical_form in doc.candidates:
frequencies[lexical_form] += 1
nb_documents += 1
if nb_documents % 1000 == 0:
logging.info("{} docs, memory used: {} mb".format(nb_documents,
sys.getsizeof(
frequencies)
/ 1024 / 1024 ))
except:
print("ERR")
# create directories from path if not exists
if os.path.dirname(output_file):
os.makedirs(os.path.dirname(output_file), exist_ok=True)
# dump the df container
with gzip.open(output_file, 'wt', encoding='utf-8') as f:
# add the number of documents as special token
first_line = '--NB_DOC--' + delimiter + str(nb_documents)
f.write(first_line + '\n')
for ngram in frequencies:
line = ngram + delimiter + str(frequencies[ngram])
f.write(line + '\n')