def preprocess_wiki(input_file, output_file):
# Import input file
if not os.path.exists(input_file):
url = 'https://dumps.wikimedia.org/zhwiki/latest/zhwiki-latest-pages-articles.xml.bz2'
logging.info('Download Wiki dump from {}'.format(url))
wget.download(url)
wiki = WikiCorpus(input_file, lemmatize=False, dictionary=[])
# Convert tradtional Chinese to simplified Chinese using OpenCC
cc = OpenCC('t2s')
# Segment the sentences into words using Jieba paddle mode
jieba.enable_paddle()
# Process Wiki text
logging.info('Start processing Wiki text')
output = open(output_file, 'w')
i = 0
for article in tqdm(wiki.get_texts()):
raw = ' '.join(article)
processed = []
# Remove non-Chinese words
for token in list(jieba.cut(cc.convert(raw))):
matched = re.findall(r'[\u4e00-\u9fff]+', token)
if matched:
processed.append(matched[0])
output.write(' '.join(processed) + '\n')
i += 1
if (i % 10000 == 0):
logging.info('Finished processing {} articles'.format(i))
output.close()
logging.info('Done')
def test_min_token_len_not_set(self):
"""
don't set the parameter token_min_len and check that 'a' as a token doesn't exists
default token_min_len=2
"""
wc = WikiCorpus(datapath(FILENAME), processes=1, lemmatize=False)
self.assertTrue(u'a' not in next(wc.get_texts()))
def test_max_token_len_set(self):
"""
set the parameter token_max_len to 16 and check that 'collectivisation' as a token exists
"""
wc = WikiCorpus(datapath(FILENAME), processes=1, token_max_len=16, lemmatize=False)
l = wc.get_texts()
self.assertTrue(u'collectivization' in next(l))
def test_min_token_len_set(self):
"""
set the parameter token_min_len to 1 and check that 'a' as a token exists
"""
wc = WikiCorpus(datapath(FILENAME), processes=1, token_min_len=1, lemmatize=False)
l = wc.get_texts()
self.assertTrue(u'a' in next(l))
def test_min_token_len_not_set(self):
"""
don't set the parameter token_min_len and check that 'a' as a token doesn't exists
default token_min_len=2
"""
wc = WikiCorpus(datapath(FILENAME), processes=1, lemmatize=False)
self.assertTrue(u'a' not in next(wc.get_texts()))
class WikiSentences:
def __init__(self, wiki_dump_path):
self.wiki = WikiCorpus(wiki_dump_path)
def __iter__(self):
for sentence in self.wiki.get_texts():
yield sentence
def test_max_token_len_not_set(self):
"""
don't set the parameter token_max_len and check that 'collectivisation' as a token doesn't exists
default token_max_len=15
"""
wc = WikiCorpus(datapath(FILENAME), processes=1, lemmatize=False)
l = wc.get_texts()
self.assertTrue(u'collectivization' not in next(l))
def test_max_token_len_not_set(self):
"""
don't set the parameter token_max_len and check that 'collectivisation' as a token doesn't exists
default token_max_len=15
"""
wc = WikiCorpus(datapath(FILENAME), processes=1, lemmatize=False)
l = wc.get_texts()
self.assertTrue(u'collectivization' not in next(l))
def test_max_token_len_set(self):
"""
set the parameter token_max_len to 16 and check that 'collectivisation' as a token exists
"""
wc = WikiCorpus(datapath(FILENAME),
processes=1,
token_max_len=16,
lemmatize=False)
self.assertTrue(u'collectivization' in next(wc.get_texts()))
def test_lower_case_set_false(self):
"""
set the parameter lower to False and check that upper case Anarchism' token exist
"""
wc = WikiCorpus(datapath(FILENAME), processes=1, lower=False, lemmatize=False)
l = wc.get_texts()
list_tokens = next(l)
self.assertTrue(u'Anarchism' in list_tokens)
self.assertTrue(u'anarchism' in list_tokens)
def test_get_texts_returns_generator_of_lists(self):
if sys.version_info < (2, 7, 0):
return
wc = WikiCorpus(datapath(FILENAME))
l = wc.get_texts()
self.assertEqual(type(l), types.GeneratorType)
first = next(l)
self.assertEqual(type(first), list)
self.assertTrue(isinstance(first[0], bytes) or isinstance(first[0], str))
def test_lower_case_set_false(self):
"""
set the parameter lower to False and check that upper case Anarchism' token exist
"""
wc = WikiCorpus(datapath(FILENAME), processes=1, lower=False, lemmatize=False)
row = wc.get_texts()
list_tokens = next(row)
self.assertTrue(u'Anarchism' in list_tokens)
self.assertTrue(u'anarchism' in list_tokens)
def test_lower_case_set_true(self):
"""
set the parameter lower to True and check that upper case 'Anarchism' token doesnt exist
"""
wc = WikiCorpus(datapath(FILENAME), processes=1, lower=True, lemmatize=False)
row = wc.get_texts()
list_tokens = next(row)
self.assertTrue(u'Anarchism' not in list_tokens)
self.assertTrue(u'anarchism' in list_tokens)
def test_min_token_len_set(self):
"""
set the parameter token_min_len to 1 and check that 'a' as a token exists
"""
wc = WikiCorpus(datapath(FILENAME),
processes=1,
token_min_len=1,
lemmatize=False)
self.assertTrue(u'a' in next(wc.get_texts()))
def test_unicode_element(self):
"""
First unicode article in this sample is
1) папа
"""
wc = WikiCorpus(datapath(FILENAME_U), processes=1)
l = wc.get_texts()
self.assertTrue(u'папа' in next(l))
class WikiCorpus:
def __init__(self, wiki_dump_path, lang):
logging.info('Parsing wiki corpus')
self.wiki = WikiCorpus(wiki_dump_path)
self.lang = lang
def __iter__(self):
for sentence in self.wiki.get_texts():
yield list(sentence)
def test_get_texts_returns_generator_of_lists(self):
if sys.version_info < (2, 7, 0):
return
wc = WikiCorpus(datapath(FILENAME))
l = wc.get_texts()
self.assertEqual(type(l), types.GeneratorType)
first = next(l)
self.assertEqual(type(first), list)
self.assertTrue(
isinstance(first[0], bytes) or isinstance(first[0], str))
def test_first_element(self):
"""
First two articles in this sample are
1) anarchism
2) autism
"""
wc = WikiCorpus(datapath(FILENAME), processes=1)
l = wc.get_texts()
self.assertTrue(u'anarchism' in next(l))
self.assertTrue(u'autism' in next(l))
def test_custom_tokenizer(self):
"""
define a custom tokenizer function and use it
"""
wc = WikiCorpus(datapath(FILENAME), processes=1, lemmatize=False, tokenizer_func=custom_tokeiner,
token_max_len=16, token_min_len=1, lower=False)
l = wc.get_texts()
list_tokens = next(l)
self.assertTrue(u'Anarchism' in list_tokens)
self.assertTrue(u'collectivization' in list_tokens)
self.assertTrue(u'a' in list_tokens)
self.assertTrue(u'i.e.' in list_tokens)
def test_custom_tokenizer(self):
"""
define a custom tokenizer function and use it
"""
wc = WikiCorpus(datapath(FILENAME), processes=1, lemmatize=False, tokenizer_func=custom_tokeiner,
token_max_len=16, token_min_len=1, lower=False)
row = wc.get_texts()
list_tokens = next(row)
self.assertTrue(u'Anarchism' in list_tokens)
self.assertTrue(u'collectivization' in list_tokens)
self.assertTrue(u'a' in list_tokens)
self.assertTrue(u'i.e.' in list_tokens)
def test_lower_case_set_true(self):
"""
set the parameter lower to True and check that upper case 'Anarchism' token doesnt exist
"""
wc = WikiCorpus(datapath(FILENAME),
processes=1,
lower=True,
lemmatize=False)
l = wc.get_texts()
list_tokens = next(l)
self.assertTrue(u'Anarchism' not in list_tokens)
self.assertTrue(u'anarchism' in list_tokens)
def test_first_element(self):
"""
First two articles in this sample are
1) anarchism
2) autism
"""
if sys.version_info < (2, 7, 0):
return
wc = WikiCorpus(datapath(FILENAME))
l = wc.get_texts()
self.assertTrue(b"anarchism" in next(l))
self.assertTrue(b"autism" in next(l))
class WikiSentences:
# reference: https://github.com/LasseRegin/gensim-word2vec-model/blob/master/train.py
def __init__(self, wiki_dump_path, lang):
logging.info('Parsing wiki corpus')
self.wiki = WikiCorpus(wiki_dump_path)
self.lang = lang
def __iter__(self):
for sentence in self.wiki.get_texts():
if self.lang == 'zh':
yield list(jieba.cut(''.join(sentence), cut_all=False))
else:
yield list(sentence)
def test_first_element(self):
"""
First two articles in this sample are
1) anarchism
2) autism
"""
if sys.version_info < (2, 7, 0):
return
wc = WikiCorpus(datapath(FILENAME))
l = wc.get_texts()
self.assertTrue(b"anarchism" in next(l))
self.assertTrue(b"autism" in next(l))
def train_and_save_model(articles_path, model_path):
corpus = WikiCorpus(articles_path, lemmatize=False, dictionary={})
sentences = list(corpus.get_texts())
params = {
'size': 200,
'window': 10,
'min_count': 10,
'workers': max(1,
multiprocessing.cpu_count() - 1),
'sample': 1E-3,
}
model = Word2Vec(sentences, **params)
model.save(model_path)
return model
def makeWikiTextEmbedding(self):
# on wiki text sentences
wiki = WikiCorpus('data/swwiki-latest-pages-articles.xml.bz2',
lemmatize=False,
dictionary={})
sentences = list(wiki.get_texts())
print("wikitext: ", len(sentences), " sentences")
self.debugPrintRandomSentences(sentences, 10)
model = gs.models.Word2Vec(sentences,
size=100,
window=5,
min_count=5,
workers=8,
sg=1,
hs=1,
iter=15)
def main():
args = setup_args()
logging.info(args)
fw = open(args.text, 'w')
corpus = WikiCorpus(args.dump,
dictionary={'a'},
tokenizer_func=tokenize_spacy)
for index, sentences in enumerate(corpus.get_texts()):
for sentence in sentences:
fw.write('{}\n'.format(sentence))
if index % 10000 == 0:
logging.info('Done Article: {}'.format(index))
return
def test_get_interesting_proportion():
dictionary = HashDictionary(id_range=100000)
dataset = Dataset(settings.dataset)
terms = dataset.all_lemmas()
wiki = RestrictedWikiCorpus(settings.corpus, terms=terms,
dictionary=dictionary)
num_pages = sum(1 for _ in wiki.get_texts())
unrestricted = WikiCorpus(settings.corpus,
dictionary=dictionary)
num_unrestricted_pages = sum(1 for _ in unrestricted.get_texts())
print "Unrestricted: %d, restricted: %d" % (
num_unrestricted_pages, num_pages)
def Word2VecTraining(Size=200, window=7, min_count=5, Language='spanish'):
#Using WikiCorpus in Spanish Version
wiki = WikiCorpus(
'D:/Gita/GITA_Master/Databases/WikiCorpus/eswiki-latest-pages-articles.xml.bz2',
lemmatize=False,
dictionary={})
corpus = list(wiki.get_texts())
#Defining Paramters
params = {
'size': Size,
'window': window,
'min_count': min_count,
'workers': max(1,
multiprocessing.cpu_count() - 1),
'sample': 1E-3,
}
#Model Training with WikiCorpus
word2vec = Word2Vec(corpus, **params)
################TODO: Save model####################
return word2vec
logger = logging.getLogger("logger")
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s')
logging.root.setLevel(level=logging.INFO)
wiki = WikiCorpus(sys.argv[1], lemmatize=False, dictionary={})
limit = None
if len(sys.argv) > 2:
limit = int(sys.argv[2])
i = 0
sentences_out_stream = open("sentences.txt", "w")
sentences_out_stream = open("sentences.txt", "w")
logger.info("building sentences list")
for text in wiki.get_texts():
while (len(text) > 0):
slice = text[0:9999]
text = text[10000:]
string = " ".join(slice) + "\n"
sentences_out_stream.write(string)
i = i + 1
logger.info("%i articles slices processed", i)
if limit != None and i > limit:
break
sentences_out_stream.close()
logger.info("building vectors from sentences")
sentences_in_stream = open("sentences.txt", "rb")
vectors = Word2Vec(LineSentence(sentences_in_stream),
size=300,
from gensim.corpora.wikicorpus import WikiCorpus
wiki = WikiCorpus('corpus/arwiki-latest-pages-articles.xml.bz2')
from gensim import utils
corpus = [{
'id': 'doc_%i' % num,
'tokens': text
} for num, text in enumerate(wiki.get_texts())]
from PyArabic import ArabicPreprocessor
preprocessor = ArabicPreprocessor()
class LabeledQID(object):
def __init__(self, filename, qid):
self.filename = filename
self.qid = qid
def __iter__(self):
# Loading test set
tree = etree.parse(self.filename)
# {QID, Qtext} dictionary for questions
questions = {}
sentence = tree.xpath('Question[@QID=' + self.qid + ']/Qtext')[0].text
uid = 0
from gensim.corpora.wikicorpus import WikiCorpus
wiki = WikiCorpus('', processes=None, lemmatize=False, dictionary=None)
texts = wiki.get_texts()
with open('wikitext.txt', 'w') as wikitext:
for text in texts:
wikitext.write(' '.join(text) + "\n")
from gensim.corpora.wikicorpus import WikiCorpus
import time
if __name__ == "__main__":
print("Importing Wikipedia")
wiki = WikiCorpus('en/enwiki-latest-pages-articles.xml.bz2',
lemmatize=False,
dictionary={})
print("Getting sentences")
start = time.time()
with open('sentences.txt', 'w') as f:
for i, text in enumerate(wiki.get_texts()):
f.write("%s\n" % text)
i = i + 1
if (i % 10000 == 0):
print('Processed ' + str(i) + ' articles')
end = time.time()
print(end - start, ' seconds elapsed')
def get_text():
wiki = WikiCorpus('ruwiki-20181020-pages-articles-multistream.xml.bz2 ')
for text in wiki.get_texts():
yield [word for word in text]
from gensim.corpora.wikicorpus import WikiCorpus
wiki = WikiCorpus('', processes=None, lemmatize=False, dictionary=None)
texts = wiki.get_texts()
with open('wikitext.txt', 'w') as wikitext:
for text in texts:
wikitext.write(' '.join(text) + "\n")
level=logging.INFO)
import datetime
from gensim.corpora.wikicorpus import WikiCorpus
print('Training: Started at {}'.format(datetime.datetime.now().time()))
wiki = WikiCorpus('K:/ruwiki-20191001-pages-articles-multistream.xml.bz2',
dictionary=False)
print('Training: Corpus loaded at {}'.format(datetime.datetime.now().time()))
from gensim.models.phrases import Phrases, Phraser
bigram = Phrases(wiki.get_texts(), min_count=30, progress_per=10000)
bigram_transformer = Phraser(bigram)
print('Training: Bigrams processed at {}'.format(
datetime.datetime.now().time()))
def text_generator_bigram():
for text in wiki.get_texts():
yield bigram_transformer[[word.decode('utf-8') for word in text]]
trigram = Phrases(text_generator_bigram(), min_count=30, progress_per=10000)
trigram_transformer = Phraser(trigram)
print('Training: Trigrams processed at {}'.format(
import sys
import json
from os import path
from gensim.corpora.wikicorpus import WikiCorpus
base_dir = path.join(path.dirname(path.realpath(__file__)), path.pardir)
wiki_filename = 'simplewiki-20171103-pages-articles-multistream.xml.bz2'
wiki_path = path.join(base_dir, 'corpora', wiki_filename)
outname = path.join(base_dir, 'corpora', 'simplewikiselect')
index = [] # Save information about articles as they've been processed.
wiki = WikiCorpus(wiki_path, dictionary=True) # dict=True avoids making vocab
wiki.metadata = True # Want article titles
print("Loading Wikipedia archive (this may take a few minutes)... ", end="")
articles = list(wiki.get_texts())
print("Done.")
num_articles = len(articles)
print("Total Number of Articles:", num_articles)
MAX_WC = 20_000_000
ARTICLE_MIN_WC = 200
ARTICLE_MAX_WC = 10000
ac = 0
wc = 0
selected = []
with open(outname + ".txt", "w") as f:
from gensim.corpora.wikicorpus import WikiCorpus
from gensim.models.word2vec import Word2Vec
import multiprocessing
import logging
# enable logging
logging.basicConfig(format="%(levelname)s - %(asctime)s: %(message)s",
datefmt='%H:%M:%S',
level=logging.INFO)
# load Wiki dump file
wiki = WikiCorpus('kawiki-latest-pages-articles.xml.bz2',
lemmatize=False,
dictionary={})
sentences = list(wiki.get_texts())
# define training parameters
params = {
'size': 200,
'window': 10,
'min_count': 10,
'workers': max(1,
multiprocessing.cpu_count() - 1),
'sample': 1E-3,
'iter': 5,
'sg': 1,
'hs': 1
}
# train and save word2vec model
word2vec = Word2Vec(sentences, **params)
from gensim.corpora.wikicorpus import WikiCorpus
from gensim.models.phrases import Phrases, Phraser
from gensim.models.word2vec import Word2Vec
wiki = WikiCorpus('/Users/pavel/PycharmProjects/NeuralNetworkWithTenser/src/wordrecognition/res/ruwiki-20190120-pages-articles-multistream1.xml-p4p204179.bz2', dictionary=False)
print('bigram')
bigram = Phrases(wiki.get_texts())
print('bigram_transformer')
bigram_transformer = Phraser(bigram)
def text_generator_bigram():
for text in wiki.get_texts():
yield bigram_transformer[[word for word in text]]
trigram = Phrases(text_generator_bigram())
print('trigram')
trigram_transformer = Phraser(trigram)
print('trigram_transformer')
def text_generator_trigram():
for text in wiki.get_texts():
yield trigram_transformer[bigram_transformer[[word for word in text]]]
print('model create')
model = Word2Vec(size=100, window=7, min_count=10, workers=10, iter=1, min_alpha=0.025)
print('build_vocab')
model.build_vocab(text_generator_trigram())
class WikiAllData():
def __init__(self,
corpus,
wiki_dict,
wordfile,
vocab_size=200000,
window_size=5):
self.w2id_dict = util.load_worddict(wordfile, vocab_size)
self.window_size = window_size
print('Starting loading Wiki Corpus...', end='')
wiki_d = Dictionary.load(wiki_dict)
self.wiki_corpus = WikiCorpus(corpus, dictionary=wiki_d)
print('[done]')
def batch_generator(self, batch_size=128):
QUEUE_END = '__QUEUE_END105834569xx'
def load(q, batch_size):
text_gen = self.wiki_corpus.get_texts()
input_batch = np.zeros(batch_size, dtype=np.int32)
label_batch = np.zeros((batch_size, 1), dtype=np.int32)
counter = 0
#w_vec : list of words in a document
for w_vec in text_gen:
id_vec = [
self.w2id_dict[w] if w in self.w2id_dict else -1
for w in w_vec
]
init_mid = window = random.randint(2, self.window_size)
# sliding window center go through the w_vec
for mid_idx in range(init_mid, len(id_vec) - window - 1):
start_idx = max(0, mid_idx - window)
end_idx = min(mid_idx + window, len(id_vec) - 1)
# go through window
for target_idx in range(start_idx, end_idx + 1):
if target_idx == mid_idx:
continue
if id_vec[target_idx] == -1:
continue
input_batch[counter] = id_vec[mid_idx]
label_batch[counter] = id_vec[target_idx]
if counter == batch_size - 1:
q.put((input_batch, label_batch))
input_batch = np.zeros(batch_size, dtype=np.int32)
label_batch = np.zeros((batch_size, 1),
dtype=np.int32)
counter = 0
else:
counter += 1
window = random.randint(2, self.window_size)
q.put(QUEUE_END)
q = queue.Queue(maxsize=500)
t = threading.Thread(target=load, args=(q, batch_size))
t.daemon = True
t.start()
while True:
q_output = q.get()
if q_output == QUEUE_END:
break
yield q_output
