def __init__(self, pass_num):

self.pass_num = pass_num

self.num_articles = 0

self.num_links = 0

def startDump(self):

#delete previous data

if self.pass_num == 0:

#models.clear()

pass

elif self.pass_num == 1:

pass

def endDump(self):

models.sync()

def handleArticle(self, current_page):

#print('ARTICLE:', repr(current_page.title))

if self.pass_num == 0:

models.articles.save(models.Article(id=current_page['id'], title=current_page['title'], revision_id=current_page['revision_id']))

elif self.pass_num == 1:

aid = models.articles.resolve_title(current_page['title'])

wikitext__ = utils.unescape_html(current_page['text'])

redirect = wikitext.parse_redirect(wikitext__)

if redirect:

redirect_dest_title, dest_frag, redirect_label = redirect

article = models.articles[aid]

if redirect_dest_title in models.articles:

dest_id = models.articles.resolve_title(redirect_dest_title)

else:

dest_id = None

#print('Broken REDIRECT FROM ', repr(current_page['title']), ' TO ', repr(redirect), file=parse_log)

#print(repr(redirect_dest_title), file=parse_log)

print >>parse_log, repr(redirect_dest_title)

article.redirect = models.Redirect(dest_id, dest_frag, redirect_label)

models.articles.save(article)

else:

#do not parse/allow links if there is a redirect. If there is a redirect, links are meaningless. besides, parse_links does not work properly with redirects.

for link_dest_title, dest_frag, link_label, snippet in wikitext.parse_links(wikitext__):

lid = self.num_links + 1 #0 is an invalid key

#assert lid not in models.links

if link_dest_title in models.articles:

dest_id = models.articles.resolve_title(link_dest_title)

else:

dest_id = None

#print('Broken LINK FROM ', repr(current_page['title']), ' TO ', repr((link_dest_title, dest_frag, link_label)), file=parse_log)

#print(repr(link_dest_title), file=parse_log)

print >>parse_log, repr(link_dest_title)

models.links.save(models.Link(src_id=aid, dest_id=dest_id, dest_fragment=dest_frag,

label=link_label, snippet=snippet, id=lid))

self.num_links += 1

'''

if lid > len(models.links):

...

else:

self.num_links += 1 #skip

'''

self.num_articles += 1

if self.num_articles % 1000 == 0:

#print(self.num_articles, self.num_links)

#A finite state machine

def __init__(self, dump_handler):

self.open_tags = []

self.char_buffer = []

self.current_page = None

self.dump_handler = dump_handler

def startDocument(self):

assert self.open_tags == []

self.dump_handler.startDump()

def endDocument(self):

assert self.open_tags == []

self.dump_handler.endDump()

def startElement(self, name, attrs):

self.open_tags.append(name)

self.char_buffer = []

if name == 'page':

assert self.current_page is None

self.current_page = {}

def endElement(self, name):

assert self.open_tags[-1] == name

self.open_tags.pop()

chars = ''.join(self.char_buffer)

if name == 'id':

if self.open_tags[-1] == 'page':

self.current_page['id'] = int(chars)

elif self.open_tags[-1] == 'revision':

self.current_page['revision_id'] = int(chars)

elif name == 'title':

self.current_page['title'] = chars

elif name == 'text':

self.current_page['text'] = chars

elif name == 'page':

self.dump_handler.handleArticle(self.current_page)

self.current_page = None

def characters(self, content):

aaa = article

redirects_traversed = 0

while article.redirect and redirects_traversed < 6:

if article.redirect.dest_id is not None:

article = models.articles[article.redirect.dest_id]

else:

return None #article redirects to non-existent article, so it itself is None

redirects_traversed += 1

if not redirects_traversed < 6:

print>>parse_log, 'Cycle in redirects detected', repr(aaa)

adj_list = {}

num_articles = 0

for article in models.articles:

article = resolve_redirects(article)

if article: #'article' does not redirect to non-existent article

for dest_id, lid in models.links.query(src_id=article.id):

if dest_id is not None:

dest = resolve_redirects(models.articles[dest_id])

if dest is None: #broken redirect

print>>parse_log, 'Broken link redirect:', 'title:', repr(article.title), 'label:', repr(models.links[lid].label), 'link.dest_id:', repr(models.links[lid].dest_id), 'target title:', repr(models.articles[models.links[lid].dest_id].title)

print>>parse_log, repr(models.articles[dest_id].title)

else:

if article.id not in adj_list:

adj_list[article.id] = []

adj_list[article.id].append(dest.id)

num_articles += 1

if num_articles % 5000 == 0:

print num_articles

for k, v in adj_list.iteritems():

#print('begin do_parse_dump')

import datetime

#print(datetime.datetime.now())

import bz2

dump_file = bz2.BZ2File(dump_filename, 'rb')

parse_dump(dump_file, 0)

dump_file.close()

#print('finished first pass parsing')

#print(datetime.datetime.now())

dump_file = bz2.BZ2File(dump_filename, 'rb')

parse_dump(dump_file, 1)

dump_file.close()

#print('finished second pass parsing')

#print(datetime.datetime.now())

#print('done do_parse_dump')

print('begin do_build_graph')

print(datetime.datetime.now())

build_graph()

print('done do_build_graph')

import os.path

current_dir = os.path.dirname(os.path.abspath(__file__))

#dump_filename = 'Z:\\wikiminer\\enwiki-20080724-pages-articles.xml.bz2'

dump_filename = 'enwiki-20080724-pages-articles.xml.bz2'

dump_filename = os.path.join(current_dir, dump_filename)

do_parse_dump(dump_filename)