def create_users(): session = Session() pgs = session.query(Page).filter(Page.title.like('User:%')) counter = 0 for page in pgs: if '/' in page.title: continue user = User.make_user_from_page(page, session) session.add(user) counter += 1 if counter % 1000 == 0: print(counter) session.commit() session.commit()
def create_users(): session = Session() pgs = session.query(Page).filter(Page.title.like('User:%')) counter = 0 for page in pgs: if '/' in page.title: continue user = User.make_user_from_page(page, session) session.add(user) counter += 1 if counter % 1000 == 0: print(counter) session.commit() session.commit()
def load_xml(): session = Session() Base.metadata.create_all(engine) xml_file = open('../../wiktionary_data/enwiktionary-20120220-pages-meta-current.xml') xml_parser = XMLPageParser(xml_file, enPage) total_lines = 0 counter = 0 for page in xml_parser: ev = session.query(Page).filter(Page.language==page.language, Page.title==page.title) if ev.count() == 0: session.add(page) if counter % 1000 == 0: print(counter) session.commit() print('committed') counter += 1 session.commit()
def load_xml(): session = Session() Base.metadata.create_all(engine) xml_file = open( '../../wiktionary_data/enwiktionary-20120220-pages-meta-current.xml') xml_parser = XMLPageParser(xml_file, enPage) total_lines = 0 counter = 0 for page in xml_parser: ev = session.query(Page).filter(Page.language == page.language, Page.title == page.title) if ev.count() == 0: session.add(page) if counter % 1000 == 0: print(counter) session.commit() print('committed') counter += 1 session.commit()