def table_parser(self, file_name, root): db = MySQLDatabase(DATABASE_HOST, DATABASE_USER, DATABASE_PASSWORD, DATABASE_NAME) db_build_view = db.get_build_view() cursor = db_build_view._cursor # setup logging LOGGING_FORMAT = '%(levelname)s:\t%(asctime)-15s %(message)s' LOGGING_PATH = 'tmp/tableclasses-dbinsert.log' logging.basicConfig(filename=LOGGING_PATH, level=logging.DEBUG, format=LOGGING_FORMAT, filemode='w') html_parser = WikipediaHTMLTableParser() zip_file_path = os.path.join(root, file_name) html = self.zip2html(zip_file_path) html_parser.feed(html.decode('utf-8')) source_article_id = file_name.split('_')[1] try: fed_parser = WikipediaFedTextParser(html_parser.get_data()) table_classes = fed_parser.table_classes(None) table_classes = list(set(table_classes)) for table_class in table_classes: self.insert_table_class(source_article_id, table_class, cursor) except KeyError: db_build_view._db_connection.rollback() logging.error('KeyError FedTextParser source article id: %s ' % source_article_id) db_build_view.commit() db_build_view.reset_cache()
def table_parser(self, file_name, root): db = MySQLDatabase(DATABASE_HOST, DATABASE_USER, DATABASE_PASSWORD, DATABASE_NAME) db_build_view = db.get_build_view() cursor = db_build_view._cursor # setup logging LOGGING_FORMAT = '%(levelname)s:\t%(asctime)-15s %(message)s' LOGGING_PATH = 'tmp/tableclasses-dbinsert.log' logging.basicConfig(filename=LOGGING_PATH, level=logging.DEBUG, format=LOGGING_FORMAT, filemode='w') html_parser = WikipediaHTMLTableParser() zip_file_path = os.path.join(root, file_name) html = self.zip2html(zip_file_path) html_parser.feed(html.decode('utf-8')) source_article_id = file_name.split('_')[1] try: fed_parser = WikipediaFedTextParser(html_parser.get_data()) table_classes = fed_parser.table_classes(None) table_classes = list(set(table_classes)) for table_class in table_classes: self.insert_table_class(source_article_id, table_class, cursor) except KeyError: db_build_view._db_connection.rollback() logging.error('KeyError FedTextParser source article id: %s ' % source_article_id) db_build_view.commit() db_build_view.reset_cache()
def __init__(self, path): #os.environ["DISPLAY"]=":1" print path os.environ["DISPLAY"]=":1" db = MySQLDatabase(DATABASE_HOST, DATABASE_USER, DATABASE_PASSWORD, DATABASE_NAME) self.db_build_view = db.get_build_view() self.cursor = self.db_build_view._cursor self.app = QApplication(sys.argv) self.path = path
def __init__(self, path): #os.environ["DISPLAY"]=":1" print path os.environ["DISPLAY"] = ":1" db = MySQLDatabase(DATABASE_HOST, DATABASE_USER, DATABASE_PASSWORD, DATABASE_NAME) self.db_build_view = db.get_build_view() self.cursor = self.db_build_view._cursor self.app = QApplication(sys.argv) self.path = path
def _extract_articles(self): INPUT_FILE = WIKI_DUMP_XML_FILE #self.read_path('Please enter the path of the wiki dump file [.xml]') #INPUT_FILE = "/home/ddimitrov/wikiwsd/data/training.xml"#self.read_path('Please enter the path of the wiki dump file [.xml]') MAX_ARTICLES_IN_QUEUE = 200 #self.read_number('How many articles should be kept in the memory at any time at most?', 200, 20, 1000) NUM_THREADS = 1 #self.read_number('How many threads shall be used to write to the database?', 20, 1, 50) CONTINUE = True #self.read_yes_no('This process might take several days to finish.\nDo you want to continue?') if CONTINUE: # measure time start = time.clock() # connect to database and create article queue db = MySQLDatabase(DATABASE_HOST, DATABASE_USER, DATABASE_PASSWORD, DATABASE_NAME) queue = Queue.Queue(maxsize=MAX_ARTICLES_IN_QUEUE) # create reader and threads reader = WikipediaReader(INPUT_FILE, queue, extract_text=False) threads = [] for i in range(0, NUM_THREADS): inserter = ArticleInserter(queue, db.get_build_view()) threads.append(inserter) # start reader reader.start() # start insert threads for thread in threads: thread.start() # wait for reading thread, queue and inserters to be done reader.join() queue.join() for thread in threads: thread.end() for thread in threads: thread.join() seconds = round(time.clock() - start) print 'Finished after %02d:%02d minutes' % (seconds / 60, seconds % 60) else: print 'Aborting...'
def _extract_articles(self): INPUT_FILE = WIKI_DUMP_XML_FILE #self.read_path('Please enter the path of the wiki dump file [.xml]') #INPUT_FILE = "/home/ddimitrov/wikiwsd/data/training.xml"#self.read_path('Please enter the path of the wiki dump file [.xml]') MAX_ARTICLES_IN_QUEUE = 200#self.read_number('How many articles should be kept in the memory at any time at most?', 200, 20, 1000) NUM_THREADS = 1#self.read_number('How many threads shall be used to write to the database?', 20, 1, 50) CONTINUE = True#self.read_yes_no('This process might take several days to finish.\nDo you want to continue?') if CONTINUE: # measure time start = time.clock() # connect to database and create article queue db = MySQLDatabase(DATABASE_HOST, DATABASE_USER, DATABASE_PASSWORD, DATABASE_NAME) queue = Queue.Queue(maxsize=MAX_ARTICLES_IN_QUEUE) # create reader and threads reader = WikipediaReader(INPUT_FILE, queue, extract_text=False) threads = [] for i in range(0, NUM_THREADS): inserter = ArticleInserter(queue, db.get_build_view()) threads.append(inserter) # start reader reader.start() # start insert threads for thread in threads: thread.start() # wait for reading thread, queue and inserters to be done reader.join() queue.join() for thread in threads: thread.end() for thread in threads: thread.join() seconds = round (time.clock() - start) print 'Finished after %02d:%02d minutes' % (seconds / 60, seconds % 60) else: print 'Aborting...'