def insert_articles(self, to_process_q): self.conn = sqlite3.connect(self.article_database) self.cursor = self.conn.cursor() while not to_process_q.empty(): try: article_object = to_process_q.get() ArgumentValidator.validate(article_object, target_type=Article) self._insert(article_object) except sqlite3.IntegrityError: print('IntegrityError: {}\n'.format(article_object)) except sqlite3.OperationalError: print('OperationalError: {}\n'.format(article_object)) self.conn.commit() self.conn.close()
def __init__(self, keywords, article_database, domain_database): super(Engine, self).__init__() # multiprocessing infrastructure self.manager = Manager() self.domain_q = self.manager.Queue() self.article_q = self.manager.Queue() self.scan_q = self.manager.Queue() self.processed_q = self.manager.Queue() self.keywords = [] ArgumentValidator.validate( article_database, target_type=str, endswith='.db' ) self.article_database = article_database ArgumentValidator.validate( domain_database, target_type=str, endswith='.db' ) self.domain_database = domain_database if keywords: ArgumentValidator.validate(keywords, target_type=list) self.keywords = keywords