def insert_articles(self, to_process_q):
self.conn = sqlite3.connect(self.article_database)
self.cursor = self.conn.cursor()
while not to_process_q.empty():
try:
article_object = to_process_q.get()
ArgumentValidator.validate(article_object, target_type=Article)
self._insert(article_object)
except sqlite3.IntegrityError:
print('IntegrityError: {}\n'.format(article_object))
except sqlite3.OperationalError:
print('OperationalError: {}\n'.format(article_object))
self.conn.commit()
self.conn.close()
def __init__(self, keywords, article_database, domain_database):
super(Engine, self).__init__()
# multiprocessing infrastructure
self.manager = Manager()
self.domain_q = self.manager.Queue()
self.article_q = self.manager.Queue()
self.scan_q = self.manager.Queue()
self.processed_q = self.manager.Queue()
self.keywords = []
ArgumentValidator.validate(
article_database, target_type=str, endswith='.db'
)
self.article_database = article_database
ArgumentValidator.validate(
domain_database, target_type=str, endswith='.db'
)
self.domain_database = domain_database
if keywords:
ArgumentValidator.validate(keywords, target_type=list)
self.keywords = keywords
