def parse(self, response):
self.driver.get(response.url)
self.scroll_down_n_times(1)
questions = self.get_questions()
time.sleep(self.SLEEP_TIME)
# best_answers = self.get_best_answers()
# time.sleep(self.SLEEP_TIME)
dates = self.get_question_dates()
time.sleep(self.SLEEP_TIME)
# assert len(questions) == len(best_answers) == len(dates)
if not self.CATEGORY:
categories = self.get_categories()
time.sleep(self.SLEEP_TIME)
assert len(questions) == len(categories)
for i in range(len(questions)):
date = DbManager.convert_date_string_to_datetime(dates[i])
category = self.CATEGORY if self.CATEGORY else categories[i]
question = { 'content': questions[i], 'upvotes': -1, 'category': category, 'timestamp': date }
DbManager.add_question(question)
# answer = { 'content': best_answers[i], 'upvotes': -1, 'is_best_answer': True, 'timestamp': datetime.now()}
# DbManager.add_answer(answer)
self.driver.close()