class Crawler(object):
waiting_time_on_failure = 10
max_tries = 10
def __init__(self, n):
self.question_factory = QuestionFactory()
self.raw_answers_factory = RawAnswersFactory()
self.n = n
def run(self):
for i in xrange(self.n):
question = self.question_factory.get()
logging.info("Getting question '%s'" % question)
for _ in xrange(self.max_tries):
try:
raw_answers = self.raw_answers_factory.get(question)
question.save()
for ra in raw_answers:
ra.save()
logging.debug("Done")
break
except RequestException as e:
logging.exception("Error building '%s'. " % question+\
"Retrying in %d seconds." % self.waiting_time_on_failure)
time.sleep(self.waiting_time_on_failure)
except Exception as e:
logging.exception("Unable to crawl question '%s'!"
% question)
def __init__(self, n):
self.question_factory = QuestionFactory()
self.raw_answers_factory = RawAnswersFactory()
self.n = n
