def fetch_articles(self):
handler_args = {
"asyn_work": self.detail_asyn,
"callback": self.detail_callback
}
gr_group = HandlerManager(task_queue=self.page_success,
concurrency=4,
success_queue=self.detail_success,
handler_args=handler_args)
gr_group.run()
def fetch_details(self):
# 生成所有列表页的链接
for i in xrange(self.start, self.end + 1):
self.page_task.put("%s%s" % (self.page_base, i))
# 构建协程参数，并运行
handler_args = {
"asyn_work": self.page_asyn,
"callback": self.page_callback
}
gr_group = HandlerManager(task_queue=self.page_task,
concurrency=3,
success_queue=self.page_success,
handler_args=handler_args)
gr_group.run()
def qualify(submission):
"""
Check if a submission qualifies to be previewed by the bot.
Conditions for preview:
(1) Submission is a link
(2) Submission has a Handler
"""
logging.info("Qualifying submission id: {}".format(submission.id))
# Check (1) Submission is a link
is_link = not submission.is_self
# Check (2) Submission has a Handler
has_handler = HandlerManager.has_handler(submission.url)
logging.debug("is_link = {}, has_handler = {}".format(is_link, has_handler))
return is_link and has_handler
def qualify(submission):
"""
Check if a submission qualifies to be previewed by the bot.
Conditions for preview:
(1) Submission is a link
(2) Submission has not been previously encountered
(3) Submission has a Handler
"""
# Check (1) Submission is a link
is_link = not submission.is_self
# Check (2) Submission is new
is_new = not DatabaseManager.check_id(submission.id)
# Check (3) Submission has a Handler
has_handler = HandlerManager.has_handler(submission.url)
return is_link and is_new and has_handler
def scan(subreddit):
"""Scan a Subreddit for new submissions."""
print("Starting scan")
for submission in subreddit.new(limit=config.LIMIT):
print("Operating on submission ID: " + submission.id)
does_qualify = qualify(submission)
if does_qualify:
print("Submission qualifies")
handler = HandlerManager.get_handler(submission.url)
comment_raw = None
try:
comment_raw = handler.handle(submission.url)
except Exception as e:
print(f"Exception occurred while handling {submission.url}")
traceback.print_exc()
if comment_raw is None:
skip(submission)
return
comment_markdown = format_comment(comment_raw)
if len(comment_markdown) < config.COMMENT_LENGTH_LIMIT:
try:
print("Attempting to post a comment")
submission.reply(comment_markdown)
print("Comment posting succeeded")
print("Attempting to write success to database")
DatabaseManager.write_id(submission.id, DatabaseActionEnum.SUCCESS)
print("Database write succeeded")
except Exception as e:
print("An error occurred:")
print(e)
else:
print("Submission is too long to be posted.")
print("Attempting to write skip to database")
DatabaseManager.write_id(submission.id, DatabaseActionEnum.SKIP)
print("Database write succeeded")
else:
skip(submission)
def scan(subreddit):
"""Scan a Subreddit for new submissions."""
logging.info("Starting scan")
for submission in subreddit.stream.submissions(skip_existing=True):
logging.info("Operating on submission")
logging.debug("Submission ID = %s, title = %s",
submission.id,
submission.title)
does_qualify = qualify(submission)
if not does_qualify:
logging.info("Comment does not qualify; skipping comment")
continue
logging.info("Submission qualifies")
handler = None
try:
logging.info("Attempting to get article handler")
handler = HandlerManager.get_handler(submission.url)
logging.info("Article handler = %s",
handler)
except Exception as exception:
logging.error("""
An error occurred while getting article handler. This should never happen.
""")
logging.error("Exception = %s", exception)
logging.info("Skipping current submission")
continue
comment_raw = None
try:
logging.info("Attempting to generate raw comment using handler")
comment_raw = handler.handle(submission.url)
logging.info("Raw comment generated")
except Exception as exception:
logging.error("An error occurred while handling URL = %s",
submission.url)
logging.error("Exception = %s", exception)
logging.info("Skipping current submission")
continue
if comment_raw is None:
logging.error("comment_raw is None; skipping current submission")
continue
logging.info("Generating formatted comment")
comment_markdown = format_comment(comment_raw)
logging.info("Formatted comment generated")
if len(comment_markdown) < config.COMMENT_LENGTH_LIMIT:
try:
logging.info("Attempting to post comment")
submission.reply(comment_markdown)
logging.info("Comment posting succeeded")
except Exception as exception:
logging.error("An error occurred while posting comment")
logging.error("Exception = %s", exception)
else:
logging.warning("Submission is too long to be posted")
def on_close(self):
self.r.decr(self.node)
HandlerManager.del_handler(self)
def on_message(self, msg):
data = HandlerManager.route_message(self, msg)
if data != '{}':
self.write_message(data)
def open(self):
self.r.incr(self.node)
HandlerManager.add_handler(self)