def fetch_articles(self): handler_args = { "asyn_work": self.detail_asyn, "callback": self.detail_callback } gr_group = HandlerManager(task_queue=self.page_success, concurrency=4, success_queue=self.detail_success, handler_args=handler_args) gr_group.run()
def fetch_details(self): # 生成所有列表页的链接 for i in xrange(self.start, self.end + 1): self.page_task.put("%s%s" % (self.page_base, i)) # 构建协程参数,并运行 handler_args = { "asyn_work": self.page_asyn, "callback": self.page_callback } gr_group = HandlerManager(task_queue=self.page_task, concurrency=3, success_queue=self.page_success, handler_args=handler_args) gr_group.run()
def qualify(submission): """ Check if a submission qualifies to be previewed by the bot. Conditions for preview: (1) Submission is a link (2) Submission has a Handler """ logging.info("Qualifying submission id: {}".format(submission.id)) # Check (1) Submission is a link is_link = not submission.is_self # Check (2) Submission has a Handler has_handler = HandlerManager.has_handler(submission.url) logging.debug("is_link = {}, has_handler = {}".format(is_link, has_handler)) return is_link and has_handler
def qualify(submission): """ Check if a submission qualifies to be previewed by the bot. Conditions for preview: (1) Submission is a link (2) Submission has not been previously encountered (3) Submission has a Handler """ # Check (1) Submission is a link is_link = not submission.is_self # Check (2) Submission is new is_new = not DatabaseManager.check_id(submission.id) # Check (3) Submission has a Handler has_handler = HandlerManager.has_handler(submission.url) return is_link and is_new and has_handler
def scan(subreddit): """Scan a Subreddit for new submissions.""" print("Starting scan") for submission in subreddit.new(limit=config.LIMIT): print("Operating on submission ID: " + submission.id) does_qualify = qualify(submission) if does_qualify: print("Submission qualifies") handler = HandlerManager.get_handler(submission.url) comment_raw = None try: comment_raw = handler.handle(submission.url) except Exception as e: print(f"Exception occurred while handling {submission.url}") traceback.print_exc() if comment_raw is None: skip(submission) return comment_markdown = format_comment(comment_raw) if len(comment_markdown) < config.COMMENT_LENGTH_LIMIT: try: print("Attempting to post a comment") submission.reply(comment_markdown) print("Comment posting succeeded") print("Attempting to write success to database") DatabaseManager.write_id(submission.id, DatabaseActionEnum.SUCCESS) print("Database write succeeded") except Exception as e: print("An error occurred:") print(e) else: print("Submission is too long to be posted.") print("Attempting to write skip to database") DatabaseManager.write_id(submission.id, DatabaseActionEnum.SKIP) print("Database write succeeded") else: skip(submission)
def scan(subreddit): """Scan a Subreddit for new submissions.""" logging.info("Starting scan") for submission in subreddit.stream.submissions(skip_existing=True): logging.info("Operating on submission") logging.debug("Submission ID = %s, title = %s", submission.id, submission.title) does_qualify = qualify(submission) if not does_qualify: logging.info("Comment does not qualify; skipping comment") continue logging.info("Submission qualifies") handler = None try: logging.info("Attempting to get article handler") handler = HandlerManager.get_handler(submission.url) logging.info("Article handler = %s", handler) except Exception as exception: logging.error(""" An error occurred while getting article handler. This should never happen. """) logging.error("Exception = %s", exception) logging.info("Skipping current submission") continue comment_raw = None try: logging.info("Attempting to generate raw comment using handler") comment_raw = handler.handle(submission.url) logging.info("Raw comment generated") except Exception as exception: logging.error("An error occurred while handling URL = %s", submission.url) logging.error("Exception = %s", exception) logging.info("Skipping current submission") continue if comment_raw is None: logging.error("comment_raw is None; skipping current submission") continue logging.info("Generating formatted comment") comment_markdown = format_comment(comment_raw) logging.info("Formatted comment generated") if len(comment_markdown) < config.COMMENT_LENGTH_LIMIT: try: logging.info("Attempting to post comment") submission.reply(comment_markdown) logging.info("Comment posting succeeded") except Exception as exception: logging.error("An error occurred while posting comment") logging.error("Exception = %s", exception) else: logging.warning("Submission is too long to be posted")
def on_close(self): self.r.decr(self.node) HandlerManager.del_handler(self)
def on_message(self, msg): data = HandlerManager.route_message(self, msg) if data != '{}': self.write_message(data)
def open(self): self.r.incr(self.node) HandlerManager.add_handler(self)