def crawl(self, args, browser): source = args['source'] list_page_url = args['list_page_url'] try: appstore_modual = __import__(source) except ImportError: util.write_log_error(self.lock, self.logger, "can not find " + source + ".py in configured appstores") return False appstore_class = appstore_modual.Store() if self.url_rd.exists(util.get_MD5(list_page_url)): util.write_log_warn(self.lock, self.logger, "list_page_url:" + list_page_url + "was crawled in past %d seconds"%(config.CRAWLED_URL_REDIS_TIMEOUT)) return False browser.desired_capabilities["phantomjs.page.settings.userAgent"] = appstore_class.PHANTOMJS_USER_AGENT util.browser_get_url(browser, list_page_url) # appstore_class.scroll_down(browser) check_more_count = appstore_class.click_checkmore(browser) util.write_log_warn(self.lock, self.logger, "列表页: " + list_page_url + "点击checkmore" + str(check_more_count) + "次后找不到checkmore") detail_urls = appstore_class.get_detail_urls(browser) if len(detail_urls) == 0: util.write_log_warn(self.lock, self.logger, "列表页: " + list_page_url + " 找到0个app") self.push_detail_page_message(args, detail_urls) next_list_page = appstore_class.click_nextpage(browser) if next_list_page: self.push_list_page_message(args, next_list_page) util.write_log_info(self.lock, self.logger,"SUCCESS : list_page_url: %s crawled"%(list_page_url)) self.url_rd.set(util.get_MD5(list_page_url),1) self.url_rd.expire(util.get_MD5(list_page_url), config.CRAWLED_URL_REDIS_TIMEOUT) return True
def crawl(self, args, browser): source = args['source'] detail_page_url = args['detail_page_url'] try: appstore_modual = __import__(source) except ImportError: util.write_log_error(self.lock, self.logger, "can not find " + source + ".py in configured appstores") return False appstore_class = appstore_modual.Store() if self.url_rd.exists(util.get_MD5(detail_page_url)): util.write_log_warn(self.lock, self.logger, "detail_page_url:" + detail_page_url + "was crawled in past %d seconds"%(config.CRAWLED_URL_REDIS_TIMEOUT)) return False browser.desired_capabilities["phantomjs.page.settings.userAgent"] = appstore_class.PHANTOMJS_USER_AGENT util.browser_get_url(browser, detail_page_url) download_url = appstore_class.get_download_url(browser) if download_url is None: util.write_log_error(self.lock, self.logger, "详情页: " + detail_page_url + " 找不到download_url") return False out_json_message = appstore_class.make_detail2download_json_message(browser) out_json_message['detail_url'] = detail_page_url out_json_message['download_url'] = download_url if args['level'] = 'high': out_json_message['next_queue'] = config.HIGH_DOWNLOAD_QUEUE