def addUrl(self, url): if url in self.visited: pass else: self.visit.put(url) Log4Spider.infoLog(self, "add a url[[[", url, "]]]", "current size:[[[", self.visit.qsize(), "]]]")
def fetch_one_url(url): env_obj = SpiderEnv(url) env = yield env_obj.gen_env() urlSeek = Jd_Home_Spider(env,app) yield urlSeek.work() for url in urlSeek.urlLists: Log4Spider.infoLog(url) Log4Spider.infoLog(len(urlSeek.urlLists))
def fetch_one_url(url): env_obj = SpiderEnv(url) env = yield env_obj.gen_env() urlSeek = BaseSpider(env, app) yield urlSeek.work() for url in urlSeek.urlLists: Log4Spider.infoLog(url) Log4Spider.infoLog(len(urlSeek.urlLists))
def fetch_one_url(url): env_obj = SpiderEnv(url) env = yield env_obj.gen_env() urlSeek = Jd_Item_Spider(env,app) yield urlSeek.work() for url in urlSeek.urlLists: Log4Spider.infoLog(url) Log4Spider.infoLog(len(urlSeek.urlLists)) global num num-=1 if num == 0: event.set()
def fetch_one_url(url): env_obj = SpiderEnv(url) env = yield env_obj.gen_env() urlSeek = Jd_Item_Spider(env, app) yield urlSeek.work() for url in urlSeek.urlLists: Log4Spider.infoLog(url) Log4Spider.infoLog(len(urlSeek.urlLists)) global num num -= 1 if num == 0: event.set()
def main(): for url in [ "http://www.jianshu.com", "http://upload-images.jianshu.io/upload_images/1679702-7e810a34f3ef8d18.jpg?imageMogr2/auto-orient/strip%7CimageView2/1/w/300/h/300" ]: env_obj = SpiderEnv(url) env = yield env_obj.gen_env() urlSeek = UrlSeekSpider(env, None) yield urlSeek.work() for url in urlSeek.urlLists: Log4Spider.infoLog(url) Log4Spider.infoLog(len(urlSeek.urlLists))
def run(self): while True: url = yield self.queue.get() Log4Spider.debugLog(self,"get url:",url) try: env = yield SpiderEnv(url).gen_env() except Exception as e: Log4Spider.errLog(self,"spider env failed url:",url,"exception:",e) continue self._find_url_handler(url) Log4Spider.infoLog(self,"url: ",url," --- class: ",self.handler_class) spider = self.handler_class(env,self.application,**self.handler_kwargs) yield spider.work() for url in spider.urlLists: Log4Spider.debugLog(self,"put url:",url) yield self.queue.put(url)
def run(self): while True: url = yield self.queue.get() Log4Spider.debugLog(self, "get url:", url) try: env = yield SpiderEnv(url).gen_env() except Exception as e: Log4Spider.errLog(self, "spider env failed url:", url, "exception:", e) continue self._find_url_handler(url) Log4Spider.infoLog(self, "url: ", url, " --- class: ", self.handler_class) spider = self.handler_class(env, self.application, **self.handler_kwargs) yield spider.work() for url in spider.urlLists: Log4Spider.debugLog(self, "put url:", url) yield self.queue.put(url)
def addUrl(self,url): if url in self.visited: pass else: self.visit.put(url) Log4Spider.infoLog(self,"add a url[[[",url,"]]]","current size:[[[",self.visit.qsize(),"]]]")
def main(): parse = HtmlFetch(base_url) result = yield parse.fetch() Log4Spider.infoLog(result)
def main(): parse = HtmlFetch(base_url) result = yield parse.fetch() Log4Spider.infoLog(result)