def _run(self): cur = ZhaopinDB.get_companys() for item in cur: search_key = item['company_name'] if RedisClient.get_search_key_detail_key(search_key): continue logging.info("++++++crawl zhaopin:->search_key: %s" % search_key) url = "http://qiye.qianzhan.com/search/all/" + urllib.quote( search_key.encode( 'utf-8')) + "?o=0&area=11&areaN=%E5%8C%97%E4%BA%AC" try: self._get_search(url) RedisClient.set_search_key_detail_key(search_key) except Error302, err: raise Error302() except Error403, err: raise Error403()
continue logging.info( "++++++crawl zhaopin:->search_key: %s" % search_key) url = "http://qiye.qianzhan.com/search/qy/" + urllib.quote( search_key.encode('utf-8')) + "?o=0&area=11&areaN=%E5%8C%97%E4%BA%AC" try: self._get_search(url) RedisClient.set_search_key_detail_key(search_key) except Error302, err: raise Error302() except Error403, err: raise Error403() except Error400, err: logging.exception("_get_search Error400, search_key:->%s, e:->%s" % (search_key, err)) RedisClient.set_search_key_detail_key(search_key) # raise err except Error404, err: logging.exception("_get_search Error404, search_key:->%s, e:->%s" % (search_key, err)) RedisClient.set_search_key_detail_key(search_key) # raise err except Exception, e: logging.exception("_get_search:->search_key: %s, %s" % (search_key, e.message)) pass def run(self): logging.info("+++++++++++++run++++++++++++++++") try: is_success = self._qianzhan_client.login() if is_success: self._run()