class SiteClient(object): def __init__(self, proxies): # self._username = username # self._password = password self._http_client = HTTPClient(proxies=proxies) pass def _verify_post(self, url, data=None, json=None, times=0): try: response = self._http_client.post(url, data, json) if response.status_code == 200: pass elif response.status_code == 302: location = response.headers['Location'] logging.debug("location: %s" % location) raise Error302() elif response.status_code == 403: raise Error403() elif response.status_code == 404: raise Error404() elif response.status_code == 502: raise Error502() elif response.status_code == 503: raise Error503() else: raise ErrorStatusCode(response.status_code) return response except HttpClientError, err: times += 1 if times < 3: return self._verify_post(url, data=data, json=json, times=times) else: raise err
class QianzhanClient(object): def __init__(self, userId, password): self._userId = userId self._password = password self._http_client = HTTPClient() pass """+++++++++++++++++++login++++++++++++""" def _per_login(self): login_page_url = "http://qiye.qianzhan.com/usercenter/login?ReturnUrl=http%3A%2F%2Fqiye.qianzhan.com%2F" response = self._http_client.get(login_page_url) return self._get_varifyimage(True) def _get_varifyimage(self, is_first=False): if is_first: varifyimage_url = "http://qiye.qianzhan.com/usercenter/varifyimage" else: varifyimage_url = "http://qiye.qianzhan.com/usercenter/varifyimage?" + str(random.random()) response = self._http_client.get(varifyimage_url) # logging.debug("verifyimage: %s" % response.content) varifycode = read_body_to_string(response.content) logging.debug("varifycode: %s" % varifycode.replace(' ', '')) return varifycode.replace(' ', '') def _do_login(self, varifycode, max_times=10): form_data = { "userId": self._userId, "password": self._password, "VerifyCode": varifycode, "sevenDays": "false" } login_url = "http://qiye.qianzhan.com/usercenter/dologin" response = self._http_client.post(login_url, form_data) logging.debug("text: %s" % response.text) try: json_obj = json.loads(response.text) except Exception, e: json_obj = {"isSuccess": False, "sMsg": "is html return"} pass logging.debug("json_obj: %s" % json_obj) if not json_obj.get("isSuccess"): # print json_obj.get("sMsg") max_times -= 1 if max_times > 0: varifycode = self._get_varifyimage() return self._do_login(varifycode, max_times) else: return False # print json_obj.get("sMsg") logging.info("cookie: %s" % response.cookies.get_dict()) return True
class SiteClient(object): def __init__(self): self._http_client = HTTPClient() pass def _verify_post(self, url, data=None, json=None, times=0, headers=default_headers, timeout=download_timeout): # headers.update({ # 'User-Agent': self._user_agent, # # "Proxy-Authorization": self.get_authHeader() # }) try: response = self._http_client.post(url=url, data=data, json=json, headers=headers, timeout=timeout) if response.status_code == 200: logging.debug(response.headers) pass elif response.status_code == 302: location = response.headers['Location'] logging.debug("location: %s" % location) raise Error302() elif response.status_code == 403: raise Error403() elif response.status_code == 404: raise Error404() elif response.status_code == 502: raise Error502() elif response.status_code == 503: raise Error503() else: raise ErrorStatusCode(response.status_code) return response except Error403, err: raise err except HttpClientError, err: times += 1 if times < 2: return self._verify_post(url, data=data, json=json, times=times, headers=headers, timeout=timeout) else: raise err
class SiteClient(object): def __init__(self, proxies={}): # self._username = username # self._password = password self._http_client = HTTPClient(proxies=proxies) self._user_agent = random.choice(USER_AGENTS) self.credit_ticket = None self.currentTimeMillis = None self._detail_url = None self._index_1_url = None self._index_2_url = None self._search_list_url = None self._qynb_detail_url = None pass # -----------------------------------------------get post------------------------------------------------------- # def _verify_post(self, url, data=None, json=None, times=0, headers=default_headers, timeout=download_timeout): headers.update({ 'User-Agent': self._user_agent, # "Proxy-Authorization": self.get_authHeader() }) try: response = self._http_client.post(url=url, data=data, json=json, headers=headers, timeout=timeout) if response.status_code == 200: logging.debug(response.headers) pass elif response.status_code == 302: location = response.headers['Location'] logging.debug("location: %s" % location) raise NeedrefreshProxyError() elif response.status_code == 403: raise Error403() elif response.status_code == 404: raise NeedrefreshProxyError() elif response.status_code == 502: raise NeedrefreshProxyError() elif response.status_code == 503: raise NeedrefreshProxyError() else: raise ErrorStatusCode(response.status_code) if response.content.find('繁访问错误页面') > 0: logging.info( "---------------||||||||||||||||||||||||-------------") raise NeedrefreshProxyError() return response except Error403, err: raise err except HttpClientError, err: times += 1 if times < 2: return self._verify_post(url, data=data, json=json, times=times, headers=headers, timeout=timeout) else: raise err
class BaseSiteClient(object): def __init__(self, config, proxies={}): self._config = config self._http_client = HTTPClient(proxies=proxies) self._user_agent = random.choice(self._config.user_agents) pass def _check_response(self, response): if response.status_code == 200: logging.debug(response.headers) pass elif response.status_code == 302: location = response.headers['Location'] logging.debug("location: %s" % location) raise Error302() elif response.status_code == 403: raise Error403() elif response.status_code == 404: raise Error404() elif response.status_code == 502: raise Error502() elif response.status_code == 503: raise Error503() else: raise ErrorStatusCode(response.status_code) return response def _verify_post(self, url, data=None, json=None, times=0, headers=None, timeout=None): if not headers: headers = self._config.default_headers if not timeout: timeout = self._config.default_timeout headers.update({ 'User-Agent': self._user_agent, # "Proxy-Authorization": self.get_authHeader() }) try: response = self._http_client.post(url=url, data=data, json=json, headers=headers, timeout=timeout) return self._check_response(response) except Error403, err: raise err except HttpClientError, err: times += 1 if times < 2: return self._verify_post(url, data=data, json=json, times=times, headers=headers, timeout=timeout) else: raise err
class SiteClient(object): def __init__(self, username, password): self._username = username self._password = password self._http_client = HTTPClient() pass """+++++++++++++++++++login++++++++++++""" def _per_login(self): login_page_url = "http://center.qianlima.com/login.jsp" response = self._http_client.get(login_page_url) return response def _do_login(self): form_data = { "username": self._username, "password": self._password, "rem_login": "******" } login_url = "http://center.qianlima.com/login_post.jsp?re_url=null" response = self._http_client.post(login_url, form_data) # logging.debug("text: %s" % response.text) # try: # json_obj = json.loads(response.text) # except Exception, e: # json_obj = {"isSuccess": False, "sMsg": "is html return"} # pass # # logging.debug("json_obj: %s" % json_obj) # # if not json_obj.get("isSuccess"): # return False logging.info("cookie: %s" % response.cookies.get_dict()) return True def login(self): # print "++++++++++++++login+++++++++++++++++" self._per_login() is_success = self._do_login() return is_success def _verify_post(self, url, data=None, json=None, **kwargs): kwargs.setdefault("allow_redirects", False) response = self._http_client.post(url, data, json, **kwargs) if response.status_code == 200: pass elif response.status_code == 302: location = response.headers['Location'] logging.debug("location: %s" % location) raise Error302() elif response.status_code == 403: raise Error403() elif response.status_code == 404: raise Error404() else: raise ErrorStatusCode() return response def _verify_get(self, url, **kwargs): kwargs.setdefault("allow_redirects", False) response = self._http_client.get(url, **kwargs) if response.status_code == 200: pass elif response.status_code == 302: location = response.headers['Location'] logging.debug("location: %s" % location) raise Error302() elif response.status_code == 403: raise Error403() elif response.status_code == 404: raise Error404() else: raise ErrorStatusCode() return response def get_company(self, url): response = self._verify_get(url) return response def get_search(self, url): response = self._verify_get(url) return response