class SiteClient(object):
def __init__(self, proxies):
# self._username = username
# self._password = password
self._http_client = HTTPClient(proxies=proxies)
pass
def _verify_post(self, url, data=None, json=None, times=0):
try:
response = self._http_client.post(url, data, json)
if response.status_code == 200:
pass
elif response.status_code == 302:
location = response.headers['Location']
logging.debug("location: %s" % location)
raise Error302()
elif response.status_code == 403:
raise Error403()
elif response.status_code == 404:
raise Error404()
elif response.status_code == 502:
raise Error502()
elif response.status_code == 503:
raise Error503()
else:
raise ErrorStatusCode(response.status_code)
return response
except HttpClientError, err:
times += 1
if times < 3:
return self._verify_post(url, data=data, json=json, times=times)
else:
raise err
class QianzhanClient(object):
def __init__(self, userId, password):
self._userId = userId
self._password = password
self._http_client = HTTPClient()
pass
"""+++++++++++++++++++login++++++++++++"""
def _per_login(self):
login_page_url = "http://qiye.qianzhan.com/usercenter/login?ReturnUrl=http%3A%2F%2Fqiye.qianzhan.com%2F"
response = self._http_client.get(login_page_url)
return self._get_varifyimage(True)
def _get_varifyimage(self, is_first=False):
if is_first:
varifyimage_url = "http://qiye.qianzhan.com/usercenter/varifyimage"
else:
varifyimage_url = "http://qiye.qianzhan.com/usercenter/varifyimage?" + str(random.random())
response = self._http_client.get(varifyimage_url)
# logging.debug("verifyimage: %s" % response.content)
varifycode = read_body_to_string(response.content)
logging.debug("varifycode: %s" % varifycode.replace(' ', ''))
return varifycode.replace(' ', '')
def _do_login(self, varifycode, max_times=10):
form_data = {
"userId": self._userId,
"password": self._password,
"VerifyCode": varifycode,
"sevenDays": "false"
}
login_url = "http://qiye.qianzhan.com/usercenter/dologin"
response = self._http_client.post(login_url, form_data)
logging.debug("text: %s" % response.text)
try:
json_obj = json.loads(response.text)
except Exception, e:
json_obj = {"isSuccess": False, "sMsg": "is html return"}
pass
logging.debug("json_obj: %s" % json_obj)
if not json_obj.get("isSuccess"):
# print json_obj.get("sMsg")
max_times -= 1
if max_times > 0:
varifycode = self._get_varifyimage()
return self._do_login(varifycode, max_times)
else:
return False
# print json_obj.get("sMsg")
logging.info("cookie: %s" % response.cookies.get_dict())
return True
class SiteClient(object):
def __init__(self):
self._http_client = HTTPClient()
pass
def _verify_post(self,
url,
data=None,
json=None,
times=0,
headers=default_headers,
timeout=download_timeout):
# headers.update({
# 'User-Agent': self._user_agent,
# # "Proxy-Authorization": self.get_authHeader()
# })
try:
response = self._http_client.post(url=url,
data=data,
json=json,
headers=headers,
timeout=timeout)
if response.status_code == 200:
logging.debug(response.headers)
pass
elif response.status_code == 302:
location = response.headers['Location']
logging.debug("location: %s" % location)
raise Error302()
elif response.status_code == 403:
raise Error403()
elif response.status_code == 404:
raise Error404()
elif response.status_code == 502:
raise Error502()
elif response.status_code == 503:
raise Error503()
else:
raise ErrorStatusCode(response.status_code)
return response
except Error403, err:
raise err
except HttpClientError, err:
times += 1
if times < 2:
return self._verify_post(url,
data=data,
json=json,
times=times,
headers=headers,
timeout=timeout)
else:
raise err
class SiteClient(object):
def __init__(self, proxies={}):
# self._username = username
# self._password = password
self._http_client = HTTPClient(proxies=proxies)
self._user_agent = random.choice(USER_AGENTS)
self.credit_ticket = None
self.currentTimeMillis = None
self._detail_url = None
self._index_1_url = None
self._index_2_url = None
self._search_list_url = None
self._qynb_detail_url = None
pass
# -----------------------------------------------get post------------------------------------------------------- #
def _verify_post(self,
url,
data=None,
json=None,
times=0,
headers=default_headers,
timeout=download_timeout):
headers.update({
'User-Agent': self._user_agent,
# "Proxy-Authorization": self.get_authHeader()
})
try:
response = self._http_client.post(url=url,
data=data,
json=json,
headers=headers,
timeout=timeout)
if response.status_code == 200:
logging.debug(response.headers)
pass
elif response.status_code == 302:
location = response.headers['Location']
logging.debug("location: %s" % location)
raise NeedrefreshProxyError()
elif response.status_code == 403:
raise Error403()
elif response.status_code == 404:
raise NeedrefreshProxyError()
elif response.status_code == 502:
raise NeedrefreshProxyError()
elif response.status_code == 503:
raise NeedrefreshProxyError()
else:
raise ErrorStatusCode(response.status_code)
if response.content.find('繁访问错误页面') > 0:
logging.info(
"---------------||||||||||||||||||||||||-------------")
raise NeedrefreshProxyError()
return response
except Error403, err:
raise err
except HttpClientError, err:
times += 1
if times < 2:
return self._verify_post(url,
data=data,
json=json,
times=times,
headers=headers,
timeout=timeout)
else:
raise err
class BaseSiteClient(object):
def __init__(self, config, proxies={}):
self._config = config
self._http_client = HTTPClient(proxies=proxies)
self._user_agent = random.choice(self._config.user_agents)
pass
def _check_response(self, response):
if response.status_code == 200:
logging.debug(response.headers)
pass
elif response.status_code == 302:
location = response.headers['Location']
logging.debug("location: %s" % location)
raise Error302()
elif response.status_code == 403:
raise Error403()
elif response.status_code == 404:
raise Error404()
elif response.status_code == 502:
raise Error502()
elif response.status_code == 503:
raise Error503()
else:
raise ErrorStatusCode(response.status_code)
return response
def _verify_post(self,
url,
data=None,
json=None,
times=0,
headers=None,
timeout=None):
if not headers:
headers = self._config.default_headers
if not timeout:
timeout = self._config.default_timeout
headers.update({
'User-Agent': self._user_agent,
# "Proxy-Authorization": self.get_authHeader()
})
try:
response = self._http_client.post(url=url,
data=data,
json=json,
headers=headers,
timeout=timeout)
return self._check_response(response)
except Error403, err:
raise err
except HttpClientError, err:
times += 1
if times < 2:
return self._verify_post(url,
data=data,
json=json,
times=times,
headers=headers,
timeout=timeout)
else:
raise err
class SiteClient(object):
def __init__(self, username, password):
self._username = username
self._password = password
self._http_client = HTTPClient()
pass
"""+++++++++++++++++++login++++++++++++"""
def _per_login(self):
login_page_url = "http://center.qianlima.com/login.jsp"
response = self._http_client.get(login_page_url)
return response
def _do_login(self):
form_data = {
"username": self._username,
"password": self._password,
"rem_login": "******"
}
login_url = "http://center.qianlima.com/login_post.jsp?re_url=null"
response = self._http_client.post(login_url, form_data)
# logging.debug("text: %s" % response.text)
# try:
# json_obj = json.loads(response.text)
# except Exception, e:
# json_obj = {"isSuccess": False, "sMsg": "is html return"}
# pass
#
# logging.debug("json_obj: %s" % json_obj)
#
# if not json_obj.get("isSuccess"):
# return False
logging.info("cookie: %s" % response.cookies.get_dict())
return True
def login(self):
# print "++++++++++++++login+++++++++++++++++"
self._per_login()
is_success = self._do_login()
return is_success
def _verify_post(self, url, data=None, json=None, **kwargs):
kwargs.setdefault("allow_redirects", False)
response = self._http_client.post(url, data, json, **kwargs)
if response.status_code == 200:
pass
elif response.status_code == 302:
location = response.headers['Location']
logging.debug("location: %s" % location)
raise Error302()
elif response.status_code == 403:
raise Error403()
elif response.status_code == 404:
raise Error404()
else:
raise ErrorStatusCode()
return response
def _verify_get(self, url, **kwargs):
kwargs.setdefault("allow_redirects", False)
response = self._http_client.get(url, **kwargs)
if response.status_code == 200:
pass
elif response.status_code == 302:
location = response.headers['Location']
logging.debug("location: %s" % location)
raise Error302()
elif response.status_code == 403:
raise Error403()
elif response.status_code == 404:
raise Error404()
else:
raise ErrorStatusCode()
return response
def get_company(self, url):
response = self._verify_get(url)
return response
def get_search(self, url):
response = self._verify_get(url)
return response
