class HTTPRequest:
def __init__(self):
self.session = Session()
def request_with_cookie(self,method,url,**kwargs):
logger.info("正在以 {} 方式 请求 {}，参数是 {} ".format(method,url,kwargs))
return self.session.request(method=method,url=url,**kwargs)
def upload(self, url, token, account_id, use_type, file_name="1.jpg", **kwargs):
file_path = os.path.join(DOCS_DIR, file_name)
# 因为后端采用的是 multipart/form-data方式,其中有 内容分隔符(boundary), 原生request不支持此方式
file = {"file": ("1.jpg", open(file_path, "rb"), "multipart/form-data"), "use_type": use_type}
#file = {"upload": ("1.jpg", open(file_path, "rb"), "multipart/form-data"), "type": use_type}
file = MultipartEncoder(fields=file)
#headers = {"Content-Type": file.content_type, "token": token}
#print(headers)
headers = {"Content-Type": file.content_type, "token": token, "account_id": account_id}
response = self.session.request("post", url=url, headers=headers, data=file, **kwargs)
print(file)# 因为转化过了 所以不能用files
logger.info("正在请求上传文件 URL:{}, file_name:{}".format(url, file_name))
return response
#用于在测试类结束的时候，关闭seesion
def session_close(self):
self.session.close()
class Spider:
def __init__(self, headers=None):
self.headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
}
if headers:
self.headers.update(headers)
self.session = Session()
self.parmas = None
def load_parmas(self, kwargs: dict):
self.parmas = kwargs
def request(self, method="GET", url=None):
if self.parmas:
result = self.session.request(method, url, **self.parmas)
else:
result = self.session.request(method, url)
return result
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.seesion.close()
del self
class MeiPai:
def __init__(self):
self.session = Session()
self.session.headers = headers
def search(self, query, search_type='mv', page=1):
query = query.strip()
# topic
if query.startswith('#'):
topic = query.strip('#')
return self.get_videos_by_topic(topic)
cache_exists, result = check_cache_and_return_result(query=query, search_type=search_type, page=page)
if cache_exists:
return result
url = 'http://www.meipai.com/search/{search_type}?'.format(search_type=search_type) + \
urlencode({'q': query, 'page': page})
resp = self.session.request('GET', url)
html = BeautifulSoup(resp.content, 'html.parser')
if search_type == 'mv':
video_links = [div.attrs['data-video'].strip() for div in html.find_all(class_='content-l-video')]
# associated_words = self.word_association(query)
# print("你是否还想搜索：" + ",".join(associated_words))
result = video_links
elif search_type == 'topic':
result = [div.text.strip().strip('#') for div in html.find_all(class_='tcard-name')]
else:
result = []
cache_search_result(query, search_type, page, result)
return result
def get_videos_by_topic(self, topic_name):
"""
get top videos by topic
:param topic_name:
:return:
"""
topic = Topic(topic_name)
topic_id = topic.topic_id
url = "http://www.meipai.com/topics/hot_timeline?page=1&count=24&tid={topic_id}".format(topic_id=topic_id)
resp = self.session.request('GET', url)
result = json.loads(resp.text)
return [media['video'] for media in result['medias']]
# get associated words
def word_association(self, word):
url = 'http://www.meipai.com/search/word_assoc?' + urlencode({'q': word})
resp = self.session.request('GET', url)
return json.loads(resp.text)
def _inner_call(*args, **kwargs):
""" Wraps Session.request """
response = Session.request(*args, **kwargs)
logger.debug('Request headers: %s', response.request.headers)
logger.debug('Response headers: %s', response.headers)
self._check_status_code(response)
return response
def download_image(url, **kwargs):
"""下载验证码图片
:param url: 验证码网址
:param kwargs: requests的请求参数一致
:return: image对象 or None
"""
response = None
times = 0
while times < 10:
try:
session = Session()
response = session.request(method='GET', url=url, **kwargs)
break
except:
times += 1
print 'Timeout, I will try to request again……'
image = None
if response != None:
file = BytesIO(response.content)
image = Image.open(file)
return image
def request(session: Session, method, url, *args, **kwargs):
resp = session.request(method, url, *args, **kwargs)
# Check if Cloudflare anti-bot is on
if (resp.status_code == 503
and resp.headers.get("Server") == "cloudflare-nginx"
and b"jschl_vc" in resp.content
and b"jschl_answer" in resp.content):
return solve_cf_challenge(session, resp, **kwargs)
# Otherwise, no Cloudflare anti-bot detected
return resp
def load_loginpage(session: Session, form_meta: Dict[str,
str]) -> Dict[str, str]:
"""
载入登录页面
:return:
* request_verification_token - 表单里的隐藏Token
* captcha_image_url - 验证码的URL，地址相对于根路径
:raise RuntimeError 加载登录页面失败
"""
url = "https://row1.vfsglobal.com/GlobalAppointment/"
payload = {}
headers = {
'Connection': 'keep-alive',
'Pragma': 'no-cache',
'Cache-Control': 'no-cache',
'Upgrade-Insecure-Requests': '1',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36',
'Sec-Fetch-User': '******',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-Mode': 'navigate',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8'
}
def parse_page(page_html):
soup = BeautifulSoup(page_html, 'html.parser')
form = soup.find('form', {'id': 'ApplicantListForm'})
return {
'request_verification_token':
form.find('input',
{'name': '__RequestVerificationToken'})['value'],
'captcha_image_url':
form.find('img', {'id': 'CaptchaImage'})['src'],
}
logger.info("尝试加载登录页面")
response = session.request("GET", url, headers=headers, data=payload)
if response.status_code == 200:
return {
**form_meta,
**parse_page(response.text),
}
raise RuntimeError('加载登录页面失败', response)
def get_csv(sess: sessions.Session, endpoint: str, **parameters):
"""
get csv data from the Intrinio API
:param sess: session
:param endpoint: endpoint
:param parameters: query parameters
:return: csv result
"""
auth = os.getenv('INTRINIO_USERNAME'), os.getenv('INTRINIO_PASSWORD')
url = '{}/{}'.format(
'https://api.intrinio.com',
endpoint + ('' if endpoint.endswith('.csv') else '.csv'))
if 'page_size' not in parameters:
parameters['page_size'] = 10000
pages = list()
for page_number in itertools.count():
parameters['page_number'] = page_number + 1
response = sess.request('GET',
url,
params=parameters,
auth=auth,
verify=True)
if not response.ok:
try:
response.raise_for_status()
except Exception as err:
logging.getLogger(__name__).error(err)
new_lines = response.content.decode('utf-8').count('\n')
if new_lines == 1:
break
info, columns, page = response.content.decode('utf-8').split('\n', 2)
if page_number == 0:
info = {s.split(':')[0]: s.split(':')[1] for s in info.split(',')}
total_pages = int(info['TOTAL_PAGES'])
pages.append(columns.lower() + '\n')
pages.append(page)
if len(page) == 0 or page_number + 1 == total_pages:
break
return ''.join(pages) if len(pages) > 0 else None
def solve_captcha(session: Session, form_meta: Dict[str, str],
howto: Callable[[PIL.Image.Image], str]) -> Dict[str, str]:
"""
解决验证码
:param session: Session
:param form_meta: 登录页面的结果
:param howto: 如何从验证码图片中解读出文字
:return:
* captcha_detext - 验证码ID
* captcha_inputtext - 验证码明文
:raise RuntimeError 获取验证码失败
"""
captcha_detext = form_meta['captcha_image_url'].split('=')[1]
url = "https://row1.vfsglobal.com/GlobalAppointment/DefaultCaptcha/Generate"
params = {'t': captcha_detext}
payload = {}
headers = {
'Connection': 'keep-alive',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36',
'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-Mode': 'no-cors',
'Referer': 'https://row1.vfsglobal.com/GlobalAppointment/',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
}
logger.info("尝试下载验证码图片")
response = session.request("GET",
url,
params=params,
headers=headers,
data=payload)
if response.status_code == 200:
from PIL import Image
from io import BytesIO
image = Image.open(BytesIO(response.content))
captcha_inputtext = howto(image)
logger.info("验证码: %s => %s", captcha_detext, captcha_inputtext)
del form_meta['captcha_image_url']
return {
**form_meta, 'captcha_detext': captcha_detext,
'captcha_inputtext': captcha_inputtext
}
raise RuntimeError('获取验证码失败', response)
def solve_cf_challenge(session: Session, resp, **original_kwargs):
sleep(5) # Cloudflare requires a delay before solving the challenge
body = resp.text
parsed_url = urlparse(resp.url)
domain = urlparse(resp.url).netloc
submit_url = "%s://%s/cdn-cgi/l/chk_jschl" % (parsed_url.scheme, domain)
cloudflare_kwargs = deepcopy(original_kwargs)
params = cloudflare_kwargs.setdefault("params", {})
headers = cloudflare_kwargs.setdefault("headers", {})
headers["Referer"] = resp.url
try:
params["jschl_vc"] = re.search(r'name="jschl_vc" value="(\w+)"',
body).group(1)
params["pass"] = re.search(r'name="pass" value="(.+?)"', body).group(1)
except Exception as e:
# Something is wrong with the page.
# This may indicate Cloudflare has changed their anti-bot
# technique. If you see this and are running the latest version,
# please open a GitHub issue so I can update the code accordingly.
raise ValueError("Unable to parse Cloudflare anti-bots page: %s %s" %
(e.message, BUG_REPORT))
# Solve the Javascript challenge
params["jschl_answer"] = str(solve_challenge(session, body) + len(domain))
# Requests transforms any request into a GET after a redirect,
# so the redirect has to be handled manually here to allow for
# performing other types of requests even as the first request.
method = resp.request.method
cloudflare_kwargs["allow_redirects"] = False
redirect = session.request(method, submit_url, **cloudflare_kwargs)
return session.request(method, redirect.headers["Location"],
**original_kwargs)
def load_appointment_page(session: Session, form_meta: Dict[str, str]):
"""
加载/GlobalAppointment/Home/SelectVAC，读取表单隐藏字段
:param
* vac_link
:return:
* request_verification_token
"""
url = BASE_URL + form_meta['vac_link']
payload = {}
headers = {
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36',
'Sec-Fetch-User': '******',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-Mode': 'navigate',
'Referer': url,
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
}
logger.info("加载预约页面")
response = session.request("GET", url, headers=headers, data=payload)
def parse_page(page_html: str) -> Dict[str, str]:
"""
加载成功以后，获取提交表单所需的信息
Une fois le chargement réussi, obtenez les informations nécessaires pour envoyer le formulaire
"""
soup = BeautifulSoup(page_html, 'html.parser')
form = soup.find('form', {'id': 'VisaApplicationForm'})
return {
'request_verification_token':
form.find('input',
{'name': '__RequestVerificationToken'})['value'],
}
if response.status_code == 200:
return {**form_meta, **parse_page(response.text)}
raise RuntimeError("预约页面加载失败", response)
class RequestsClient(HttpClient):
"""An implementation of HttpClient that uses Requests as its HTTP Client
Attributes:
timeout (int): The default timeout for all API requests.
"""
def __init__(self, timeout=60, cache=False, max_retries=None, verify=True):
"""The constructor.
Args:
timeout (float): The default global timeout(seconds).
"""
self.timeout = timeout
self.session = Session()
retries = Retry(total=max_retries)
self.session.mount('https://', HTTPAdapter(max_retries=retries))
self.session.verify = verify
def execute(self, request):
"""Execute a given HttpRequest to get a string response back
Args:
request (HttpRequest): The given HttpRequest to execute.
Returns:
HttpResponse: The response of the HttpRequest.
"""
response = self.session.request(HttpMethodEnum.to_string(
request.http_method),
request.query_url,
headers=request.headers,
params=request.query_parameters,
data=request.parameters,
timeout=self.timeout)
return self.convert_response(response, request)
def convert_response(self, response, http_request):
"""Converts the Response object of the HttpClient into an
HttpResponse object.
Args:
response (dynamic): The original response object.
http_request (HttpRequest): The original HttpRequest object.
Returns:
HttpResponse: The converted HttpResponse object.
"""
return HttpResponse(response.status_code, response.text,
response.headers, http_request)
def _check_appointment(session: Session, form_meta: Dict[str, str],
location_code, location_name) -> Optional[dict]:
"""
检查空席位
:param
* request_verification_token
* vac_link
"""
url = "https://row1.vfsglobal.com/GlobalAppointment/Account/GetEarliestVisaSlotDate"
payload = f"countryId=6&missionId=7&LocationId={location_code}&VisaCategoryId=2815"
headers = {
'Connection': 'keep-alive',
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Origin': 'https://row1.vfsglobal.com',
'X-Requested-With': 'XMLHttpRequest',
'__RequestVerificationToken': form_meta['request_verification_token'],
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-Mode': 'cors',
'Referer': BASE_URL + form_meta['vac_link'],
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
}
estimate_login_time = \
LAST_LOGIN_TIME + timedelta(minutes=20) - datetime.now()
logger.info("获取 %s 预约空位，预计离下次要求登录还有 %s", location_name,
estimate_login_time)
response = session.request("POST", url, headers=headers, data=payload)
if response.text == '""':
return None
if response.text.startswith("{"):
return json.loads(response.text)
if response.text.startswith("<"):
raise RuntimeError("获取失败，账号失效")
logger.info("预约空位: %s", response.text)
raise RuntimeError("获取失败", response.text)
def download_image(url, **kwargs):
"""下载验证码图片
:param url: 验证码网址
:param kwargs: requests的请求参数一致
:return: image对象 or None
"""
response = None
times = 0
while times < 10:
try:
session = Session()
response = session.request('GET', url=url, **kwargs)
if response.status_code == 200:
break
except:
times += 1
print('Download failed, try it again.')
if response:
return response.content
else:
return response
class PortalConnection(object):
"""
Connection to HubSpot
:param authentication_key: This can be either an :class:`APIKey` or an \
:class:`OAuthKey` instance
:param basestring change_source: The string passed to HubSpot as \
``auditId`` in the query string
"""
_API_URL = 'https://api.hubapi.com'
def __init__(self, authentication_key, change_source):
super(PortalConnection, self).__init__()
self._authentication_handler = \
_QueryStringAuthenticationHandler(authentication_key)
self._change_source = change_source
self._session = Session()
self._session.headers['User-Agent'] = _USER_AGENT
http_adapter = HTTPAdapter(max_retries=_HTTP_CONNECTION_MAX_RETRIES)
self._session.mount('', http_adapter)
def send_get_request(self, url_path, query_string_args=None):
"""
Send a GET request to HubSpot
:param basestring url_path: The URL path to the endpoint
:param dict query_string_args: The query string arguments
:return: Decoded version of the ``JSON`` that HubSpot put in \
the body of the response.
"""
return self._send_request('GET', url_path, query_string_args)
def send_post_request(self, url_path, body_deserialization):
"""
Send a POST request to HubSpot
:param basestring url_path: The URL path to the endpoint
:param dict body_deserialization: The request's body message \
deserialized
:return: Decoded version of the ``JSON`` that HubSpot put in \
the body of the response.
"""
return self._send_request(
'POST',
url_path,
body_deserialization=body_deserialization,
)
def send_put_request(self, url_path, body_deserialization):
"""
Send a PUT request to HubSpot
:param basestring url_path: The URL path to the endpoint
:param body_deserialization: The request's body message deserialized
:return: Decoded version of the ``JSON`` that HubSpot put in \
the body of the response.
"""
return self._send_request(
'PUT',
url_path,
body_deserialization=body_deserialization,
)
def send_delete_request(self, url_path):
"""
Send a DELETE request to HubSpot
:param basestring url_path: The URL path to the endpoint
:return: Decoded version of the ``JSON`` that HubSpot put in \
the body of the response.
"""
return self._send_request('DELETE', url_path)
def _send_request(
self,
method,
url_path,
query_string_args=None,
body_deserialization=None,
):
url = self._API_URL + url_path
query_string_args = query_string_args or {}
query_string_args = dict(query_string_args, auditId=self._change_source)
request_headers = \
{'content-type': 'application/json'} if body_deserialization else {}
if body_deserialization:
request_body_serialization = json_serialize(body_deserialization)
else:
request_body_serialization = None
response = self._session.request(
method,
url,
params=query_string_args,
auth=self._authentication_handler,
data=request_body_serialization,
headers=request_headers,
)
response_body_deserialization = \
self._deserialize_response_body(response)
return response_body_deserialization
@classmethod
def _deserialize_response_body(cls, response):
cls._require_successful_response(response)
cls._require_json_response(response)
if response.status_code == HTTP_STATUS_OK:
response_body_deserialization = response.json()
elif response.status_code in _HTTP_STATUS_CODES_WITH_EMPTY_BODIES:
response_body_deserialization = None
else:
exception_message = \
'Unsupported response status {}'.format(response.status_code)
raise HubspotUnsupportedResponseError(exception_message)
return response_body_deserialization
@staticmethod
def _require_successful_response(response):
if 400 <= response.status_code < 500:
response_data = response.json()
error_data = _HUBSPOT_ERROR_RESPONSE_SCHEMA(response_data)
if response.status_code == HTTP_STATUS_UNAUTHORIZED:
exception_class = HubspotAuthenticationError
else:
exception_class = HubspotClientError
raise exception_class(
error_data['message'],
error_data['requestId'],
)
elif 500 <= response.status_code < 600:
raise HubspotServerError(response.reason, response.status_code)
@staticmethod
def _require_json_response(response):
content_type_header_value = response.headers.get('Content-Type')
if not content_type_header_value:
exception_message = 'Response does not specify a Content-Type'
raise HubspotUnsupportedResponseError(exception_message)
content_type = content_type_header_value.split(';')[0].lower()
if content_type != 'application/json':
exception_message = \
'Unsupported response content type {}'.format(content_type)
raise HubspotUnsupportedResponseError(exception_message)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self._session.close()
class TestClient(object):
def __init__(self):
self._session = Session()
self._session.hooks["response"] = [
_hook_check_status_code,
_hook_check_errcode,
]
def _request(self, method, path, **kwargs):
return self._session.request(method, BASE_URL + path, **kwargs)
def _get(self, path, params={}, **kwargs):
return self._request('GET', path, params=params, **kwargs)
def _post(self, path, data={}, **kwargs):
return self._request('POST', path, data=data, **kwargs)
def create_admin(self, username, password, type, internal_token):
return self._post("/_internal/create_admin", {
"username": username,
"password": password,
"type": type,
},
headers={"Authorization": internal_token})
def create_activity(self, date, begin, end, site, root_token):
return self._post("/activity/create", {
"date": date,
"begin": begin,
"end": end,
"site": site,
},
headers={"Authorization": root_token})
def create_order(self, model, description, repairType, period, email,
user_token):
return self._post("/order/create", {
"model": model,
"description": description,
"repairType": repairType,
"period": period,
"email": email,
},
headers={"Authorization": user_token})
def get_existed_order(self, user_token):
return self._get("/order/get_existed",
headers={
"Authorization": user_token,
})
def get_register_token(self, admin_token):
return self._get("/order/get_register_token",
headers={
"Authorization": admin_token,
})
def register_order(self, oid, token, user_token):
return self._post("/order/register", {
"oid": oid,
"token": token,
},
headers={"Authorization": user_token})
def _get_id(self):
session = Session()
resp = session.request('GET', 'http://www.meipai.com/topic/' + quote_plus(self.name),
headers=headers)
html = BeautifulSoup(resp.content, 'html.parser')
return html.find(class_='topic-txt').attrs['data-id']
class PortalConnection(object):
"""
Connection to HubSpot
:param authentication_key: This can be either an :class:`APIKey` or an \
:class:`OAuthKey` instance
:param basestring change_source: The string passed to HubSpot as \
``auditId`` in the query string
"""
_API_URL = 'https://api.hubapi.com'
def __init__(self, authentication_key, change_source):
super(PortalConnection, self).__init__()
self._authentication_handler = \
_QueryStringAuthenticationHandler(authentication_key)
self._change_source = change_source
self._session = Session()
self._session.headers['User-Agent'] = _USER_AGENT
http_adapter = HTTPAdapter(max_retries=_HTTP_CONNECTION_MAX_RETRIES)
self._session.mount('', http_adapter)
def send_get_request(self, url_path, query_string_args=None):
"""
Send a GET request to HubSpot
:param basestring url_path: The URL path to the endpoint
:param dict query_string_args: The query string arguments
:return: Decoded version of the ``JSON`` that HubSpot put in \
the body of the response.
"""
return self._send_request('GET', url_path, query_string_args)
def send_post_request(self, url_path, body_deserialization):
"""
Send a POST request to HubSpot
:param basestring url_path: The URL path to the endpoint
:param dict body_deserialization: The request's body message \
deserialized
:return: Decoded version of the ``JSON`` that HubSpot put in \
the body of the response.
"""
return self._send_request(
'POST',
url_path,
body_deserialization=body_deserialization,
)
def send_put_request(self, url_path, body_deserialization):
"""
Send a PUT request to HubSpot
:param basestring url_path: The URL path to the endpoint
:param body_deserialization: The request's body message deserialized
:return: Decoded version of the ``JSON`` that HubSpot put in \
the body of the response.
"""
return self._send_request(
'PUT',
url_path,
body_deserialization=body_deserialization,
)
def send_delete_request(self, url_path):
"""
Send a DELETE request to HubSpot
:param basestring url_path: The URL path to the endpoint
:return: Decoded version of the ``JSON`` that HubSpot put in \
the body of the response.
"""
return self._send_request('DELETE', url_path)
def _send_request(
self,
method,
url_path,
query_string_args=None,
body_deserialization=None,
):
url = self._API_URL + url_path
query_string_args = query_string_args or {}
query_string_args = dict(query_string_args,
auditId=self._change_source)
request_headers = \
{'content-type': 'application/json'} if body_deserialization else {}
if body_deserialization:
request_body_serialization = json_serialize(body_deserialization)
else:
request_body_serialization = None
response = self._session.request(
method,
url,
params=query_string_args,
auth=self._authentication_handler,
data=request_body_serialization,
headers=request_headers,
)
response_body_deserialization = \
self._deserialize_response_body(response)
return response_body_deserialization
@classmethod
def _deserialize_response_body(cls, response):
cls._require_successful_response(response)
if response.status_code == HTTP_STATUS_OK:
cls._require_json_response(response)
response_body_deserialization = response.json() or None
elif response.status_code in _HTTP_STATUS_CODES_WITH_EMPTY_BODIES:
response_body_deserialization = None
else:
exception_message = \
'Unsupported response status {}'.format(response.status_code)
raise HubspotUnsupportedResponseError(exception_message)
return response_body_deserialization
@staticmethod
def _require_successful_response(response):
if 400 <= response.status_code < 500:
response_data = response.json()
error_data = _HUBSPOT_ERROR_RESPONSE_SCHEMA(response_data)
if response.status_code == HTTP_STATUS_UNAUTHORIZED:
exception_class = HubspotAuthenticationError
else:
exception_class = HubspotClientError
raise exception_class(
error_data['message'],
error_data['requestId'],
)
elif 500 <= response.status_code < 600:
raise HubspotServerError(response.reason, response.status_code)
@staticmethod
def _require_json_response(response):
content_type_header_value = response.headers.get('Content-Type')
if not content_type_header_value:
exception_message = 'Response does not specify a Content-Type'
raise HubspotUnsupportedResponseError(exception_message)
content_type = content_type_header_value.split(';')[0].lower()
if content_type != 'application/json':
exception_message = \
'Unsupported response content type {}'.format(content_type)
raise HubspotUnsupportedResponseError(exception_message)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self._session.close()
class Connection(object):
"""
Permite crear/gestionar la conexion con Hubspot.
Parametros:
API-KEY
"""
HS_API_URL = "http://api.hubapi.com"
def __init__(self, apikey_value):
super(Connection, self).__init__()
self.APIKEY_VALUE = apikey_value
self.APIKEY = "?hapikey=" + self.APIKEY_VALUE
self.session = Session()
self.session.headers['User-Agent'] = 'Frank Collaboration'
def send_request(self, method, xurl, query_string_args=None, parameters=None,):
"""
Metodo generico de comunicacion con la api de hubspot
Parametros:
method: metodo de comunicacion,
xurl: url pacial de la API,
parameters: Diccionario con los parametros para la api
"""
url = self.HS_API_URL + xurl + self.APIKEY
query_string_args = query_string_args or {}
query_string_args = dict(query_string_args)
request_headers = \
{'content-type': 'application/json'} if parameters else {}
if parameters:
request_parameters = json.dumps(parameters)
else:
request_parameters = None
response = self.session.request(
method,
url,
params=query_string_args,
data=request_parameters,
headers=request_headers,
)
return response
def send_get_request(self, url_path, query_string_args=None):
"""
Metodo GET de comunicacion con la api de hubspot
Parametros:
url_path: url de la funcion de la api a implementar
"""
return self.send_request('GET', url_path, query_string_args)
def send_post_request(self, url_path, parameters):
"""
Metodo POST de comunicacion con la api de hubspot
Parametros:
url_path: url de la funcion de la api a implementar
parameters: Parametros requeridos por la funcion
"""
return self.send_request('POST', url_path, parameters=parameters)
def send_put_request(self, url_path, parameters):
"""
Metodo PUT de comunicacion con la api de hubspot
Parametros:
url_path: url de la funcion de la api a implementar
parameters: Parametros requeridos por la funcion
"""
return self.send_request('PUT', url_path, parameters=parameters)
class MWS(object):
""" Base Amazon API class """
# This is used to post/get to the different uris used by amazon per api
# ie. /Orders/2011-01-01
# All subclasses must define their own URI only if needed
URI = "/"
# The API version varies in most amazon APIs
VERSION = "2009-01-01"
# There seem to be some xml namespace issues. therefore every api subclass
# is recommended to define its namespace, so that it can be referenced
# like so AmazonAPISubclass.NS.
# For more information see http://stackoverflow.com/a/8719461/389453
NS = ''
# Some APIs are available only to either a "Merchant" or "Seller"
# the type of account needs to be sent in every call to the amazon MWS.
# This constant defines the exact name of the parameter Amazon expects
# for the specific API being used.
# All subclasses need to define this if they require another account type
# like "Merchant" in which case you define it like so.
# ACCOUNT_TYPE = "Merchant"
# Which is the name of the parameter for that specific account type.
ACCOUNT_TYPE = "SellerId"
def __init__(self,
access_key,
secret_key,
account_id,
domain='https://mws.amazonservices.com',
uri="",
version=""):
self.access_key = access_key
self.secret_key = secret_key
self.account_id = account_id
self.domain = domain
self.uri = uri or self.URI
self.version = version or self.VERSION
self.session = Session()
bucket_key = getattr(settings, 'RUNSCOPE_BUCKET_KEY', None)
if bucket_key:
logger.info("Redirecting API calls for MWS to runscope")
self.configure_runscope(bucket_key)
def configure_runscope(self, bucket_key):
"""
Configure all connections to be proxied through runscope for easier
debugging and logging of all requests and responses. *bucket_key* is
API for the bucket you want to use for all the request. Check Runscope
for more details on that.
"""
try:
from requests_runscope import RunscopeAdapter
except ImportError:
logger.error(
"Could not import runscope adapter. Is requests-runscope "
"installed? Try running pip install requests-runscope.")
else:
logger.info('Mounting runscope proxy adapter for bucket {}'.format(
bucket_key))
self.session.mount('https://', RunscopeAdapter(bucket_key))
self.session.mount('http://', RunscopeAdapter(bucket_key))
def _get_quote_params(self, params):
quoted_params = []
for key in sorted(params):
value = urllib.quote(unicode(params[key]).encode('utf-8'),
safe='-_.~')
quoted_params.append("{}={}".format(key, value))
return '&'.join(quoted_params)
def make_request(self, extra_data, method="GET", **kwargs):
"""
Make request to Amazon MWS API with these parameters
"""
# Remove all keys with an empty value because
# Amazon's MWS does not allow such a thing.
extra_data = remove_empty(extra_data)
params = {
'AWSAccessKeyId': self.access_key,
self.ACCOUNT_TYPE: self.account_id,
'SignatureVersion': '2',
'Timestamp': self.get_timestamp(),
'Version': self.version,
'SignatureMethod': 'HmacSHA256',
}
params.update(extra_data)
logger.debug("Request Parameters: {}".format(params))
request_description = self._get_quote_params(params)
signature = self.calc_signature(method, request_description)
logger.debug('Domain: {} URI: {}'.format(self.domain, self.uri))
url = '%s%s?%s&Signature=%s' % (self.domain, self.uri,
request_description,
urllib.quote(signature))
headers = {'User-Agent': 'python-amazon-mws/0.0.1 (Language=Python)'}
headers.update(kwargs.get('extra_headers', {}))
try:
# Some might wonder as to why i don't pass the params dict as the
# params argument to request. My answer is, here i have to get the
# url parsed string of params in order to sign it, so if i pass the
# params dict as params to request, request will repeat that step
# because it will need to convert the dict to a url parsed string,
# so why do it twice if i can just pass the full url :).
response = self.session.request(method,
url,
data=kwargs.get('body', ''),
headers=headers)
response.raise_for_status()
# When retrieving data from the response object, be aware that
# response.content returns the content in bytes while response.text
# calls response.content and converts it to unicode.
data = response.content
# I do not check the headers to decide which content structure to
# server simply because sometimes Amazon's MWS API returns XML
# error responses with "text/plain" as the Content-Type.
action = extra_data.get('Action')
if not action.endswith('Result'):
action = "{}Result".format(action)
try:
parsed_response = DictWrapper(data, action)
except XMLError:
parsed_response = DataWrapper(data, response.headers)
except HTTPError, e:
error = MWSError(unicode(e))
error.response = e.response
error.url = url
logger.error("Received {} with message: {}".format(
unicode(e), e.response.content))
raise error
# Store the response object in the parsed_response for quick access
parsed_response.response = response
logger.debug("Received response: {}".format(response.content))
return parsed_response
class MWS(object):
""" Base Amazon API class """
# This is used to post/get to the different uris used by amazon per api
# ie. /Orders/2011-01-01
# All subclasses must define their own URI only if needed
URI = "/"
# The API version varies in most amazon APIs
VERSION = "2009-01-01"
# There seem to be some xml namespace issues. therefore every api subclass
# is recommended to define its namespace, so that it can be referenced
# like so AmazonAPISubclass.NS.
# For more information see http://stackoverflow.com/a/8719461/389453
NS = ''
# Some APIs are available only to either a "Merchant" or "Seller"
# the type of account needs to be sent in every call to the amazon MWS.
# This constant defines the exact name of the parameter Amazon expects
# for the specific API being used.
# All subclasses need to define this if they require another account type
# like "Merchant" in which case you define it like so.
# ACCOUNT_TYPE = "Merchant"
# Which is the name of the parameter for that specific account type.
ACCOUNT_TYPE = "SellerId"
def __init__(self, access_key, secret_key, account_id,
domain='https://mws.amazonservices.com', uri="", version=""):
self.access_key = access_key
self.secret_key = secret_key
self.account_id = account_id
self.domain = domain
self.uri = uri or self.URI
self.version = version or self.VERSION
self.session = Session()
bucket_key = getattr(settings, 'RUNSCOPE_BUCKET_KEY', None)
if bucket_key:
logger.info("Redirecting API calls for MWS to runscope")
self.configure_runscope(bucket_key)
def configure_runscope(self, bucket_key):
"""
Configure all connections to be proxied through runscope for easier
debugging and logging of all requests and responses. *bucket_key* is
API for the bucket you want to use for all the request. Check Runscope
for more details on that.
"""
try:
from requests_runscope import RunscopeAdapter
except ImportError:
logger.error(
"Could not import runscope adapter. Is requests-runscope "
"installed? Try running pip install requests-runscope."
)
else:
logger.info(
'Mounting runscope proxy adapter for bucket {}'.format(
bucket_key
)
)
self.session.mount('https://', RunscopeAdapter(bucket_key))
self.session.mount('http://', RunscopeAdapter(bucket_key))
def _get_quote_params(self, params):
quoted_params = []
for key in sorted(params):
value = urllib.quote(
unicode(params[key]).encode('utf-8'),
safe='-_.~'
)
quoted_params.append("{}={}".format(key, value))
return '&'.join(quoted_params)
def make_request(self, extra_data, method="GET", **kwargs):
"""
Make request to Amazon MWS API with these parameters
"""
# Remove all keys with an empty value because
# Amazon's MWS does not allow such a thing.
extra_data = remove_empty(extra_data)
params = {
'AWSAccessKeyId': self.access_key,
self.ACCOUNT_TYPE: self.account_id,
'SignatureVersion': '2',
'Timestamp': self.get_timestamp(),
'Version': self.version,
'SignatureMethod': 'HmacSHA256',
}
params.update(extra_data)
logger.debug("Request Parameters: {}".format(params))
request_description = self._get_quote_params(params)
signature = self.calc_signature(method, request_description)
logger.debug('Domain: {} URI: {}'.format(self.domain, self.uri))
url = '%s%s?%s&Signature=%s' % (self.domain, self.uri,
request_description,
urllib.quote(signature))
headers = {'User-Agent': 'python-amazon-mws/0.0.1 (Language=Python)'}
headers.update(kwargs.get('extra_headers', {}))
try:
# Some might wonder as to why i don't pass the params dict as the
# params argument to request. My answer is, here i have to get the
# url parsed string of params in order to sign it, so if i pass the
# params dict as params to request, request will repeat that step
# because it will need to convert the dict to a url parsed string,
# so why do it twice if i can just pass the full url :).
response = self.session.request(
method, url, data=kwargs.get('body', ''), headers=headers)
response.raise_for_status()
# When retrieving data from the response object, be aware that
# response.content returns the content in bytes while response.text
# calls response.content and converts it to unicode.
data = response.content
# I do not check the headers to decide which content structure to
# server simply because sometimes Amazon's MWS API returns XML
# error responses with "text/plain" as the Content-Type.
action = extra_data.get('Action')
if not action.endswith('Result'):
action = "{}Result".format(action)
try:
parsed_response = DictWrapper(data, action)
except XMLError:
parsed_response = DataWrapper(data, response.headers)
except HTTPError, e:
error = MWSError(unicode(e))
error.response = e.response
error.url = url
logger.error(
"Received {} with message: {}".format(
unicode(e),
e.response.content
)
)
raise error
# Store the response object in the parsed_response for quick access
parsed_response.response = response
logger.debug("Received response: {}".format(response.content))
return parsed_response
def simulate_login(session: Session, form_meta: Dict[str,
str]) -> Dict[str, str]:
"""
模拟登录
:param:
* request_verification_token
* captcha_detext
* captcha_inputtext
* username
* password
:return:
* vac_link - 访问/GlobalAppointment/Home/SelectVAC的链接
:raise:
* RuntimeError 登录失败
"""
url = "https://row1.vfsglobal.com/GlobalAppointment/"
payload = {
'__RequestVerificationToken': form_meta['request_verification_token'],
'Mission': '',
'Country': '',
'Center': '',
'IsGoogleCaptchaEnabled': 'False',
'reCaptchaURL':
'https://www.google.com/recaptcha/api/siteverify?secret={0}&response={1}',
'reCaptchaPublicKey': '6Ld-Kg8UAAAAAK6U2Ur94LX8-Agew_jk1pQ3meJ1',
'EmailId': form_meta['username'],
'Password': form_meta['password'],
'CaptchaDeText': form_meta['captcha_detext'],
'CaptchaInputText': form_meta['captcha_inputtext'],
}
headers = {
'Connection': 'keep-alive',
'Cache-Control': 'max-age=0',
'Origin': 'https://row1.vfsglobal.com',
'Upgrade-Insecure-Requests': '1',
'Content-Type': 'application/x-www-form-urlencoded',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36',
'Sec-Fetch-User': '******',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-Mode': 'navigate',
'Referer': 'https://row1.vfsglobal.com/GlobalAppointment/',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
}
def parse_page(page_html: str) -> Dict[str, str]:
"""
登录成功以后，获取预约页面的访问链接
"""
soup = BeautifulSoup(page_html, 'html.parser')
def check_account_banned():
banner = soup.find('div', {'class': 'validation-summary-errors'})
if banner is None:
return
raise RuntimeError('登录失败', banner.text.strip())
check_account_banned()
def is_vac_link(tag: Tag):
return tag.has_attr('href') and \
tag['href'].startswith('/GlobalAppointment/Home/SelectVAC')
return {'vac_link': soup.find(is_vac_link)['href']}
logger.info("尝试用 %s 进行登录", form_meta['username'])
response = session.request("POST", url, headers=headers, data=payload)
if response.status_code == 200: # follow redirection
del form_meta['request_verification_token']
del form_meta['captcha_detext']
del form_meta['captcha_inputtext']
global LAST_LOGIN_TIME
LAST_LOGIN_TIME = datetime.now()
return {**form_meta, **parse_page(response.text)}
raise RuntimeError('登录失败', response)
class Connection:
_API_URL = 'https://www.2degreesnetwork.com/api'
def __init__(self, auth, timeout=None, api_url=None):
super(Connection, self).__init__()
self._api_url = api_url or self._API_URL
self._authentication_handler = auth
self._session = Session()
self._session.headers['User-Agent'] = _USER_AGENT
self._timeout = timeout
http_adapter = HTTPAdapter(max_retries=_HTTP_CONNECTION_MAX_RETRIES)
self._session.mount('', http_adapter)
def send_get_request(self, url, query_string_args=None):
"""
Send a GET request
:param str url: The URL or URL path to the endpoint
:param dict query_string_args: The query string arguments
:return: Decoded version of the ``JSON`` the remote put in \
the body of the response.
"""
return self._send_request('GET', url, query_string_args)
def send_head_request(self, url, query_string_args=None):
"""
Send a HEAD request
:param str url: The URL or URL path to the endpoint
:param dict query_string_args: The query string arguments
"""
return self._send_request('HEAD', url, query_string_args)
def send_post_request(self, url, body_deserialization=None):
"""
Send a POST request
:param str url: The URL or URL path to the endpoint
:param dict body_deserialization: The request's body message \
deserialized
:return: Decoded version of the ``JSON`` the remote put in \
the body of the response.
"""
return self._send_request(
'POST',
url,
body_deserialization=body_deserialization,
)
def send_put_request(self, url, body_deserialization):
"""
Send a PUT request
:param str url: The URL or URL path to the endpoint
:param body_deserialization: The request's body message deserialized
:return: Decoded version of the ``JSON`` the remote put in \
the body of the response.
"""
return self._send_request(
'PUT',
url,
body_deserialization=body_deserialization,
)
def send_delete_request(self, url):
"""
Send a DELETE request
:param str url: The URL or URL path to the endpoint
:return: Decoded version of the ``JSON`` the remote put in \
the body of the response.
"""
return self._send_request('DELETE', url)
def _send_request(
self,
method,
url,
query_string_args=None,
body_deserialization=None,
):
if url.startswith(self._api_url):
url = url
else:
url = self._api_url + url
query_string_args = query_string_args or {}
request_headers = \
{'content-type': 'application/json'} if body_deserialization else {}
if body_deserialization:
request_body_serialization = json_serialize(body_deserialization)
else:
request_body_serialization = None
response = self._session.request(
method,
url,
params=query_string_args,
auth=self._authentication_handler,
data=request_body_serialization,
headers=request_headers,
timeout=self._timeout,
)
self._require_successful_response(response)
self._require_deserializable_response_body(response)
return response
@staticmethod
def _require_successful_response(response):
if 400 <= response.status_code < 500:
if response.status_code == HTTPStatus.UNAUTHORIZED:
exception_class = AuthenticationError
elif response.status_code == HTTPStatus.FORBIDDEN:
exception_class = AccessDeniedError
elif response.status_code == HTTPStatus.NOT_FOUND:
exception_class = NotFoundError
else:
exception_class = ClientError
raise exception_class()
elif 500 <= response.status_code < 600:
raise ServerError(response.reason, response.status_code)
@classmethod
def _require_deserializable_response_body(cls, response):
if response.status_code in (HTTPStatus.OK, HTTPStatus.NO_CONTENT):
if response.content:
cls._require_json_response(response)
else:
exception_message = \
'Unsupported response status {}'.format(response.status_code)
raise UnsupportedResponseError(exception_message)
@staticmethod
def _require_json_response(response):
content_type_header_value = response.headers.get('Content-Type')
if not content_type_header_value:
exception_message = 'Response does not specify a Content-Type'
raise UnsupportedResponseError(exception_message)
content_type = content_type_header_value.split(';')[0].lower()
if content_type != 'application/json':
exception_message = \
'Unsupported response content type {}'.format(content_type)
raise UnsupportedResponseError(exception_message)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self._session.close()