class HTTPRequest: def __init__(self): self.session = Session() def request_with_cookie(self,method,url,**kwargs): logger.info("正在以 {} 方式 请求 {},参数是 {} ".format(method,url,kwargs)) return self.session.request(method=method,url=url,**kwargs) def upload(self, url, token, account_id, use_type, file_name="1.jpg", **kwargs): file_path = os.path.join(DOCS_DIR, file_name) # 因为后端采用的是 multipart/form-data方式,其中有 内容分隔符(boundary), 原生request不支持此方式 file = {"file": ("1.jpg", open(file_path, "rb"), "multipart/form-data"), "use_type": use_type} #file = {"upload": ("1.jpg", open(file_path, "rb"), "multipart/form-data"), "type": use_type} file = MultipartEncoder(fields=file) #headers = {"Content-Type": file.content_type, "token": token} #print(headers) headers = {"Content-Type": file.content_type, "token": token, "account_id": account_id} response = self.session.request("post", url=url, headers=headers, data=file, **kwargs) print(file)# 因为转化过了 所以不能用files logger.info("正在请求上传文件 URL:{}, file_name:{}".format(url, file_name)) return response #用于在测试类结束的时候,关闭seesion def session_close(self): self.session.close()
class Spider: def __init__(self, headers=None): self.headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36", } if headers: self.headers.update(headers) self.session = Session() self.parmas = None def load_parmas(self, kwargs: dict): self.parmas = kwargs def request(self, method="GET", url=None): if self.parmas: result = self.session.request(method, url, **self.parmas) else: result = self.session.request(method, url) return result def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.seesion.close() del self
class MeiPai: def __init__(self): self.session = Session() self.session.headers = headers def search(self, query, search_type='mv', page=1): query = query.strip() # topic if query.startswith('#'): topic = query.strip('#') return self.get_videos_by_topic(topic) cache_exists, result = check_cache_and_return_result(query=query, search_type=search_type, page=page) if cache_exists: return result url = 'http://www.meipai.com/search/{search_type}?'.format(search_type=search_type) + \ urlencode({'q': query, 'page': page}) resp = self.session.request('GET', url) html = BeautifulSoup(resp.content, 'html.parser') if search_type == 'mv': video_links = [div.attrs['data-video'].strip() for div in html.find_all(class_='content-l-video')] # associated_words = self.word_association(query) # print("你是否还想搜索:" + ",".join(associated_words)) result = video_links elif search_type == 'topic': result = [div.text.strip().strip('#') for div in html.find_all(class_='tcard-name')] else: result = [] cache_search_result(query, search_type, page, result) return result def get_videos_by_topic(self, topic_name): """ get top videos by topic :param topic_name: :return: """ topic = Topic(topic_name) topic_id = topic.topic_id url = "http://www.meipai.com/topics/hot_timeline?page=1&count=24&tid={topic_id}".format(topic_id=topic_id) resp = self.session.request('GET', url) result = json.loads(resp.text) return [media['video'] for media in result['medias']] # get associated words def word_association(self, word): url = 'http://www.meipai.com/search/word_assoc?' + urlencode({'q': word}) resp = self.session.request('GET', url) return json.loads(resp.text)
def _inner_call(*args, **kwargs): """ Wraps Session.request """ response = Session.request(*args, **kwargs) logger.debug('Request headers: %s', response.request.headers) logger.debug('Response headers: %s', response.headers) self._check_status_code(response) return response
def download_image(url, **kwargs): """下载验证码图片 :param url: 验证码网址 :param kwargs: requests的请求参数一致 :return: image对象 or None """ response = None times = 0 while times < 10: try: session = Session() response = session.request(method='GET', url=url, **kwargs) break except: times += 1 print 'Timeout, I will try to request again……' image = None if response != None: file = BytesIO(response.content) image = Image.open(file) return image
def request(session: Session, method, url, *args, **kwargs): resp = session.request(method, url, *args, **kwargs) # Check if Cloudflare anti-bot is on if (resp.status_code == 503 and resp.headers.get("Server") == "cloudflare-nginx" and b"jschl_vc" in resp.content and b"jschl_answer" in resp.content): return solve_cf_challenge(session, resp, **kwargs) # Otherwise, no Cloudflare anti-bot detected return resp
def load_loginpage(session: Session, form_meta: Dict[str, str]) -> Dict[str, str]: """ 载入登录页面 :return: * request_verification_token - 表单里的隐藏Token * captcha_image_url - 验证码的URL,地址相对于根路径 :raise RuntimeError 加载登录页面失败 """ url = "https://row1.vfsglobal.com/GlobalAppointment/" payload = {} headers = { 'Connection': 'keep-alive', 'Pragma': 'no-cache', 'Cache-Control': 'no-cache', 'Upgrade-Insecure-Requests': '1', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36', 'Sec-Fetch-User': '******', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3', 'Sec-Fetch-Site': 'none', 'Sec-Fetch-Mode': 'navigate', 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8' } def parse_page(page_html): soup = BeautifulSoup(page_html, 'html.parser') form = soup.find('form', {'id': 'ApplicantListForm'}) return { 'request_verification_token': form.find('input', {'name': '__RequestVerificationToken'})['value'], 'captcha_image_url': form.find('img', {'id': 'CaptchaImage'})['src'], } logger.info("尝试加载登录页面") response = session.request("GET", url, headers=headers, data=payload) if response.status_code == 200: return { **form_meta, **parse_page(response.text), } raise RuntimeError('加载登录页面失败', response)
def get_csv(sess: sessions.Session, endpoint: str, **parameters): """ get csv data from the Intrinio API :param sess: session :param endpoint: endpoint :param parameters: query parameters :return: csv result """ auth = os.getenv('INTRINIO_USERNAME'), os.getenv('INTRINIO_PASSWORD') url = '{}/{}'.format( 'https://api.intrinio.com', endpoint + ('' if endpoint.endswith('.csv') else '.csv')) if 'page_size' not in parameters: parameters['page_size'] = 10000 pages = list() for page_number in itertools.count(): parameters['page_number'] = page_number + 1 response = sess.request('GET', url, params=parameters, auth=auth, verify=True) if not response.ok: try: response.raise_for_status() except Exception as err: logging.getLogger(__name__).error(err) new_lines = response.content.decode('utf-8').count('\n') if new_lines == 1: break info, columns, page = response.content.decode('utf-8').split('\n', 2) if page_number == 0: info = {s.split(':')[0]: s.split(':')[1] for s in info.split(',')} total_pages = int(info['TOTAL_PAGES']) pages.append(columns.lower() + '\n') pages.append(page) if len(page) == 0 or page_number + 1 == total_pages: break return ''.join(pages) if len(pages) > 0 else None
def solve_captcha(session: Session, form_meta: Dict[str, str], howto: Callable[[PIL.Image.Image], str]) -> Dict[str, str]: """ 解决验证码 :param session: Session :param form_meta: 登录页面的结果 :param howto: 如何从验证码图片中解读出文字 :return: * captcha_detext - 验证码ID * captcha_inputtext - 验证码明文 :raise RuntimeError 获取验证码失败 """ captcha_detext = form_meta['captcha_image_url'].split('=')[1] url = "https://row1.vfsglobal.com/GlobalAppointment/DefaultCaptcha/Generate" params = {'t': captcha_detext} payload = {} headers = { 'Connection': 'keep-alive', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36', 'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8', 'Sec-Fetch-Site': 'same-origin', 'Sec-Fetch-Mode': 'no-cors', 'Referer': 'https://row1.vfsglobal.com/GlobalAppointment/', 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', } logger.info("尝试下载验证码图片") response = session.request("GET", url, params=params, headers=headers, data=payload) if response.status_code == 200: from PIL import Image from io import BytesIO image = Image.open(BytesIO(response.content)) captcha_inputtext = howto(image) logger.info("验证码: %s => %s", captcha_detext, captcha_inputtext) del form_meta['captcha_image_url'] return { **form_meta, 'captcha_detext': captcha_detext, 'captcha_inputtext': captcha_inputtext } raise RuntimeError('获取验证码失败', response)
def solve_cf_challenge(session: Session, resp, **original_kwargs): sleep(5) # Cloudflare requires a delay before solving the challenge body = resp.text parsed_url = urlparse(resp.url) domain = urlparse(resp.url).netloc submit_url = "%s://%s/cdn-cgi/l/chk_jschl" % (parsed_url.scheme, domain) cloudflare_kwargs = deepcopy(original_kwargs) params = cloudflare_kwargs.setdefault("params", {}) headers = cloudflare_kwargs.setdefault("headers", {}) headers["Referer"] = resp.url try: params["jschl_vc"] = re.search(r'name="jschl_vc" value="(\w+)"', body).group(1) params["pass"] = re.search(r'name="pass" value="(.+?)"', body).group(1) except Exception as e: # Something is wrong with the page. # This may indicate Cloudflare has changed their anti-bot # technique. If you see this and are running the latest version, # please open a GitHub issue so I can update the code accordingly. raise ValueError("Unable to parse Cloudflare anti-bots page: %s %s" % (e.message, BUG_REPORT)) # Solve the Javascript challenge params["jschl_answer"] = str(solve_challenge(session, body) + len(domain)) # Requests transforms any request into a GET after a redirect, # so the redirect has to be handled manually here to allow for # performing other types of requests even as the first request. method = resp.request.method cloudflare_kwargs["allow_redirects"] = False redirect = session.request(method, submit_url, **cloudflare_kwargs) return session.request(method, redirect.headers["Location"], **original_kwargs)
def load_appointment_page(session: Session, form_meta: Dict[str, str]): """ 加载/GlobalAppointment/Home/SelectVAC,读取表单隐藏字段 :param * vac_link :return: * request_verification_token """ url = BASE_URL + form_meta['vac_link'] payload = {} headers = { 'Connection': 'keep-alive', 'Upgrade-Insecure-Requests': '1', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36', 'Sec-Fetch-User': '******', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3', 'Sec-Fetch-Site': 'same-origin', 'Sec-Fetch-Mode': 'navigate', 'Referer': url, 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', } logger.info("加载预约页面") response = session.request("GET", url, headers=headers, data=payload) def parse_page(page_html: str) -> Dict[str, str]: """ 加载成功以后,获取提交表单所需的信息 Une fois le chargement réussi, obtenez les informations nécessaires pour envoyer le formulaire """ soup = BeautifulSoup(page_html, 'html.parser') form = soup.find('form', {'id': 'VisaApplicationForm'}) return { 'request_verification_token': form.find('input', {'name': '__RequestVerificationToken'})['value'], } if response.status_code == 200: return {**form_meta, **parse_page(response.text)} raise RuntimeError("预约页面加载失败", response)
class RequestsClient(HttpClient): """An implementation of HttpClient that uses Requests as its HTTP Client Attributes: timeout (int): The default timeout for all API requests. """ def __init__(self, timeout=60, cache=False, max_retries=None, verify=True): """The constructor. Args: timeout (float): The default global timeout(seconds). """ self.timeout = timeout self.session = Session() retries = Retry(total=max_retries) self.session.mount('https://', HTTPAdapter(max_retries=retries)) self.session.verify = verify def execute(self, request): """Execute a given HttpRequest to get a string response back Args: request (HttpRequest): The given HttpRequest to execute. Returns: HttpResponse: The response of the HttpRequest. """ response = self.session.request(HttpMethodEnum.to_string( request.http_method), request.query_url, headers=request.headers, params=request.query_parameters, data=request.parameters, timeout=self.timeout) return self.convert_response(response, request) def convert_response(self, response, http_request): """Converts the Response object of the HttpClient into an HttpResponse object. Args: response (dynamic): The original response object. http_request (HttpRequest): The original HttpRequest object. Returns: HttpResponse: The converted HttpResponse object. """ return HttpResponse(response.status_code, response.text, response.headers, http_request)
def _check_appointment(session: Session, form_meta: Dict[str, str], location_code, location_name) -> Optional[dict]: """ 检查空席位 :param * request_verification_token * vac_link """ url = "https://row1.vfsglobal.com/GlobalAppointment/Account/GetEarliestVisaSlotDate" payload = f"countryId=6&missionId=7&LocationId={location_code}&VisaCategoryId=2815" headers = { 'Connection': 'keep-alive', 'Accept': 'application/json, text/javascript, */*; q=0.01', 'Origin': 'https://row1.vfsglobal.com', 'X-Requested-With': 'XMLHttpRequest', '__RequestVerificationToken': form_meta['request_verification_token'], 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Sec-Fetch-Site': 'same-origin', 'Sec-Fetch-Mode': 'cors', 'Referer': BASE_URL + form_meta['vac_link'], 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', } estimate_login_time = \ LAST_LOGIN_TIME + timedelta(minutes=20) - datetime.now() logger.info("获取 %s 预约空位,预计离下次要求登录还有 %s", location_name, estimate_login_time) response = session.request("POST", url, headers=headers, data=payload) if response.text == '""': return None if response.text.startswith("{"): return json.loads(response.text) if response.text.startswith("<"): raise RuntimeError("获取失败,账号失效") logger.info("预约空位: %s", response.text) raise RuntimeError("获取失败", response.text)
def download_image(url, **kwargs): """下载验证码图片 :param url: 验证码网址 :param kwargs: requests的请求参数一致 :return: image对象 or None """ response = None times = 0 while times < 10: try: session = Session() response = session.request('GET', url=url, **kwargs) if response.status_code == 200: break except: times += 1 print('Download failed, try it again.') if response: return response.content else: return response
class PortalConnection(object): """ Connection to HubSpot :param authentication_key: This can be either an :class:`APIKey` or an \ :class:`OAuthKey` instance :param basestring change_source: The string passed to HubSpot as \ ``auditId`` in the query string """ _API_URL = 'https://api.hubapi.com' def __init__(self, authentication_key, change_source): super(PortalConnection, self).__init__() self._authentication_handler = \ _QueryStringAuthenticationHandler(authentication_key) self._change_source = change_source self._session = Session() self._session.headers['User-Agent'] = _USER_AGENT http_adapter = HTTPAdapter(max_retries=_HTTP_CONNECTION_MAX_RETRIES) self._session.mount('', http_adapter) def send_get_request(self, url_path, query_string_args=None): """ Send a GET request to HubSpot :param basestring url_path: The URL path to the endpoint :param dict query_string_args: The query string arguments :return: Decoded version of the ``JSON`` that HubSpot put in \ the body of the response. """ return self._send_request('GET', url_path, query_string_args) def send_post_request(self, url_path, body_deserialization): """ Send a POST request to HubSpot :param basestring url_path: The URL path to the endpoint :param dict body_deserialization: The request's body message \ deserialized :return: Decoded version of the ``JSON`` that HubSpot put in \ the body of the response. """ return self._send_request( 'POST', url_path, body_deserialization=body_deserialization, ) def send_put_request(self, url_path, body_deserialization): """ Send a PUT request to HubSpot :param basestring url_path: The URL path to the endpoint :param body_deserialization: The request's body message deserialized :return: Decoded version of the ``JSON`` that HubSpot put in \ the body of the response. """ return self._send_request( 'PUT', url_path, body_deserialization=body_deserialization, ) def send_delete_request(self, url_path): """ Send a DELETE request to HubSpot :param basestring url_path: The URL path to the endpoint :return: Decoded version of the ``JSON`` that HubSpot put in \ the body of the response. """ return self._send_request('DELETE', url_path) def _send_request( self, method, url_path, query_string_args=None, body_deserialization=None, ): url = self._API_URL + url_path query_string_args = query_string_args or {} query_string_args = dict(query_string_args, auditId=self._change_source) request_headers = \ {'content-type': 'application/json'} if body_deserialization else {} if body_deserialization: request_body_serialization = json_serialize(body_deserialization) else: request_body_serialization = None response = self._session.request( method, url, params=query_string_args, auth=self._authentication_handler, data=request_body_serialization, headers=request_headers, ) response_body_deserialization = \ self._deserialize_response_body(response) return response_body_deserialization @classmethod def _deserialize_response_body(cls, response): cls._require_successful_response(response) cls._require_json_response(response) if response.status_code == HTTP_STATUS_OK: response_body_deserialization = response.json() elif response.status_code in _HTTP_STATUS_CODES_WITH_EMPTY_BODIES: response_body_deserialization = None else: exception_message = \ 'Unsupported response status {}'.format(response.status_code) raise HubspotUnsupportedResponseError(exception_message) return response_body_deserialization @staticmethod def _require_successful_response(response): if 400 <= response.status_code < 500: response_data = response.json() error_data = _HUBSPOT_ERROR_RESPONSE_SCHEMA(response_data) if response.status_code == HTTP_STATUS_UNAUTHORIZED: exception_class = HubspotAuthenticationError else: exception_class = HubspotClientError raise exception_class( error_data['message'], error_data['requestId'], ) elif 500 <= response.status_code < 600: raise HubspotServerError(response.reason, response.status_code) @staticmethod def _require_json_response(response): content_type_header_value = response.headers.get('Content-Type') if not content_type_header_value: exception_message = 'Response does not specify a Content-Type' raise HubspotUnsupportedResponseError(exception_message) content_type = content_type_header_value.split(';')[0].lower() if content_type != 'application/json': exception_message = \ 'Unsupported response content type {}'.format(content_type) raise HubspotUnsupportedResponseError(exception_message) def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): self._session.close()
class TestClient(object): def __init__(self): self._session = Session() self._session.hooks["response"] = [ _hook_check_status_code, _hook_check_errcode, ] def _request(self, method, path, **kwargs): return self._session.request(method, BASE_URL + path, **kwargs) def _get(self, path, params={}, **kwargs): return self._request('GET', path, params=params, **kwargs) def _post(self, path, data={}, **kwargs): return self._request('POST', path, data=data, **kwargs) def create_admin(self, username, password, type, internal_token): return self._post("/_internal/create_admin", { "username": username, "password": password, "type": type, }, headers={"Authorization": internal_token}) def create_activity(self, date, begin, end, site, root_token): return self._post("/activity/create", { "date": date, "begin": begin, "end": end, "site": site, }, headers={"Authorization": root_token}) def create_order(self, model, description, repairType, period, email, user_token): return self._post("/order/create", { "model": model, "description": description, "repairType": repairType, "period": period, "email": email, }, headers={"Authorization": user_token}) def get_existed_order(self, user_token): return self._get("/order/get_existed", headers={ "Authorization": user_token, }) def get_register_token(self, admin_token): return self._get("/order/get_register_token", headers={ "Authorization": admin_token, }) def register_order(self, oid, token, user_token): return self._post("/order/register", { "oid": oid, "token": token, }, headers={"Authorization": user_token})
def _get_id(self): session = Session() resp = session.request('GET', 'http://www.meipai.com/topic/' + quote_plus(self.name), headers=headers) html = BeautifulSoup(resp.content, 'html.parser') return html.find(class_='topic-txt').attrs['data-id']
class PortalConnection(object): """ Connection to HubSpot :param authentication_key: This can be either an :class:`APIKey` or an \ :class:`OAuthKey` instance :param basestring change_source: The string passed to HubSpot as \ ``auditId`` in the query string """ _API_URL = 'https://api.hubapi.com' def __init__(self, authentication_key, change_source): super(PortalConnection, self).__init__() self._authentication_handler = \ _QueryStringAuthenticationHandler(authentication_key) self._change_source = change_source self._session = Session() self._session.headers['User-Agent'] = _USER_AGENT http_adapter = HTTPAdapter(max_retries=_HTTP_CONNECTION_MAX_RETRIES) self._session.mount('', http_adapter) def send_get_request(self, url_path, query_string_args=None): """ Send a GET request to HubSpot :param basestring url_path: The URL path to the endpoint :param dict query_string_args: The query string arguments :return: Decoded version of the ``JSON`` that HubSpot put in \ the body of the response. """ return self._send_request('GET', url_path, query_string_args) def send_post_request(self, url_path, body_deserialization): """ Send a POST request to HubSpot :param basestring url_path: The URL path to the endpoint :param dict body_deserialization: The request's body message \ deserialized :return: Decoded version of the ``JSON`` that HubSpot put in \ the body of the response. """ return self._send_request( 'POST', url_path, body_deserialization=body_deserialization, ) def send_put_request(self, url_path, body_deserialization): """ Send a PUT request to HubSpot :param basestring url_path: The URL path to the endpoint :param body_deserialization: The request's body message deserialized :return: Decoded version of the ``JSON`` that HubSpot put in \ the body of the response. """ return self._send_request( 'PUT', url_path, body_deserialization=body_deserialization, ) def send_delete_request(self, url_path): """ Send a DELETE request to HubSpot :param basestring url_path: The URL path to the endpoint :return: Decoded version of the ``JSON`` that HubSpot put in \ the body of the response. """ return self._send_request('DELETE', url_path) def _send_request( self, method, url_path, query_string_args=None, body_deserialization=None, ): url = self._API_URL + url_path query_string_args = query_string_args or {} query_string_args = dict(query_string_args, auditId=self._change_source) request_headers = \ {'content-type': 'application/json'} if body_deserialization else {} if body_deserialization: request_body_serialization = json_serialize(body_deserialization) else: request_body_serialization = None response = self._session.request( method, url, params=query_string_args, auth=self._authentication_handler, data=request_body_serialization, headers=request_headers, ) response_body_deserialization = \ self._deserialize_response_body(response) return response_body_deserialization @classmethod def _deserialize_response_body(cls, response): cls._require_successful_response(response) if response.status_code == HTTP_STATUS_OK: cls._require_json_response(response) response_body_deserialization = response.json() or None elif response.status_code in _HTTP_STATUS_CODES_WITH_EMPTY_BODIES: response_body_deserialization = None else: exception_message = \ 'Unsupported response status {}'.format(response.status_code) raise HubspotUnsupportedResponseError(exception_message) return response_body_deserialization @staticmethod def _require_successful_response(response): if 400 <= response.status_code < 500: response_data = response.json() error_data = _HUBSPOT_ERROR_RESPONSE_SCHEMA(response_data) if response.status_code == HTTP_STATUS_UNAUTHORIZED: exception_class = HubspotAuthenticationError else: exception_class = HubspotClientError raise exception_class( error_data['message'], error_data['requestId'], ) elif 500 <= response.status_code < 600: raise HubspotServerError(response.reason, response.status_code) @staticmethod def _require_json_response(response): content_type_header_value = response.headers.get('Content-Type') if not content_type_header_value: exception_message = 'Response does not specify a Content-Type' raise HubspotUnsupportedResponseError(exception_message) content_type = content_type_header_value.split(';')[0].lower() if content_type != 'application/json': exception_message = \ 'Unsupported response content type {}'.format(content_type) raise HubspotUnsupportedResponseError(exception_message) def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): self._session.close()
class Connection(object): """ Permite crear/gestionar la conexion con Hubspot. Parametros: API-KEY """ HS_API_URL = "http://api.hubapi.com" def __init__(self, apikey_value): super(Connection, self).__init__() self.APIKEY_VALUE = apikey_value self.APIKEY = "?hapikey=" + self.APIKEY_VALUE self.session = Session() self.session.headers['User-Agent'] = 'Frank Collaboration' def send_request(self, method, xurl, query_string_args=None, parameters=None,): """ Metodo generico de comunicacion con la api de hubspot Parametros: method: metodo de comunicacion, xurl: url pacial de la API, parameters: Diccionario con los parametros para la api """ url = self.HS_API_URL + xurl + self.APIKEY query_string_args = query_string_args or {} query_string_args = dict(query_string_args) request_headers = \ {'content-type': 'application/json'} if parameters else {} if parameters: request_parameters = json.dumps(parameters) else: request_parameters = None response = self.session.request( method, url, params=query_string_args, data=request_parameters, headers=request_headers, ) return response def send_get_request(self, url_path, query_string_args=None): """ Metodo GET de comunicacion con la api de hubspot Parametros: url_path: url de la funcion de la api a implementar """ return self.send_request('GET', url_path, query_string_args) def send_post_request(self, url_path, parameters): """ Metodo POST de comunicacion con la api de hubspot Parametros: url_path: url de la funcion de la api a implementar parameters: Parametros requeridos por la funcion """ return self.send_request('POST', url_path, parameters=parameters) def send_put_request(self, url_path, parameters): """ Metodo PUT de comunicacion con la api de hubspot Parametros: url_path: url de la funcion de la api a implementar parameters: Parametros requeridos por la funcion """ return self.send_request('PUT', url_path, parameters=parameters)
class MWS(object): """ Base Amazon API class """ # This is used to post/get to the different uris used by amazon per api # ie. /Orders/2011-01-01 # All subclasses must define their own URI only if needed URI = "/" # The API version varies in most amazon APIs VERSION = "2009-01-01" # There seem to be some xml namespace issues. therefore every api subclass # is recommended to define its namespace, so that it can be referenced # like so AmazonAPISubclass.NS. # For more information see http://stackoverflow.com/a/8719461/389453 NS = '' # Some APIs are available only to either a "Merchant" or "Seller" # the type of account needs to be sent in every call to the amazon MWS. # This constant defines the exact name of the parameter Amazon expects # for the specific API being used. # All subclasses need to define this if they require another account type # like "Merchant" in which case you define it like so. # ACCOUNT_TYPE = "Merchant" # Which is the name of the parameter for that specific account type. ACCOUNT_TYPE = "SellerId" def __init__(self, access_key, secret_key, account_id, domain='https://mws.amazonservices.com', uri="", version=""): self.access_key = access_key self.secret_key = secret_key self.account_id = account_id self.domain = domain self.uri = uri or self.URI self.version = version or self.VERSION self.session = Session() bucket_key = getattr(settings, 'RUNSCOPE_BUCKET_KEY', None) if bucket_key: logger.info("Redirecting API calls for MWS to runscope") self.configure_runscope(bucket_key) def configure_runscope(self, bucket_key): """ Configure all connections to be proxied through runscope for easier debugging and logging of all requests and responses. *bucket_key* is API for the bucket you want to use for all the request. Check Runscope for more details on that. """ try: from requests_runscope import RunscopeAdapter except ImportError: logger.error( "Could not import runscope adapter. Is requests-runscope " "installed? Try running pip install requests-runscope.") else: logger.info('Mounting runscope proxy adapter for bucket {}'.format( bucket_key)) self.session.mount('https://', RunscopeAdapter(bucket_key)) self.session.mount('http://', RunscopeAdapter(bucket_key)) def _get_quote_params(self, params): quoted_params = [] for key in sorted(params): value = urllib.quote(unicode(params[key]).encode('utf-8'), safe='-_.~') quoted_params.append("{}={}".format(key, value)) return '&'.join(quoted_params) def make_request(self, extra_data, method="GET", **kwargs): """ Make request to Amazon MWS API with these parameters """ # Remove all keys with an empty value because # Amazon's MWS does not allow such a thing. extra_data = remove_empty(extra_data) params = { 'AWSAccessKeyId': self.access_key, self.ACCOUNT_TYPE: self.account_id, 'SignatureVersion': '2', 'Timestamp': self.get_timestamp(), 'Version': self.version, 'SignatureMethod': 'HmacSHA256', } params.update(extra_data) logger.debug("Request Parameters: {}".format(params)) request_description = self._get_quote_params(params) signature = self.calc_signature(method, request_description) logger.debug('Domain: {} URI: {}'.format(self.domain, self.uri)) url = '%s%s?%s&Signature=%s' % (self.domain, self.uri, request_description, urllib.quote(signature)) headers = {'User-Agent': 'python-amazon-mws/0.0.1 (Language=Python)'} headers.update(kwargs.get('extra_headers', {})) try: # Some might wonder as to why i don't pass the params dict as the # params argument to request. My answer is, here i have to get the # url parsed string of params in order to sign it, so if i pass the # params dict as params to request, request will repeat that step # because it will need to convert the dict to a url parsed string, # so why do it twice if i can just pass the full url :). response = self.session.request(method, url, data=kwargs.get('body', ''), headers=headers) response.raise_for_status() # When retrieving data from the response object, be aware that # response.content returns the content in bytes while response.text # calls response.content and converts it to unicode. data = response.content # I do not check the headers to decide which content structure to # server simply because sometimes Amazon's MWS API returns XML # error responses with "text/plain" as the Content-Type. action = extra_data.get('Action') if not action.endswith('Result'): action = "{}Result".format(action) try: parsed_response = DictWrapper(data, action) except XMLError: parsed_response = DataWrapper(data, response.headers) except HTTPError, e: error = MWSError(unicode(e)) error.response = e.response error.url = url logger.error("Received {} with message: {}".format( unicode(e), e.response.content)) raise error # Store the response object in the parsed_response for quick access parsed_response.response = response logger.debug("Received response: {}".format(response.content)) return parsed_response
class MWS(object): """ Base Amazon API class """ # This is used to post/get to the different uris used by amazon per api # ie. /Orders/2011-01-01 # All subclasses must define their own URI only if needed URI = "/" # The API version varies in most amazon APIs VERSION = "2009-01-01" # There seem to be some xml namespace issues. therefore every api subclass # is recommended to define its namespace, so that it can be referenced # like so AmazonAPISubclass.NS. # For more information see http://stackoverflow.com/a/8719461/389453 NS = '' # Some APIs are available only to either a "Merchant" or "Seller" # the type of account needs to be sent in every call to the amazon MWS. # This constant defines the exact name of the parameter Amazon expects # for the specific API being used. # All subclasses need to define this if they require another account type # like "Merchant" in which case you define it like so. # ACCOUNT_TYPE = "Merchant" # Which is the name of the parameter for that specific account type. ACCOUNT_TYPE = "SellerId" def __init__(self, access_key, secret_key, account_id, domain='https://mws.amazonservices.com', uri="", version=""): self.access_key = access_key self.secret_key = secret_key self.account_id = account_id self.domain = domain self.uri = uri or self.URI self.version = version or self.VERSION self.session = Session() bucket_key = getattr(settings, 'RUNSCOPE_BUCKET_KEY', None) if bucket_key: logger.info("Redirecting API calls for MWS to runscope") self.configure_runscope(bucket_key) def configure_runscope(self, bucket_key): """ Configure all connections to be proxied through runscope for easier debugging and logging of all requests and responses. *bucket_key* is API for the bucket you want to use for all the request. Check Runscope for more details on that. """ try: from requests_runscope import RunscopeAdapter except ImportError: logger.error( "Could not import runscope adapter. Is requests-runscope " "installed? Try running pip install requests-runscope." ) else: logger.info( 'Mounting runscope proxy adapter for bucket {}'.format( bucket_key ) ) self.session.mount('https://', RunscopeAdapter(bucket_key)) self.session.mount('http://', RunscopeAdapter(bucket_key)) def _get_quote_params(self, params): quoted_params = [] for key in sorted(params): value = urllib.quote( unicode(params[key]).encode('utf-8'), safe='-_.~' ) quoted_params.append("{}={}".format(key, value)) return '&'.join(quoted_params) def make_request(self, extra_data, method="GET", **kwargs): """ Make request to Amazon MWS API with these parameters """ # Remove all keys with an empty value because # Amazon's MWS does not allow such a thing. extra_data = remove_empty(extra_data) params = { 'AWSAccessKeyId': self.access_key, self.ACCOUNT_TYPE: self.account_id, 'SignatureVersion': '2', 'Timestamp': self.get_timestamp(), 'Version': self.version, 'SignatureMethod': 'HmacSHA256', } params.update(extra_data) logger.debug("Request Parameters: {}".format(params)) request_description = self._get_quote_params(params) signature = self.calc_signature(method, request_description) logger.debug('Domain: {} URI: {}'.format(self.domain, self.uri)) url = '%s%s?%s&Signature=%s' % (self.domain, self.uri, request_description, urllib.quote(signature)) headers = {'User-Agent': 'python-amazon-mws/0.0.1 (Language=Python)'} headers.update(kwargs.get('extra_headers', {})) try: # Some might wonder as to why i don't pass the params dict as the # params argument to request. My answer is, here i have to get the # url parsed string of params in order to sign it, so if i pass the # params dict as params to request, request will repeat that step # because it will need to convert the dict to a url parsed string, # so why do it twice if i can just pass the full url :). response = self.session.request( method, url, data=kwargs.get('body', ''), headers=headers) response.raise_for_status() # When retrieving data from the response object, be aware that # response.content returns the content in bytes while response.text # calls response.content and converts it to unicode. data = response.content # I do not check the headers to decide which content structure to # server simply because sometimes Amazon's MWS API returns XML # error responses with "text/plain" as the Content-Type. action = extra_data.get('Action') if not action.endswith('Result'): action = "{}Result".format(action) try: parsed_response = DictWrapper(data, action) except XMLError: parsed_response = DataWrapper(data, response.headers) except HTTPError, e: error = MWSError(unicode(e)) error.response = e.response error.url = url logger.error( "Received {} with message: {}".format( unicode(e), e.response.content ) ) raise error # Store the response object in the parsed_response for quick access parsed_response.response = response logger.debug("Received response: {}".format(response.content)) return parsed_response
def simulate_login(session: Session, form_meta: Dict[str, str]) -> Dict[str, str]: """ 模拟登录 :param: * request_verification_token * captcha_detext * captcha_inputtext * username * password :return: * vac_link - 访问/GlobalAppointment/Home/SelectVAC的链接 :raise: * RuntimeError 登录失败 """ url = "https://row1.vfsglobal.com/GlobalAppointment/" payload = { '__RequestVerificationToken': form_meta['request_verification_token'], 'Mission': '', 'Country': '', 'Center': '', 'IsGoogleCaptchaEnabled': 'False', 'reCaptchaURL': 'https://www.google.com/recaptcha/api/siteverify?secret={0}&response={1}', 'reCaptchaPublicKey': '6Ld-Kg8UAAAAAK6U2Ur94LX8-Agew_jk1pQ3meJ1', 'EmailId': form_meta['username'], 'Password': form_meta['password'], 'CaptchaDeText': form_meta['captcha_detext'], 'CaptchaInputText': form_meta['captcha_inputtext'], } headers = { 'Connection': 'keep-alive', 'Cache-Control': 'max-age=0', 'Origin': 'https://row1.vfsglobal.com', 'Upgrade-Insecure-Requests': '1', 'Content-Type': 'application/x-www-form-urlencoded', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36', 'Sec-Fetch-User': '******', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3', 'Sec-Fetch-Site': 'same-origin', 'Sec-Fetch-Mode': 'navigate', 'Referer': 'https://row1.vfsglobal.com/GlobalAppointment/', 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', } def parse_page(page_html: str) -> Dict[str, str]: """ 登录成功以后,获取预约页面的访问链接 """ soup = BeautifulSoup(page_html, 'html.parser') def check_account_banned(): banner = soup.find('div', {'class': 'validation-summary-errors'}) if banner is None: return raise RuntimeError('登录失败', banner.text.strip()) check_account_banned() def is_vac_link(tag: Tag): return tag.has_attr('href') and \ tag['href'].startswith('/GlobalAppointment/Home/SelectVAC') return {'vac_link': soup.find(is_vac_link)['href']} logger.info("尝试用 %s 进行登录", form_meta['username']) response = session.request("POST", url, headers=headers, data=payload) if response.status_code == 200: # follow redirection del form_meta['request_verification_token'] del form_meta['captcha_detext'] del form_meta['captcha_inputtext'] global LAST_LOGIN_TIME LAST_LOGIN_TIME = datetime.now() return {**form_meta, **parse_page(response.text)} raise RuntimeError('登录失败', response)
class Connection: _API_URL = 'https://www.2degreesnetwork.com/api' def __init__(self, auth, timeout=None, api_url=None): super(Connection, self).__init__() self._api_url = api_url or self._API_URL self._authentication_handler = auth self._session = Session() self._session.headers['User-Agent'] = _USER_AGENT self._timeout = timeout http_adapter = HTTPAdapter(max_retries=_HTTP_CONNECTION_MAX_RETRIES) self._session.mount('', http_adapter) def send_get_request(self, url, query_string_args=None): """ Send a GET request :param str url: The URL or URL path to the endpoint :param dict query_string_args: The query string arguments :return: Decoded version of the ``JSON`` the remote put in \ the body of the response. """ return self._send_request('GET', url, query_string_args) def send_head_request(self, url, query_string_args=None): """ Send a HEAD request :param str url: The URL or URL path to the endpoint :param dict query_string_args: The query string arguments """ return self._send_request('HEAD', url, query_string_args) def send_post_request(self, url, body_deserialization=None): """ Send a POST request :param str url: The URL or URL path to the endpoint :param dict body_deserialization: The request's body message \ deserialized :return: Decoded version of the ``JSON`` the remote put in \ the body of the response. """ return self._send_request( 'POST', url, body_deserialization=body_deserialization, ) def send_put_request(self, url, body_deserialization): """ Send a PUT request :param str url: The URL or URL path to the endpoint :param body_deserialization: The request's body message deserialized :return: Decoded version of the ``JSON`` the remote put in \ the body of the response. """ return self._send_request( 'PUT', url, body_deserialization=body_deserialization, ) def send_delete_request(self, url): """ Send a DELETE request :param str url: The URL or URL path to the endpoint :return: Decoded version of the ``JSON`` the remote put in \ the body of the response. """ return self._send_request('DELETE', url) def _send_request( self, method, url, query_string_args=None, body_deserialization=None, ): if url.startswith(self._api_url): url = url else: url = self._api_url + url query_string_args = query_string_args or {} request_headers = \ {'content-type': 'application/json'} if body_deserialization else {} if body_deserialization: request_body_serialization = json_serialize(body_deserialization) else: request_body_serialization = None response = self._session.request( method, url, params=query_string_args, auth=self._authentication_handler, data=request_body_serialization, headers=request_headers, timeout=self._timeout, ) self._require_successful_response(response) self._require_deserializable_response_body(response) return response @staticmethod def _require_successful_response(response): if 400 <= response.status_code < 500: if response.status_code == HTTPStatus.UNAUTHORIZED: exception_class = AuthenticationError elif response.status_code == HTTPStatus.FORBIDDEN: exception_class = AccessDeniedError elif response.status_code == HTTPStatus.NOT_FOUND: exception_class = NotFoundError else: exception_class = ClientError raise exception_class() elif 500 <= response.status_code < 600: raise ServerError(response.reason, response.status_code) @classmethod def _require_deserializable_response_body(cls, response): if response.status_code in (HTTPStatus.OK, HTTPStatus.NO_CONTENT): if response.content: cls._require_json_response(response) else: exception_message = \ 'Unsupported response status {}'.format(response.status_code) raise UnsupportedResponseError(exception_message) @staticmethod def _require_json_response(response): content_type_header_value = response.headers.get('Content-Type') if not content_type_header_value: exception_message = 'Response does not specify a Content-Type' raise UnsupportedResponseError(exception_message) content_type = content_type_header_value.split(';')[0].lower() if content_type != 'application/json': exception_message = \ 'Unsupported response content type {}'.format(content_type) raise UnsupportedResponseError(exception_message) def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): self._session.close()