class HTTPRequest:
def __init__(self):
self.session = Session()
def request_with_cookie(self,method,url,**kwargs):
logger.info("正在以 {} 方式 请求 {}，参数是 {} ".format(method,url,kwargs))
return self.session.request(method=method,url=url,**kwargs)
def upload(self, url, token, account_id, use_type, file_name="1.jpg", **kwargs):
file_path = os.path.join(DOCS_DIR, file_name)
# 因为后端采用的是 multipart/form-data方式,其中有 内容分隔符(boundary), 原生request不支持此方式
file = {"file": ("1.jpg", open(file_path, "rb"), "multipart/form-data"), "use_type": use_type}
#file = {"upload": ("1.jpg", open(file_path, "rb"), "multipart/form-data"), "type": use_type}
file = MultipartEncoder(fields=file)
#headers = {"Content-Type": file.content_type, "token": token}
#print(headers)
headers = {"Content-Type": file.content_type, "token": token, "account_id": account_id}
response = self.session.request("post", url=url, headers=headers, data=file, **kwargs)
print(file)# 因为转化过了 所以不能用files
logger.info("正在请求上传文件 URL:{}, file_name:{}".format(url, file_name))
return response
#用于在测试类结束的时候，关闭seesion
def session_close(self):
self.session.close()
class HttpRequest:
def __init__(self):
self.session = Session()
def __del__(self):
self.session.close()
def request(self, method, url, data=None, params=None, json=None, headers=None, cookies=None, timeout=None):
if method.lower() == 'get':
my_log.info(f'Sending {method}:{url} {params}')
res = self.session.get(url=url, params=params, headers=headers, cookies=cookies, timeout=timeout)
elif method.lower() == 'post':
if json:
my_log.info(f'Sending {method}:{url} {json}')
res = self.session.post(url=url, json=json, headers=headers, cookies=cookies, timeout=timeout)
else:
my_log.info(f'Sending {method}:{url} {data}')
res = self.session.post(url=url, data=data, headers=headers, cookies=cookies, timeout=timeout)
else:
res = None
if res.status_code == 404:
my_log.error(f'404 not found!')
raise RuntimeError
return res.text
def close(self):
self.session.close()
class HTTPRequest2(object):
"""记录cookies信息给下一次请求使用"""
def __init__(self):
# 创建session对象
self.session = Session()
def request(self, method, url,
params=None, data=None,
headers=None, cookies=None, json=None):
method = method.lower()
if method == "post":
# 判断是否使用json来传参（适用于接口项目有使用json传参）
if json:
logging.info("正在发送请求，请求地址：{}， 请求参数：{}".format(url, json))
return self.session.post(url=url, json=json, headers=headers, cookies=cookies)
else:
logging.info("正在发送请求，请求地址：{}， 请求参数：{}".format(url, data))
return self.session.post(url=url, data=data, headers=headers, cookies=cookies)
elif method == "get":
logging.info("正在发送请求，请求地址：{}， 请求参数：{}".format(url, params))
return self.session.get(url=url, params=params, headers=headers, cookies=cookies)
def close(self):
self.session.close()
def is_session(s: Session, member_id: int) -> Session:
url = 'http://canvas.tongji.edu.cn/api/v1/planner/items?per_page=50&start_date=' + \
datetime.now().strftime('%Y-%m-%d')
r = s.get(url)
data = json.loads(r.text.replace('while(1);', ''))
if 'error' in data or r.status_code == 401:
print("检测到canvas登录状态丢失，现尝试重新登录canvas")
s.close()
s = createlink(member_id)
return s
class HttpClient:
""" 使用requests库封装的高可靠 Http client
:param max_connect_retries: The maximum number of retries each connection
should attempt.Note, this applies only to failed DNS lookups, socket
connections and connection timeouts, never to requests where data has
made it to the server.
:param max_request_tries: The maximum times of tries each request
should attempt.
"""
def __init__(self, max_connect_retries=0, max_request_tries=0):
self.timeout = DEFAULT_TIMOUT
self.max_connect_retries = (max_connect_retries
or DEFAULT_CONNECT_RETRIES)
self.max_request_tries = max_request_tries or DEFAULT_REQUEST_TRIES
self.session = Session()
retries = Retry(connect=2, read=2, status=2, redirect=2)
self.session.mount('https://', HTTPAdapter(max_retries=retries))
self.session.mount('http://', HTTPAdapter(max_retries=retries))
def get(self, url, content_type='json', max_request_times=0, timeout=0):
max_times = max_request_times or self.max_request_tries
has_request_times = 0
data = None
while has_request_times < max_times:
try:
res = self.session.get(url, timeout=timeout or self.timeout)
data = res.json() if content_type == 'json' else res.text
if not data:
has_request_times = has_request_times + 1
continue
else:
break
except requests.exceptions.ConnectionError as e:
print("socket连接错误或读取超时", e.__class__)
break
except Exception:
# raise
has_request_times = has_request_times + 1
continue
if not data:
print("尝试了{}次请求依然失败".format(has_request_times + 1))
else:
print("尝试了{}次请求成功".format(has_request_times + 1))
self.session.close()
return data
def post(self, url):
pass
class HTTPRequest(object):
def __init__(self):
# 创建session对象
self.session = Session()
# 记录cookies信息，给下一次请求使用
def request(self,
method,
url,
params=None,
data=None,
headers=None,
cookies=None,
json=None):
# 将方法名转化为小写
method = method.lower()
if method == "post":
# 判断是否使用json来传参，适用于接口项目使用json传参
if json:
logging.info("正在发送请求，请求地址：{}，请求参数：{}".format(url, json))
response = self.session.post(url=url,
json=json,
headers=headers,
cookies=cookies)
return response
else:
logging.info("正在发送请求，请求地址：{}，请求参数：{}".format(url, data))
response = self.session.post(url=url,
data=data,
headers=headers,
cookies=cookies)
return response
elif method == "get":
logging.info("正在发送请求，请求地址：{}，请求参数：{}".format(url, params))
response = self.session.get(url=url,
params=params,
headers=headers,
cookies=cookies)
return response
elif method == "put":
logging.info("正在发送请求，请求地址：{}，请求参数：{}".format(url, data))
response = self.session.put(url=url,
data=data,
headers=headers,
cookies=cookies)
return response
def close(self):
self.session.close()
def downloadEp(self, ep_id: int, path: str):
"下载一个章节"
if not os.path.exists(path):
os.mkdir(path)
_s = Session()
_list = self.getDownloadList(ep_id)
for ii in range(len(_list)):
try:
with open(os.path.join(path, f'{ii+1:0>2}.jpg'), 'wb') as f:
f.write(_s.get(_list[ii]).content)
except Exception as e:
print(f'在{path}下写入{ii+1:0>2}.jpg时异常', repr(e))
_s.close()
class MySession:
def __init__(self, cfg: DictConfig):
self.cfg = cfg
def __enter__(self) -> Session:
auth = get_latest(self.cfg, ['token_type', 'access_token'])
HEADER = {
'Authorization': f'{auth["token_type"]} {auth["access_token"]}'
}
self.session = Session()
self.session.headers.update(HEADER)
return self.session
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
self.session.close()
class BdXhsApi(object):
def __init__(self,
host=XHS_BASE_HOST,
prefix="fe_api/burdock/baidu/v2/",
scheme="https"):
self._host = host
self._prefix = prefix
self._base_url = "{}://{}/{}".format(scheme, host, prefix)
self._session = Session()
ui = random.randint(0, len(USER_AGENTS) - 1)
headers = {
"User-Agent": USER_AGENTS[ui],
"Host": self._host,
"Connection": "close",
"Content-Type": "application/json",
"Accept-Language": "zh-cn"
}
self._session.headers.update(headers)
self._session.keep_alive = False
def _get(self, route, sign_code, retry=1):
url = urljoin(self._base_url, route)
headers = {"X-Sign": "X{}".format(sign_code)}
try:
ret = self._session.get(url, headers=headers, timeout=(2, 5))
LOG.info("GET {} {}".format(ret.url, ret.status_code))
except (requests.exceptions.ConnectTimeout,
requests.exceptions.ConnectionError) as e:
LOG.error(e)
return None
error = ret.text
import pdb
pdb.set_trace()
if 200 <= ret.status_code < 300:
data = ret.json()
code = data.get("code")
if code == 0:
return data["data"]
if retry:
return self._get(route, sign_code, retry - 1)
raise Exception('Error url:{} -{}'.format(url, error))
def get_homefeed_categories(self):
route = "homefeed/categories"
sign_code = "144fc3e8f0039f6fdf80d42152805e2d"
result = self._get(route, sign_code=sign_code)
return result
def get_search_tending(self):
route = "search/trending"
sign_code = "1d2bce3df0a71e0b906c0ef74a7637ab"
result = self._get(route, sign_code=sign_code)
return result
def close(self):
return self._session.close()
def sxyprn_download(session: Session, logger: Logger):
"""
This does the downloads
:param session:
:param logger:
:return:
"""
url_parsed = urllib.parse.urlparse(ConfigUrl.url)
# noinspection PyProtectedMember
base_url = url_parsed._replace(path="", params="", query="",
fragment="").geturl()
num_pages = None
counter = 0
urls = UrlSet()
for url in url_generator(url=ConfigUrl.url):
logger.info(f"loading [{url}]")
response = session_get(session=session, url=url)
response.raise_for_status()
root = get_html_dom_content(response)
if num_pages is None:
num_pages = len(root.xpath("//div[contains(@class,'ctrl_el')]"))
if num_pages == 0:
num_pages = 1
if ConfigDebugUrls.save:
with tempfile.NamedTemporaryFile(delete=False) as f:
logger.info(f"writing file [{f.name}]")
f.write(get_element_as_bytes(root))
elements = root.xpath("//a[contains(@class,'js-pop')]")
for element in elements:
href = element.attrib["href"]
url = urllib.parse.urljoin(base_url, href)
no_fluff = urllib.parse.urlparse(url)._replace(
params="", query="", fragment="").geturl()
urls.append(no_fluff)
counter += 1
if counter == num_pages:
break
session.close()
logger.info(f"got total [{len(urls.urls_list)}] urls")
logger.info(f"got [{urls.appended_twice}] appended twice urls")
youtube_dl_download_urls(urls.urls_list)
class PortalConnection(object):
"""
Connection to HubSpot
:param authentication_key: This can be either an :class:`APIKey` or an \
:class:`OAuthKey` instance
:param basestring change_source: The string passed to HubSpot as \
``auditId`` in the query string
"""
_API_URL = 'https://api.hubapi.com'
def __init__(self, authentication_key, change_source):
super(PortalConnection, self).__init__()
self._authentication_handler = \
_QueryStringAuthenticationHandler(authentication_key)
self._change_source = change_source
self._session = Session()
self._session.headers['User-Agent'] = _USER_AGENT
http_adapter = HTTPAdapter(max_retries=_HTTP_CONNECTION_MAX_RETRIES)
self._session.mount('', http_adapter)
def send_get_request(self, url_path, query_string_args=None):
"""
Send a GET request to HubSpot
:param basestring url_path: The URL path to the endpoint
:param dict query_string_args: The query string arguments
:return: Decoded version of the ``JSON`` that HubSpot put in \
the body of the response.
"""
return self._send_request('GET', url_path, query_string_args)
def send_post_request(self, url_path, body_deserialization):
"""
Send a POST request to HubSpot
:param basestring url_path: The URL path to the endpoint
:param dict body_deserialization: The request's body message \
deserialized
:return: Decoded version of the ``JSON`` that HubSpot put in \
the body of the response.
"""
return self._send_request(
'POST',
url_path,
body_deserialization=body_deserialization,
)
def send_put_request(self, url_path, body_deserialization):
"""
Send a PUT request to HubSpot
:param basestring url_path: The URL path to the endpoint
:param body_deserialization: The request's body message deserialized
:return: Decoded version of the ``JSON`` that HubSpot put in \
the body of the response.
"""
return self._send_request(
'PUT',
url_path,
body_deserialization=body_deserialization,
)
def send_delete_request(self, url_path):
"""
Send a DELETE request to HubSpot
:param basestring url_path: The URL path to the endpoint
:return: Decoded version of the ``JSON`` that HubSpot put in \
the body of the response.
"""
return self._send_request('DELETE', url_path)
def _send_request(
self,
method,
url_path,
query_string_args=None,
body_deserialization=None,
):
url = self._API_URL + url_path
query_string_args = query_string_args or {}
query_string_args = dict(query_string_args,
auditId=self._change_source)
request_headers = \
{'content-type': 'application/json'} if body_deserialization else {}
if body_deserialization:
request_body_serialization = json_serialize(body_deserialization)
else:
request_body_serialization = None
response = self._session.request(
method,
url,
params=query_string_args,
auth=self._authentication_handler,
data=request_body_serialization,
headers=request_headers,
)
response_body_deserialization = \
self._deserialize_response_body(response)
return response_body_deserialization
@classmethod
def _deserialize_response_body(cls, response):
cls._require_successful_response(response)
if response.status_code == HTTP_STATUS_OK:
cls._require_json_response(response)
response_body_deserialization = response.json() or None
elif response.status_code in _HTTP_STATUS_CODES_WITH_EMPTY_BODIES:
response_body_deserialization = None
else:
exception_message = \
'Unsupported response status {}'.format(response.status_code)
raise HubspotUnsupportedResponseError(exception_message)
return response_body_deserialization
@staticmethod
def _require_successful_response(response):
if 400 <= response.status_code < 500:
response_data = response.json()
error_data = _HUBSPOT_ERROR_RESPONSE_SCHEMA(response_data)
if response.status_code == HTTP_STATUS_UNAUTHORIZED:
exception_class = HubspotAuthenticationError
else:
exception_class = HubspotClientError
raise exception_class(
error_data['message'],
error_data['requestId'],
)
elif 500 <= response.status_code < 600:
raise HubspotServerError(response.reason, response.status_code)
@staticmethod
def _require_json_response(response):
content_type_header_value = response.headers.get('Content-Type')
if not content_type_header_value:
exception_message = 'Response does not specify a Content-Type'
raise HubspotUnsupportedResponseError(exception_message)
content_type = content_type_header_value.split(';')[0].lower()
if content_type != 'application/json':
exception_message = \
'Unsupported response content type {}'.format(content_type)
raise HubspotUnsupportedResponseError(exception_message)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self._session.close()
class Client(object):
def __init__(self, api_url='', sugar_auth=False, **kwargs):
self.api_url = api_url
self.params = kwargs
self._sugar_auth = sugar_auth
verify = True
if client.no_check_certificate.value:
verify = False
elif client.certfile.value:
verify = client.certfile.value
headers = {'Accept-Language': ad.default_lang()}
if self._sugar_auth:
privkey_path = sugar.privkey_path()
if not exists(privkey_path):
_logger.warning('Sugar session was never started, '
'fallback to anonymous mode')
self._sugar_auth = False
else:
uid = sugar.uid()
headers['sugar_user'] = uid
headers['sugar_user_signature'] = _sign(privkey_path, uid)
self._session = Session(headers=headers, verify=verify, prefetch=False)
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
def close(self):
self._session.close()
def exists(self, path):
response = self.request('GET', path, allowed=[404])
return response.status_code != 404
def get(self, path_=None, **kwargs):
response = self.request('GET', path_, params=kwargs)
return self._decode_reply(response)
def post(self, path_=None, data_=None, **kwargs):
response = self.request('POST',
path_,
json.dumps(data_),
headers={'Content-Type': 'application/json'},
params=kwargs)
return self._decode_reply(response)
def put(self, path_=None, data_=None, **kwargs):
response = self.request('PUT',
path_,
json.dumps(data_),
headers={'Content-Type': 'application/json'},
params=kwargs)
return self._decode_reply(response)
def delete(self, path_=None, **kwargs):
response = self.request('DELETE', path_, params=kwargs)
return self._decode_reply(response)
def request(self,
method,
path=None,
data=None,
headers=None,
allowed=None,
params=None,
**kwargs):
if not path:
path = ['']
if not isinstance(path, basestring):
path = '/'.join([i.strip('/') for i in [self.api_url] + path])
if params is None:
params = self.params
else:
params.update(self.params)
while True:
try:
response = requests.request(method,
path,
data=data,
headers=headers,
session=self._session,
params=params,
**kwargs)
except requests.exceptions.SSLError:
_logger.warning('Use --no-check-certificate to avoid checks')
raise
if response.status_code != 200:
if response.status_code == 401:
enforce(self._sugar_auth,
'Operation is not available in anonymous mode')
_logger.info('User is not registered on the server, '
'registering')
self._register()
continue
if allowed and response.status_code in allowed:
return response
content = response.content
try:
error = json.loads(content)
except Exception:
_logger.debug('Got %s HTTP error for %r request:\n%s',
response.status_code, path, content)
response.raise_for_status()
else:
raise RuntimeError(error['error'])
return response
def call(self, request, response=None):
params = request.copy()
method = params.pop('method')
document = params.pop('document') if 'document' in params else None
guid = params.pop('guid') if 'guid' in params else None
prop = params.pop('prop') if 'prop' in params else None
path = []
if document:
path.append(document)
if guid:
path.append(guid)
if prop:
path.append(prop)
if request.content_type == 'application/json':
request.content = json.dumps(request.content)
headers = None
if request.content is not None:
headers = {}
headers['Content-Type'] = \
request.content_type or 'application/octet-stream'
headers['Content-Length'] = str(len(request.content))
elif request.content_stream is not None:
headers = {}
headers['Content-Type'] = \
request.content_type or 'application/octet-stream'
# TODO Avoid reading the full content at once
request.content = request.content_stream.read()
headers['Content-Length'] = str(len(request.content))
reply = self.request(method,
path,
data=request.content,
params=params,
headers=headers,
allowed=[303],
allow_redirects=request.allow_redirects)
if reply.status_code == 303:
raise Redirect(reply.headers['Location'])
if response is not None:
if 'Content-Disposition' in reply.headers:
response['Content-Disposition'] = \
reply.headers['Content-Disposition']
if 'Content-Type' in reply.headers:
response.content_type = reply.headers['Content-Type']
result = self._decode_reply(reply)
if result is None:
result = reply.raw
return result
def subscribe(self):
return _Subscription(self, _RECONNECTION_NUMBER)
def _register(self):
self.post(
['user'], {
'name': sugar.nickname() or '',
'color': sugar.color() or '#000000,#000000',
'machine_sn': sugar.machine_sn() or '',
'machine_uuid': sugar.machine_uuid() or '',
'pubkey': sugar.pubkey(),
})
def _decode_reply(self, response):
if response.headers.get('Content-Type') == 'application/json':
return json.loads(response.content)
else:
return response.content
class Spider:
# todo 针对每次请求不同的`header`来重新加载缓存
# todo 增加字段`data`,存`post`字段
# todo 增加字段`header`, 存header
# 强制使用缓存
FORCE_CACHE = 2
# 运行使用缓存
ENABLE_CACHE = 1
# 不使用缓存
DISABLE_CACHE = 0
cache: CacheBase
session: Session
headers_generator: T_Headers_Generator
sleeper: T_Sleeper
request_middlewares: List[RequestMiddlewareBase]
response_middlewares: List[ResponseMiddlewareBase]
proxy_pool: Optional[ProxyPoolBase]
__encoding: str
cache_mode: Literal[0, 1, 2]
timeout: Tuple[int, int]
alive_time: int
retry: int
def __init__(
self,
cache: Optional[CacheBase] = None,
session: Optional[Session] = None,
headers_generator: Optional[T_Headers_Generator] = None,
sleeper: Optional[T_Sleeper] = None,
request_middlewares: Optional[List[RequestMiddlewareBase]] = None,
response_middlewares: Optional[List[ResponseMiddlewareBase]] = None,
proxy_pool: Optional[ProxyPoolBase] = None,
__encoding: Optional[str] = None,
cache_mode: Optional[Literal[0, 1, 2]] = None,
timeout: Optional[Tuple[int, int]] = None,
alive_time: Optional[int] = None,
retry: Optional[int] = None,
):
self.cache = cache or NoCache()
if not session:
self.session = Session()
self.sleeper = sleeper or NoSleeper()
self.request_middlewares = request_middlewares or []
self.response_middlewares = response_middlewares or []
self.proxy_pool = proxy_pool
self.__encoding = __encoding
# todo
self.headers_generator = headers_generator or get_random_header
self.cache_mode = cache_mode or Spider.DISABLE_CACHE
self.timeout = timeout or (5, 5)
self.alive_time = alive_time or 3
self.retry = retry or 3
self.update_headers()
@classmethod
def get_cache_spider(cls) -> 'Spider':
spider = cls(SqliteCache())
spider.cache_mode = Spider.ENABLE_CACHE
return spider
def set_sleeper(self, sleeper: T_Sleeper):
if not isinstance(sleeper, Callable):
raise RuntimeError('参数必须是Callable')
self.sleeper = sleeper
def set_random_sleeper(self, a=5, b=10):
self.set_sleeper(RandomTimeSleeper(a, b))
def set_sep_sleeper(self, sep_time=10):
self.set_sleeper(SepTimeSleeper(sep_time))
@property
def encoding(self):
return self.__encoding
@encoding.setter
def encoding(self, encoding):
self.__encoding = encoding
def set_cookie(self, cookie: str):
"""设置cookie
:param cookie: 一个 `cookie` 字符串
"""
cookie = {'Cookie': cookie}
self.session.cookies = requests.sessions.cookiejar_from_dict(
cookie, cookiejar=None, overwrite=True)
def add_request_middlewares(
self, request_middlewares: Union[RequestMiddlewareBase,
List[RequestMiddlewareBase]]):
if isinstance(request_middlewares, list):
self.request_middlewares.extend(request_middlewares)
else:
self.request_middlewares.append(request_middlewares)
def add_response_middlewares(
self, response_middlewares: Union[ResponseMiddlewareBase,
List[ResponseMiddlewareBase]]):
if isinstance(response_middlewares, list):
self.response_middlewares.extend(response_middlewares)
else:
self.response_middlewares.append(response_middlewares)
def remove_request_middlewares(self, request_middlewares):
del self.request_middlewares[request_middlewares]
def remove_response_middlewares(self, response_middlewares):
del self.response_middlewares[response_middlewares]
def get_response(self,
req: PreparedRequest,
timeout=None,
allow_redirects=True,
cache_mode=None,
alive_time=None,
retry=None):
"""低级 api 发送请求， 在这里进行 request_middleware， response_middleware 的处理"""
cache_mode = cache_mode or self.cache_mode
alive_time = alive_time or self.alive_time
timeout = timeout or self.timeout
retry = retry or self.retry
cache_signal = req.url
if cache_mode == Spider.DISABLE_CACHE:
# 如果禁用这个url的缓存, 则将之从缓存文件删除
self.cache.clear_cache(cache_signal)
# 如果 `is_force_cache` is True 则, 不论缓存是否过期, 都从缓存加载
elif cache_mode == Spider.FORCE_CACHE and self.cache.is_cached(
cache_signal, ignore_date=True):
logger.debug('从缓存: {} <- {}', limit_text(req.url, 100), self.cache)
response = self.cache.from_cache(cache_signal, force=True)
if self.encoding:
response.encoding = self.encoding
return response
elif cache_mode == Spider.ENABLE_CACHE and self.cache.is_cached(
cache_signal, ignore_date=False):
logger.debug('从缓存: {} <- {}', limit_text(cache_signal, 100),
self.cache)
response = self.cache.from_cache(cache_signal, force=False)
if self.encoding:
response.encoding = self.encoding
return response
retry = retry
while retry:
try:
# 间隔时间
response = self.session.send(req,
timeout=timeout or (5, 5),
allow_redirects=allow_redirects)
if self.encoding:
response.encoding = self.encoding
response = Response(response)
self.sleeper()
if len(response.response.history) >= 1:
history = [each.url for each in response.response.history]
history.append(response.url)
logger.debug(
'页面重定向: {}',
'->'.join(['[{}]'.format(i) for i in history]))
if response.response.ok:
if cache_mode == Spider.ENABLE_CACHE:
self.cache.cache(response.url, response, alive_time)
else:
raise HTTPBadCodeError(f'坏HTTP响应', response)
return response
except requests.Timeout as e:
if retry == 1:
raise e
logger.debug('超时,重试---{}'.format(str(4 - retry)))
except requests.RequestException as e:
if retry == 1:
logger.error('取消重试---{}'.format(str(4 - retry)))
raise e
logger.error('HTTP报错---{}'.format(str(4 - retry)))
# todo 对于失败的`url`保存到另一个`log`文件
except HTTPBadCodeError as e:
if retry == 1:
logger.info('坏HTTP响应, 取消重试({})'.format(
e.args[1].status_code))
return e.args[1]
logger.debug('坏HTTP响应({})---{}', e.args[1].status_code,
4 - retry)
finally:
retry -= 1
def lunch(self,
method,
url,
*,
cache_mode=None,
alive_time=None,
timeout=None,
retry=None,
headers=None,
data=None,
params=None,
cookies=None,
hooks=None):
"""高级 api 发送请求， 处理网页缓存等功能"""
# 发送请求
req = requests.Request(
method=method.upper(),
url=url,
headers=headers,
files=None,
data=data or {},
json=None,
params=params or {},
auth=None,
cookies=cookies,
hooks=hooks,
)
req = self.session.prepare_request(req)
for request_middleware in self.request_middlewares:
req = request_middleware(self, req)
resp = self.get_response(req,
cache_mode=cache_mode,
alive_time=alive_time,
timeout=timeout,
retry=retry)
for response_middleware in self.response_middlewares:
resp = response_middleware(self, resp)
return resp
def get(self, *args, **kwargs) -> [Response, requests.Response, object]:
"""获取网页
:param args: (元组)`url`的各个路径:
:param kwargs: 包含`requests`库所有选项
:keyword alive_time: Union[datetime, int]缓存存活日期
:keyword cache: 是否使用缓存
:keyword sep_time: 间隔时间
:return: Union[Response, requests.Response, object]
"""
# 获取`alive_time`, `url`参数
resp = self.lunch('get', *args, **kwargs)
return resp
def post(self, *args, **kwargs) -> Response:
"""获取网页
:param args: (元组)`url`的各个路径:
:param kwargs: 包含`requests`库所有选项
:keyword alive_time: Union[datetime, int]缓存存活日期
:keyword cache: 是否使用缓存
:keyword sep_time: 间隔时间
:return: Union[Response, requests.Response, object]
"""
resp = self.lunch('post', *args, **kwargs)
return resp
def update_headers(self):
"""调用`self.headers_generator`来更新头"""
self.session.headers.update(self.headers_generator())
def close(self):
self.session.close()
class Porter:
DEFAULTS_PATH = "default_settings/"
def __init__(self, user="", password="", host="http://rocketchat:3000", destination="settings.json"):
self.username = os.environ.get('API_USER', user)
self.password = os.environ.get('API_PASS', password)
self.host = os.environ.get('API_HOST', host)
self.destination = os.environ.get('SETTINGS_PATH', destination)
self.session = Session()
self._wait_for_rocketchat()
def _wait_for_rocketchat(self, rocket=None):
self.rocket = rocket
timeout = 60
logger.info("starting to wait for rocketchat...")
while timeout:
time.sleep(1)
try:
if self.rocket is None:
self.rocket = RocketChat(self.username, self.password, server_url=self.host, session=self.session)
ret = self.rocket.info().json()
if ret.get('success'):
logger.info("rocketchat available!")
return
except ConnectionError:
pass
except:
pass
timeout =- 1
logger.error("Timeout while waiting for rocketchat!")
def import_settings(self):
with open(self.destination, 'r') as f:
settings_to_import = json.load(f)
logger.info(f"Importing {len(settings_to_import)} settings...")
self._import(settings_to_import)
logger.info("Finished importing settings!")
def _import(self, settings):
import_oauth = {}
for _id, value in settings.items():
if _id.startswith("Accounts_OAuth_Custom-"):
custom_name = _id.split('-')
if len(custom_name) == 2:
import_oauth[custom_name[1]] = True
continue
self.rocket.settings_update(_id, value)
if import_oauth:
# create all neccessary OAuth services
for i, oauth_service_name in enumerate(import_oauth.keys()):
logger.debug(f'create {oauth_service_name}')
json_ = json.dumps({'msg': 'method', 'method': 'addOAuthService', 'params': [oauth_service_name], 'id': 1337+i})
self.rocket.call_api_post("method.call/addOAuthService", message=json_)
for _id, value in settings.items():
if _id.startswith("Accounts_OAuth_Custom-"):
self.rocket.settings_update(_id, value)
def _export(self):
data = self.rocket.settings().json()
settings = {sett_.get('_id'): sett_.get('value') for sett_ in data.get('settings', [])}
if not settings:
logger.error('Nothing returned from server, just "{}"!'.format(data))
return {}
api_default_count = settings.get('API_Default_Count', 50)
# Get remaining
for call in range(1, int(data.get('total', 1) / api_default_count)+1):
data = self.rocket.settings(count=api_default_count, offset=call*api_default_count).json()
for sett_ in data.get('settings', []):
settings[sett_.get('_id')] = sett_.get('value')
return settings
def export_changed(self):
logger.info("Exporting all non-default settings...")
version = self.rocket.info().json().get('info', {}).get('version', 0)
if not version:
logger.error('Could not get server version!')
return
try:
with open(os.path.join(self.DEFAULTS_PATH, '{}.json'.format(version)), 'r') as f:
settings_defaults = json.load(f)
except FileNotFoundError:
logger.error('No defaults for version {} found in the directory {}!'.format(version, self.DEFAULTS_PATH))
return
settings = self._export()
settings_updated = {}
for setting_id, setting_value in settings.items():
if setting_id not in settings_defaults or setting_value != settings_defaults.get(setting_id):
settings_updated[setting_id] = setting_value
with open(self.destination, 'w') as f:
json.dump(settings_updated, f)
logger.info(f"Finished exporting {len(settings_updated)} updated settings")
def export_all(self):
logger.info("Exporting all settings...")
settings = self._export()
with open(self.destination, 'w') as f:
json.dump(settings, f)
logger.info(f"Finished exporting {len(settings)} settings")
def close(self):
self.session.close()
class StrainRegistry:
# Folder Collections
FEATURED = "FEATURED"
PERSONAL = "PERSONAL"
SHARED = "SHARED"
# don't care about SAMPLES, PENDING, or DRAFTS
ALL_COLLECTIONS = (FEATURED, PERSONAL, SHARED)
def __init__(self):
self.auth = None
self.session = None
def __enter__(self):
self.session = Session()
self.session.auth = self.auth
def __exit__(self, exc_type, exc_value, exc_traceback):
self.session.close()
self.session = None
def build_entry_url(self, entry_id):
return self._rest(f"parts/{entry_id}")
def build_folder_url(self, folder_id):
return self._rest(f"folders/{folder_id}")
def create_folder(self, folder_name):
self._check_session()
try:
response = self.session.post(
self._rest("folders"),
json={"folderName": folder_name},
)
response.raise_for_status()
return Folder(self, response.json())
except Exception as e:
raise RegistryError("Could not create folder") from e
def get_entry(self, entry_id):
self._check_session()
try:
response = self.session.get(self.build_entry_url(entry_id))
response.raise_for_status()
return Entry(self, response.json())
except Exception as e:
raise RegistryError("Could not load Registry Entry") from e
def get_folder(self, folder_id):
self._check_session()
try:
response = self.session.get(self.build_folder_url(folder_id))
response.raise_for_status()
return Folder(self, response.json())
except Exception as e:
raise RegistryError("Could not load Registry Folder") from e
def iter_entries(self, collection="available", **extra):
self._check_session()
def entries_api(start):
response = self.session.get(
self._rest(f"collections/{collection}/entries"),
params={
"start": start,
**extra
},
)
response.raise_for_status()
raw_results = response.json()["data"]
return [Entry(self, item) for item in raw_results]
try:
yield from self._yield_paged_records(entries_api)
except Exception as e:
raise RegistryError("Could not iterate Registry Entries") from e
def list_entries(self, collection="available", **extra):
self._check_session()
try:
response = self.session.get(
self._rest(f"collections/{collection}/entries"),
params=extra,
)
response.raise_for_status()
return [Entry(self, item) for item in response.json()["data"]]
except Exception as e:
raise RegistryError("Could not list Registry Entries") from e
def list_folders(self, collection="FEATURED"):
self._check_session()
try:
# this endpoint does not support paging
response = self.session.get(
self._rest(f"collections/{collection}/folders"))
response.raise_for_status()
return [Folder(self, item) for item in response.json()]
except Exception as e:
raise RegistryError("Could not list Registry Folders") from e
def login(self, user):
if key_id := getattr(settings, "ICE_KEY_ID", None):
self.auth = HmacAuth(key_id=key_id, username=user.email)
return self
def _remove_session(key: str, session: Session) -> None:
session.close()
class SessionState:
def __init__(self, cache_file: Path, cache_key: str, redis: str, user_requested=False):
self.user_requested = user_requested
self._cache_file = cache_file
self._cache_key = cache_key
self._cache: Optional[SqliteDict] = None
random.seed()
self._session_key = random.randint(0, 999999)
self._redis = Redis(host=redis)
if not user_requested:
self._open()
if self._cache_key != self._cache.get("_cache_key_", None):
self._cache.close()
self._cache: Optional[SqliteDict] = None
self._cache_file.unlink()
self._open()
self._cache["_cache_key_"] = self._cache_key
self.session = Session()
# noinspection PyTypeChecker
self.session.mount(
'https://',
HTTPAdapter(max_retries=Retry(total=3, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504])))
self.sites = {}
self.wikidata = Sparql()
self.primary_site = self.get_site(primary_domain)
def __enter__(self):
return self
def __exit__(self, typ, value, traceback):
self.session.close()
if self._cache is not None:
self._cache.close()
self._cache = None
print(f'Closed SQL connection for {self._session_key} at {datetime.utcnow()}')
def _open(self):
if self._cache is None:
print(f'Opening SQL connection for {self._session_key} at {datetime.utcnow()}')
self._cache_file.parent.mkdir(parents=True, exist_ok=True)
self._cache = SqliteDict(self._cache_file, autocommit=True)
def get_site(self, domain: Domain) -> WikiSite:
try:
return self.sites[domain]
except KeyError:
# noinspection PyTypeChecker
site = WikiSite(domain, self.session, domain == primary_domain)
if self.user_requested:
site.maxlag = None
self.sites[domain] = site
return site
def delete_cached_items(self, prefix: str) -> None:
self._open()
for vv in {v for v in self._cache.keys() if v.startswith(prefix)}:
del self._cache[vv]
def del_obj(self, key: str) -> Any:
self._redis.delete(self.redis_key(key))
self._open()
print(f"%% del {key}")
return self._cache.pop(key, None)
def load_obj(self, key: str, default: Any = None) -> Any:
value = self._redis.get(self.redis_key(key))
if value is not None:
return loads(value)
self._open()
print(f"%% load {key}")
value = self._cache.get(key, default)
self._redis.set(self.redis_key(key), dumps(value))
return value
def save_obj(self, key: str, value: Any):
self._open()
print(f"%% save {key}")
self._cache[key] = value
self._redis.set(self.redis_key(key), dumps(value))
def redis_key(self, key: str):
return self._cache_key + key
class WxXhsApi(object):
def __init__(self, host=XHS_BASE_HOST, prefix="wx_mp_api/sns/v1/", scheme="https"):
self._host = host
self._prefix = prefix
self._base_url = "{}://{}/{}".format(scheme, host, prefix)
self._session = Session()
ui = random.randint(0, len(USER_AGENTS) - 1)
headers = {
"User-Agent": USER_AGENTS[ui],
"Host": self._host,
"Connection": "close",
"Accept-Encoding": "br, gzip, deflate",
"Content-Type": "application/json",
"Accept-Language": "zh-cn",
"Device-Fingerprint":
"WHJMrwNw1k/Gy/sC6Z1D0XzFNbmyE3cyfJCjTR5D+eJ4GPjHvuEU1skE1O3fkhMVWGWPZ3E6FqIOaBmFkRMqt6xFlRX"
"tfTfVBdCW1tldyDzmauSxIJm5Txg==1487582755342",
"Authorization": "ea521d9b-c1fb-4f91-8560-bfa160171e5a"
}
sid = "1572606846528931138054"
self._sid = "session.{}".format(sid)
self._session.headers.update(headers)
self._session.keep_alive = False
def _get(self, route, params=None, retry=1):
url = urljoin(self._base_url, route)
if params:
params.update(sid=self._sid)
try:
ret = self._session.get(url, params=params, timeout=(5, 10))
LOG.info("GET {} {}".format(ret.url, ret.status_code))
except (requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError)as e:
LOG.error(e)
return None
error = ret.text
if 200 <= ret.status_code < 300:
ret.encoding = 'utf-8'
data = ret.json()
success = data.get("success")
if success:
return data["data"]
else:
print("result:{}-{}".format(data.get("result"), data.get("msg")))
if retry:
return self._get(route, params, retry - 1)
raise Exception('Error url:{} -{}'.format(url, error))
# def get_homefeed_categories(self):
# route = "homefeed/categories"
# result = self._get(route)
# return result
#
# def get_search_tending(self):
# route = "search/trending"
# result = self._get(route)
# return result
def search_notes(self, keyword, page=1, per_page=20, sort="general"):
route = "search/notes"
params = {
"keyword": keyword,
"page": page,
"per_page": per_page,
"sort": sort
}
result = self._get(route, params)
return result
def get_note_info_list(self, note_id):
route = "note/{}/single_feed".format(note_id)
result = self._get(route)
return result
def close(self):
return self._session.close()
class Client:
def __init__(self, username, password, proxy):
self.ses = Session()
self.loggedIn = False
self.username = username
self.password = password
self.proxy = proxy
def Login(self):
if self.loggedIn == True:
return None
loginData = {
"password": self.password,
"username": self.username,
"queryParams": "{}"
}
homePageResponse = self.ses.get(
"https://www.instagram.com/accounts/login/")
loginHeaders = {
"Accept": "*/*",
"Accept-Encoding": "gzip,deflate,br",
"Accept-Language": "en-US,en;q=0.5",
"Connection": "keep-alive",
"Content-Type": "application/x-www-form-urlencoded",
"Host": "www.instagram.com",
"Referer": "https://www.instagram.com/accounts/login/",
"X-Requested-With": "XMLHttpRequest",
"X-Instagram-AJAX": "1",
"User-Agent": USER_AGENT,
"X-CSRFToken": homePageResponse.cookies.get_dict()["csrftoken"],
}
loginCookies = {
"rur": "PRN",
"csrftoken": homePageResponse.cookies.get_dict()["csrftoken"],
"mcd": homePageResponse.cookies.get_dict()["mcd"],
"mid": homePageResponse.cookies.get_dict()["mid"]
}
self.ses.headers.update(loginHeaders)
self.ses.cookies.update(loginCookies)
loginPostResponse = self.ses.post(
"https://www.instagram.com/accounts/login/ajax/", data=loginData)
if loginPostResponse.status_code == 200 and loginPostResponse.json(
)["authenticated"] == True:
self.loggedIn = True
mainPageResponse = self.ses.get("https://www.instagram.com/")
self.ses.cookies.update(mainPageResponse.cookies)
def Spam(self, username, userid):
if self.loggedIn == False:
return None
link = "https://www.instagram.com/" + username + "/"
profileGetResponse = self.ses.get(link)
self.ses.cookies.update(profileGetResponse.cookies)
spamHeaders = {
"Accept": "*/*",
"Accept-Encoding": "gzip,deflate,br",
"Accept-Language": "en-US,en;q=0.5",
"Connection": "keep-alive",
"Content-Type": "application/x-www-form-urlencoded",
"DNT": "1",
"Host": "www.instagram.com",
"X-Instagram-AJAX": "2",
"X-Requested-With": "XMLHttpRequest",
"Referer": link,
"User-Agent": USER_AGENT,
"X-CSRFToken": profileGetResponse.cookies.get_dict()["csrftoken"],
}
"Hichigo was here THT"
spamData = {"reason_id": "1", "source_name": "profile"}
self.ses.headers.update(spamHeaders)
spamPostResponse = self.ses.post("https://www.instagram.com/users/" +
userid + "/report/",
data=spamData)
if spamPostResponse.status_code == 200 and spamPostResponse.json(
)["description"] == "Your reports help keep our community free of spam.":
self.ses.close()
return True
else:
return False
class Api(object):
session: Session
job_type: JobType
url: str
def __init__(self,
job_type: JobType,
url: str,
api_key: str,
headers: Dict[str, str] = {}):
self.session = Session()
self.job_type = job_type
self.url = self._normalize_url(url)
self.session.headers.update({"X-Api-Key": api_key, **headers})
def __enter__(self) -> Api:
return self
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
self.session.close()
def _normalize_url(self, url: str) -> str:
return url if url.endswith("/") else f"{url}/"
def _response_json(self, response: Response, url: str) -> Any:
if not response.ok:
raise Exception(
f"failed to check status for {url} got {response.status_code}")
if not response.text:
logger.error("%s response_text: %s", url, response.text)
raise Exception(
f"no response in status for {url}. Is the server set up correctly?"
)
return response.json()
def get(self, url: str) -> Any:
response = self.session.get(url=url)
return self._response_json(response=response, url=url)
def post(self, url: str, json: Dict[Any, Any]) -> Any:
response = self.session.post(url=url, json=json)
return self._response_json(response=response, url=url)
def status(self) -> Status:
full_url = routes.status(job_type=self.job_type, url=self.url)
json = self.get(url=full_url)
return Status.parse_obj(json)
def profile(self) -> Profiles:
full_url = routes.profile(job_type=self.job_type, url=self.url)
json = self.get(url=full_url)
return list(map(Profile.parse_obj, json))
def tag(self) -> Tags:
full_url = routes.tag(job_type=self.job_type, url=self.url)
json = self.get(url=full_url)
return list(map(Tag.parse_obj, json))
def language(self) -> Languages:
# Only Sonarr supports setting languageProfileId
if self.job_type != JobType.Sonarr:
return []
full_url = routes.language(job_type=self.job_type, url=self.url)
json = self.get(url=full_url)
return list(map(Language.parse_obj, json))
def metadata(self) -> Profiles:
# Only Lidarr supports setting metadataProfileId
if self.job_type != JobType.Lidarr:
return []
full_url = routes.metadata(job_type=self.job_type, url=self.url)
json = self.get(url=full_url)
return list(map(Profile.parse_obj, json))
def content(self) -> ContentItems:
full_url = routes.content(job_type=self.job_type, url=self.url)
json = self.get(url=full_url)
if self.job_type is JobType.Sonarr:
return list(map(SonarrContent.parse_obj, json))
elif self.job_type is JobType.Radarr:
return list(map(RadarrContent.parse_obj, json))
elif self.job_type is JobType.Lidarr:
return list(map(LidarrContent.parse_obj, json))
else:
_assert_never(self.job_type)
def save(self, content_item: ContentItem) -> Any:
full_url = routes.content(job_type=self.job_type, url=self.url)
return self.post(url=full_url, json=content_item.dict(by_alias=True))
ticketId, status, hardVersion, DeviceFac, Android, AppVersion,
url_new, MobilePhone, picExist, Desc
]
op_list = []
for item in or_list:
if "'" not in str(item):
op_list.append(str(item))
continue
try:
op_list.append(re.search("：(.*)'", str(item)).group(1))
except AttributeError:
op_list.append(re.search("'(.*)'", str(item)).group(1))
# f.write(f"{url_new}\t{MobilePhone}\t{AppVersion}\t{Desc}\n")
for i in range(op_list.__len__()):
my_excel.w_data_origin(line, i + 1, op_list[i])
line += 1
end = datetime.datetime.now()
print(ticketId, end - start)
my_session.close()
my_excel.close()
my_excel.save()
# #
# 前一位：
# ../div[@="class"]/preceding-sibling::div[1]
# 后一位：
# ../div[@="class"]/following-sibling::div[1]
class Connection:
_API_URL = 'https://www.2degreesnetwork.com/api'
def __init__(self, auth, timeout=None, api_url=None):
super(Connection, self).__init__()
self._api_url = api_url or self._API_URL
self._authentication_handler = auth
self._session = Session()
self._session.headers['User-Agent'] = _USER_AGENT
self._timeout = timeout
http_adapter = HTTPAdapter(max_retries=_HTTP_CONNECTION_MAX_RETRIES)
self._session.mount('', http_adapter)
def send_get_request(self, url, query_string_args=None):
"""
Send a GET request
:param str url: The URL or URL path to the endpoint
:param dict query_string_args: The query string arguments
:return: Decoded version of the ``JSON`` the remote put in \
the body of the response.
"""
return self._send_request('GET', url, query_string_args)
def send_head_request(self, url, query_string_args=None):
"""
Send a HEAD request
:param str url: The URL or URL path to the endpoint
:param dict query_string_args: The query string arguments
"""
return self._send_request('HEAD', url, query_string_args)
def send_post_request(self, url, body_deserialization=None):
"""
Send a POST request
:param str url: The URL or URL path to the endpoint
:param dict body_deserialization: The request's body message \
deserialized
:return: Decoded version of the ``JSON`` the remote put in \
the body of the response.
"""
return self._send_request(
'POST',
url,
body_deserialization=body_deserialization,
)
def send_put_request(self, url, body_deserialization):
"""
Send a PUT request
:param str url: The URL or URL path to the endpoint
:param body_deserialization: The request's body message deserialized
:return: Decoded version of the ``JSON`` the remote put in \
the body of the response.
"""
return self._send_request(
'PUT',
url,
body_deserialization=body_deserialization,
)
def send_delete_request(self, url):
"""
Send a DELETE request
:param str url: The URL or URL path to the endpoint
:return: Decoded version of the ``JSON`` the remote put in \
the body of the response.
"""
return self._send_request('DELETE', url)
def _send_request(
self,
method,
url,
query_string_args=None,
body_deserialization=None,
):
if url.startswith(self._api_url):
url = url
else:
url = self._api_url + url
query_string_args = query_string_args or {}
request_headers = \
{'content-type': 'application/json'} if body_deserialization else {}
if body_deserialization:
request_body_serialization = json_serialize(body_deserialization)
else:
request_body_serialization = None
response = self._session.request(
method,
url,
params=query_string_args,
auth=self._authentication_handler,
data=request_body_serialization,
headers=request_headers,
timeout=self._timeout,
)
self._require_successful_response(response)
self._require_deserializable_response_body(response)
return response
@staticmethod
def _require_successful_response(response):
if 400 <= response.status_code < 500:
if response.status_code == HTTPStatus.UNAUTHORIZED:
exception_class = AuthenticationError
elif response.status_code == HTTPStatus.FORBIDDEN:
exception_class = AccessDeniedError
elif response.status_code == HTTPStatus.NOT_FOUND:
exception_class = NotFoundError
else:
exception_class = ClientError
raise exception_class()
elif 500 <= response.status_code < 600:
raise ServerError(response.reason, response.status_code)
@classmethod
def _require_deserializable_response_body(cls, response):
if response.status_code in (HTTPStatus.OK, HTTPStatus.NO_CONTENT):
if response.content:
cls._require_json_response(response)
else:
exception_message = \
'Unsupported response status {}'.format(response.status_code)
raise UnsupportedResponseError(exception_message)
@staticmethod
def _require_json_response(response):
content_type_header_value = response.headers.get('Content-Type')
if not content_type_header_value:
exception_message = 'Response does not specify a Content-Type'
raise UnsupportedResponseError(exception_message)
content_type = content_type_header_value.split(';')[0].lower()
if content_type != 'application/json':
exception_message = \
'Unsupported response content type {}'.format(content_type)
raise UnsupportedResponseError(exception_message)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self._session.close()
class PortalConnection(object):
"""
Connection to HubSpot
:param authentication_key: This can be either an :class:`APIKey` or an \
:class:`OAuthKey` instance
:param basestring change_source: The string passed to HubSpot as \
``auditId`` in the query string
"""
_API_URL = 'https://api.hubapi.com'
def __init__(self, authentication_key, change_source):
super(PortalConnection, self).__init__()
self._authentication_handler = \
_QueryStringAuthenticationHandler(authentication_key)
self._change_source = change_source
self._session = Session()
self._session.headers['User-Agent'] = _USER_AGENT
http_adapter = HTTPAdapter(max_retries=_HTTP_CONNECTION_MAX_RETRIES)
self._session.mount('', http_adapter)
def send_get_request(self, url_path, query_string_args=None):
"""
Send a GET request to HubSpot
:param basestring url_path: The URL path to the endpoint
:param dict query_string_args: The query string arguments
:return: Decoded version of the ``JSON`` that HubSpot put in \
the body of the response.
"""
return self._send_request('GET', url_path, query_string_args)
def send_post_request(self, url_path, body_deserialization):
"""
Send a POST request to HubSpot
:param basestring url_path: The URL path to the endpoint
:param dict body_deserialization: The request's body message \
deserialized
:return: Decoded version of the ``JSON`` that HubSpot put in \
the body of the response.
"""
return self._send_request(
'POST',
url_path,
body_deserialization=body_deserialization,
)
def send_put_request(self, url_path, body_deserialization):
"""
Send a PUT request to HubSpot
:param basestring url_path: The URL path to the endpoint
:param body_deserialization: The request's body message deserialized
:return: Decoded version of the ``JSON`` that HubSpot put in \
the body of the response.
"""
return self._send_request(
'PUT',
url_path,
body_deserialization=body_deserialization,
)
def send_delete_request(self, url_path):
"""
Send a DELETE request to HubSpot
:param basestring url_path: The URL path to the endpoint
:return: Decoded version of the ``JSON`` that HubSpot put in \
the body of the response.
"""
return self._send_request('DELETE', url_path)
def _send_request(
self,
method,
url_path,
query_string_args=None,
body_deserialization=None,
):
url = self._API_URL + url_path
query_string_args = query_string_args or {}
query_string_args = dict(query_string_args, auditId=self._change_source)
request_headers = \
{'content-type': 'application/json'} if body_deserialization else {}
if body_deserialization:
request_body_serialization = json_serialize(body_deserialization)
else:
request_body_serialization = None
response = self._session.request(
method,
url,
params=query_string_args,
auth=self._authentication_handler,
data=request_body_serialization,
headers=request_headers,
)
response_body_deserialization = \
self._deserialize_response_body(response)
return response_body_deserialization
@classmethod
def _deserialize_response_body(cls, response):
cls._require_successful_response(response)
cls._require_json_response(response)
if response.status_code == HTTP_STATUS_OK:
response_body_deserialization = response.json()
elif response.status_code in _HTTP_STATUS_CODES_WITH_EMPTY_BODIES:
response_body_deserialization = None
else:
exception_message = \
'Unsupported response status {}'.format(response.status_code)
raise HubspotUnsupportedResponseError(exception_message)
return response_body_deserialization
@staticmethod
def _require_successful_response(response):
if 400 <= response.status_code < 500:
response_data = response.json()
error_data = _HUBSPOT_ERROR_RESPONSE_SCHEMA(response_data)
if response.status_code == HTTP_STATUS_UNAUTHORIZED:
exception_class = HubspotAuthenticationError
else:
exception_class = HubspotClientError
raise exception_class(
error_data['message'],
error_data['requestId'],
)
elif 500 <= response.status_code < 600:
raise HubspotServerError(response.reason, response.status_code)
@staticmethod
def _require_json_response(response):
content_type_header_value = response.headers.get('Content-Type')
if not content_type_header_value:
exception_message = 'Response does not specify a Content-Type'
raise HubspotUnsupportedResponseError(exception_message)
content_type = content_type_header_value.split(';')[0].lower()
if content_type != 'application/json':
exception_message = \
'Unsupported response content type {}'.format(content_type)
raise HubspotUnsupportedResponseError(exception_message)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self._session.close()
class BiliApi:
def __init__(self, headers: Optional[Dict[str, str]]):
if not headers:
headers = _default_headers
self._session = Session()
self._session.headers.update(headers)
self._session.trust_env = True
self._islogin = False
self._show_name = None
def login_by_cookie(self,
cookieData,
checkBanned=True,
strict=False) -> bool:
'''
登录并获取账户信息
cookieData dict 账户cookie
checkBanned bool 检查是否被封禁
strict bool 是否严格限制cookie在.bilibili.com域名之下
'''
if strict:
from yarl import URL
self._session.cookies.update(cookieData,
URL('https://.bilibili.com'))
else:
self._session.cookies.update(cookieData)
self.refreshInfo()
if not self._islogin:
return False
if 'bili_jct' in cookieData:
self._bili_jct = cookieData["bili_jct"]
else:
self._bili_jct = ''
self._isBanned = None
if checkBanned:
code = (self.likeCv(10743282))["code"]
if code != 0 and code != 65006 and code != -404:
self._isBanned = True
import warnings
warnings.warn(f'{self._name}:账号异常，请检查bili_jct参数是否有效或本账号是否被封禁')
else:
self._isBanned = False
return True
@property
def banned(self):
'''是否账号被异常封禁'''
return self._isBanned
@property
def islogin(self):
'''是否登录'''
return self._islogin
@property
def myexp(self) -> int:
'''获取登录的账户的经验'''
return self._exp
@property
def mycoin(self) -> int:
'''获取登录的账户的硬币数量'''
return self._coin
@property
def vipType(self) -> int:
'''获取登录的账户的vip类型'''
return self._vip
@property
def name(self) -> str:
'''获取用于显示的用户名'''
return self._show_name
@name.setter
def name(self, name: str) -> None:
'''设置用于显示的用户名'''
self._show_name = name
@property
def username(self) -> str:
'''获取登录的账户用户名'''
return self._name
@property
def uid(self) -> int:
'''获取登录的账户uid'''
return self._uid
@property
def level(self) -> int:
'''获取登录的账户等级'''
return self._level
def refreshInfo(self):
'''刷新账户信息(需要先登录)'''
ret = self.getWebNav()
if ret["code"] != 0:
self._islogin = False
return
self._islogin = True
self._name = ret["data"]["uname"]
self._uid = ret["data"]["mid"]
self._vip = ret["data"]["vipType"]
self._level = ret["data"]["level_info"]["current_level"]
self._verified = ret["data"]["mobile_verified"]
self._coin = ret["data"]["money"]
self._exp = ret["data"]["level_info"]["current_exp"]
if not self._show_name:
self._show_name = self._name
def xliveRoomInit(self, rid: int = 1):
'''
获取房间初始化信息(房间短id转长id)
id int 直播间id
'''
url = f'https://api.live.bilibili.com/room/v1/Room/room_init?id={rid}'
with self._session.get(url) as r:
return r.json()
def xliveMsgSend(
self,
roomid: int,
msg: str,
color: int = 16777215,
fontsize: int = 25,
mode: int = 1,
bubble: int = 0,
):
'''
直播间发送消息
roomid int 直播间id
msg str 要发送的消息
color int 字体颜色
fontsize int 字体大小
mode int 发送模式，应该是控制滚动，底部这些
bubble int 未知
'''
url = 'https://api.live.bilibili.com/msg/send'
post_data = {
"color": color,
"fontsize": fontsize,
"mode": mode,
"msg": msg,
"rnd": int(time.time()),
"roomid": roomid,
"bubble": bubble,
"csrf_token": self._bili_jct,
"csrf": self._bili_jct
}
with self._session.post(url, data=post_data) as r:
return r.json()
def likeCv(self, cvid: int, type=1) -> Dict:
'''
点赞专栏
cvid int 专栏id
type int 类型
'''
url = 'https://api.bilibili.com/x/article/like'
post_data = {"id": cvid, "type": type, "csrf": self._bili_jct}
with self._session.post(url, data=post_data) as r:
return r.json()
def getWebNav(self) -> Dict[str, Any]:
'''取导航信息'''
url = "https://api.bilibili.com/x/web-interface/nav"
with self._session.get(url) as r:
ret = r.json()
return ret
def __enter__(self):
return self
def __exit__(self, *exc) -> None:
self.close()
def close(self) -> None:
self._session.close()
