async def test_function():
session = asks.Session()
client = SlackAPICurio(session=session, token=token)
response = await client._request(
"POST", "https://slack.com/api//api.test", {},
{"token": token})
return response[0], response[1]
def get_http_session(connections=50):
ssl_context = ssl.SSLContext()
ssl_context.verify_mode = ssl.CERT_NONE
ssl_context.check_hostname = False
session = asks.Session(connections=connections,
ssl_context=ssl_context)
return session
async def ready(self):
self.menu_session.headers = config.fake_header
self.menu_session.headers.update({"User-Agent": self.random_ua()})
self.async_session = asks.Session(connections=config.max_connections)
self.async_session.headers = config.fake_header
self.main_url = f"https://{self.domain}/"
self.url_list = {
"citys":
urljoin(self.main_url, "city/"),
"old":
lambda page: urljoin(
self.main_url,
f"/{self.current_city}/ershoufang/pg{page}/?_t=1"),
"new":
lambda page: urljoin(
self.main_url,
f"/{self.current_city}/loupan/pg{page}/?_t=1&source=index",
),
"chuzu":
lambda page: urljoin(
self.main_url,
f"/chuzu/{self.current_city}/zufang/pg{page}/?ajax=1"),
"xiaoqu":
lambda page: urljoin(
self.main_url, f"/{self.current_city}/xiaoqu/pg{page}/?_t=1"),
"zhuangxiu":
lambda page: urljoin(
self.main_url,
f"/home/{self.current_city}/zhuangxiu/list/getCompanyInfoList?tagID=&page={page}&pageSize=10&timer={int(time.time())}",
),
}
list(map(init_path, [config.debug_file, config.datas_file]))
def __init__(
self,
config: dict,
):
# use 2 connections per streaming endpoint (stocks, opts)
# TODO: when we have more then one account key then this should scale
# linearly with that.
self._sess = asks.Session(connections=4)
self.api = _API(self)
self._conf = config
self._is_practice = _use_practice_account or (config['questrade'].get(
'is_practice', False))
self._auth_ep = _refresh_token_ep.format(
'practice' if self._is_practice else '')
self.access_data = {}
self._reload_config(config=config)
self._symbol_cache: Dict[str, int] = {}
self._optids2contractinfo = {}
self._contract2ids = {}
# for blocking during token refresh
self._has_access = trio.Event()
self._has_access.set()
self._request_not_in_progress = trio.Event()
self._request_not_in_progress.set()
self._mutex = trio.StrictFIFOLock()
async def main(self, agents):
self.session = asks.Session(
connections=CONCURRENT_REQUESTS,
headers={"Authorization": f"Token {settings.CALLHUB_API_KEY}"},
base_location=settings.CALLHUB_API_DOMAIN,
endpoint="/v1/",
)
# Callhub limite à 13 requêtes par seconde, on prend une mini-marge.
self.bucket = TrioTokenBucket(5, RATE_LIMIT)
self.conn_capacity = trio.CapacityLimiter(CONCURRENT_REQUESTS)
equipes = await self.recuperer_equipes()
for a in agents:
if a["team"] not in equipes:
logger.info(f"Équipe {a['team']} inexistante.")
agents = [a for a in agents if a["team"] in equipes]
existants = await self.recuperer_agents_existants()
a_ajouter = [a for a in agents if a["email"] not in existants]
logger.info(
f"{len(a_ajouter)} agents à ajouter. Délai estimé {self.temps_estime(len(a_ajouter))}."
)
async with trio.open_nursery() as nursery:
for a in a_ajouter:
await nursery.start(self.creer_agent, a)
def __init__(self) -> None:
self._sesh = asks.Session(connections=4)
self._sesh.base_location = _url
self._sesh.headers.update({
'User-Agent':
'krakenex/2.1.0 (+https://github.com/veox/python3-krakenex)'
})
async def test_session_stateful(server):
s = asks.Session(server.http_test_url, persist_cookies=True)
await s.get(cookies={"Test-Cookie": "Test Cookie Value"})
assert ":".join(
str(x) for x in _TEST_LOC) in s._cookie_tracker.domain_dict.keys()
assert (s._cookie_tracker.domain_dict[":".join(
str(x) for x in _TEST_LOC)][0].value == '"Test Cookie Value"')
def __init__(self, tasks, sessionConnections=10):
'''
:param taskList: taskList dict,
{'public':{
mothed:get|post
cookies:cookies字典
headers:headers字典
responseType:text|json|stream 仅在为stream时result保存的是stream的临时文件，否则保存返回的text
json可以为用来标示结果是否为json
}
'taskList':[{
url:'http://www.qq.com',必填
params:get参数字典
data:post数据，字典或是str
#结果数据
responseType:text|json|stream 仅在为stream时result保存的是stream的临时文件，否则保存返回的text
json可以为用来标示结果是否为json
requestOk:1或0,返回的状态码为200即成功。
callbackOk:1,0,如果callback函数成功执行即为1
result:如果respnseType 为 text或 json,结果就是返回的html或json串
如果 reonsetype 为 stream 结果为 stream保存的文件名
如果 responseType 为 空，结果保存在 response里，否则response为空
response:asks.response，只果请求成功,都会有response,否由为None
retryTime:重试次数
error:最后一次重试出错的错误信息
}]}
'''
publicArgs = tasks['public']
self.method = publicArgs['method']
self.cookies = publicArgs[
'cookies'] if 'cookies' in publicArgs else None
self.responseType = publicArgs[
'responseType'] if 'resonseType' in publicArgs else 'text'
self.taskList = tasks['taskList']
self.s = asks.Session(connections=sessionConnections)
self.s.headers.update({
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Accept-Encoding': 'gzip',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive'
})
self.s.headers.update(publicArgs['headers'])
for i in self.taskList:
i.update({
'requestOk': 0,
'result': '',
'retryTime': 0,
'response': None,
'error': '',
'responseType': self.responseType
})
if 'params' not in i:
i.update({'params': None})
if 'data' not in i:
i.update({'data': None})
def __init__(self, account, private_key, *args, max_connections=None, **kwargs):
super().__init__(*args, **kwargs)
if max_connections is None:
max_connections = 30
self._session = asks.Session(connections=max_connections)
self._auth = _BigQueryAuthentication(self._session, account, private_key)
async def test_function():
delay = 1
start = datetime.datetime.now()
session = asks.Session()
client = SlackAPICurio(session=session, token=token)
await client.sleep(delay)
stop = datetime.datetime.now()
return datetime.timedelta(seconds=delay + 1) > (stop - start) > datetime.timedelta(seconds=delay)
async def run(self):
async with self._daemon() as server, asks.Session() as session, connect(
port=TCP_PORT
) as client:
self._server = server
self._session = session
self._client = client
yield self
async def test_session_smallpool(server):
async def worker(s):
r = await s.get(path="/get")
assert r.status_code == 200
s = asks.Session(server.http_test_url, connections=2)
async with create_task_group() as g:
for _ in range(10):
await g.spawn(worker, s)
def __init__(self, version=APPINFO['version_code']):
super(SignUtil, self).__init__()
self.version = version
self.sign = {
"expired": arrow.Arrow(2000, 1, 1, 0, 0, 0),
"common_params": None,
"token": None
}
self.s = asks.Session(_API, connections=5)
async def worker(tasks: List[Tuple[int, float]]) -> None:
count = 0
start = now = time.time()
s = asks.Session()
while (now - start < duration):
await s.get(url)
count += 1
now = time.time()
tasks.append((count, now - start))
def __init__(self, username='', password='', auth=None):
if auth is None:
if username or password:
auth = BasicAuthenticator(None, username, password)
elif username or password:
raise RuntimeError("Conflicting authentication:"
" use user+pass or auth, not both")
self.authenticator = auth
self.websockets = set()
self.session = asks.Session(connections=3)
async def test_session_unknown_kwargs():
session = asks.Session("https://httpbin.org/get")
try:
await session.request("GET", ko=7, foo=0, bar=3, s***e=3)
except TypeError as e:
# yes, i chose "ko" to make the line 79 characters :D
assert e.args == (
"request() got an unexpected keyword argument 'ko'", )
else:
pytest.fail("Passing unknown kwargs does not raise TypeError")
def __init__(self, token: str, **kwargs):
self._token = token
self._session = kwargs.get('session', asks.Session())
self.rate_limiter = RateLimiter()
self._responses = []
self.headers = {
'User-Agent': self.user_agent,
'Authorization': kwargs.get('app', 'Bot') + ' ' + self._token,
}
async def init_session(self):
self.session = asks.Session(connections=config.conns)
self.session.header = {
"User-Agent": random.choice(config.ua_list),
"Accept":
"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01",
"Accept-Encoding": "gzip, deflate, sdch",
"Accept-Language": "zh-CN,zh;q=0.8",
"Referer": "http://www.gpsspg.com",
}
def __init__(self, auth, connections=DEFAULT_CONNECTIONS, session=None, rate_limiter=True, rate=RATE, user=None, logging=False):
self.auth = auth
if session:
self.session = session
else:
self.session = asks.Session(connections=connections, persist_cookies=True)
self.session.base_location = API_BASE
if user:
self.as_user(user)
self.session.headers.update(
{'Authorization': 'Bearer ' + self.auth.token})
self.rate_limiter(RATE, MAX_TOKENS)
async def test_connection_httpbin(api):
url_to_mock = 'http://httpbin.org/get'
Entry.single_register(Entry.GET,
url_to_mock,
body=json.dumps({"args": {}}),
headers={'content-type': 'application/json'})
with Mocketizer():
session = asks.Session()
resp = await session.get(url_to_mock, )
assert "args" in resp.json() and resp["args"] == {}
async def init_session(self):
self.session = asks.Session()
self.session.headers = {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Cache-Control": "no-cache",
"Host": "www.ceic.ac.cn",
"Pragma": "no-cache",
"Proxy-Connection": "keep-alive",
"Referer": "http://news.ceic.ac.cn/index.html",
"Upgrade-Insecure-Requests": "1",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36",
} # 定义访问类header
def run():
url = "http://example.com"
replies.add(replies.GET, url, body=b"test")
calls = [0]
class DummyAdapter(asks.adapters.HTTPAdapter):
def send(self, *a, **k):
calls[0] += 1
return super(DummyAdapter, self).send(*a, **k)
# Test that the adapter is actually used
session = asks.Session()
session.mount("http://", DummyAdapter())
resp = session.get(url, allow_redirects=False)
assert calls[0] == 1
# Test that the response is still correctly emulated
session = asks.Session()
session.mount("http://", DummyAdapter())
resp = session.get(url)
assert_response(resp, "test")
async def run(count: int = 1, batch: int = 1) -> None:
sema = asyncio.Semaphore(batch)
async def get(s: asks.Session) -> bool:
async with sema:
resp = await s.get(URL)
return resp.status_code == 200 and resp.text == 'OK\n'
fails = 0
s = asks.Session(connections=batch)
tasks = [get(s) for _ in range(count)]
for task in asyncio.as_completed(tasks):
fails += not await task
assert fails == 0
def _make_session(lib, loop=None):
if lib == 'asyncio':
try:
import aiohttp
except ImportError:
raise ImportError(
"To use tokage in asyncio mode, it requires the `aiohttp` module."
)
return aiohttp.ClientSession(loop=loop)
try:
import asks
except ImportError:
raise ImportError(
"To use tokage in curio/trio mode, it requires the `asks` module."
)
return asks.Session()
def __init__(
self,
name: str = '',
api_key: str = '',
secret: str = ''
) -> None:
self._sesh = asks.Session(connections=4)
self._sesh.base_location = _url
self._sesh.headers.update({
'User-Agent':
'krakenex/2.1.0 (+https://github.com/veox/python3-krakenex)'
})
self._pairs: list[str] = []
self._name = name
self._api_key = api_key
self._secret = secret
async def curl(self,
url,
params,
data=None,
headers=IPHONE_HEADER,
method='GET',
retries=2,
timeout=CURL_TIMEOUT):
'''抖音的签名请求函数'''
if retries <= 0:
logging.error(
f"curl {url} with method={method} failed, return None!")
return None
try:
# print(params)
s_params = await self.get_signed_params(params)
# print(s_params)
if method.upper() == 'GET':
resp = await self.s.get(url,
params=s_params,
data=data,
headers=IPHONE_HEADER,
verify=False,
timeout=timeout)
elif method.upper() == 'POST':
resp = await self.s.post(url,
params=s_params,
data=data,
headers=IPHONE_HEADER,
verify=False,
timeout=timeout)
else:
logging.error(f"undefined method={method} for url={url}")
return None
logging.debug(
f"curl response from {url} is {resp} with body: {trim(resp.text)}"
)
return resp
except Exception as e:
logging.warning(
f"curl {url} with method={method} failed, retry with new signed params!"
)
await self.get_sign_params(True) # force fresh self.sign
self.s = asks.Session(_API, connections=5) # restart session
return await self.curl(url, params, data, headers, method,
retries - 1)
def __init__(self, url: str, code: str) -> None:
# 5 is a guess, we know we need more than 1 to support the SSE
# subscription but no idea what the real optimum is
self.session = asks.Session(connections=5, persist_cookies=True)
self.uid = f"{int(time.time())}-{secrets.token_hex(3)}"
self.last_event = 0
self.url = url
if code.startswith('~'):
code = code[1:]
# I could validate this for being real @p but I don't want to
if not re.match(r"^([a-z]{6}-){3}[a-z]{6}$", code):
raise ValueError("invalid code format")
self.code = code
self.channel_url = f"{self.url}/~/channel/{self.uid}"
self.subscriptions: Dict[int, Subscription] = {}
self._stream_rv = None
self.deleted = False
self.verbose = True
def __init__(self, index_url, target_dir):
asks.init("trio")
self.index_url = index_url
if "@" in index_url:
from asks import BasicAuth, DigestAuth
index_url = index_url[8:]
parts = index_url.partition("@")
user, pw = parts[0].split(":")
self.auth = BasicAuth((user, pw))
# TODO: use the original protocol instead of hardcoded https
self.index_url = "https://" + parts[2]
else:
self.auth = None
self.target_dir = trio.Path(target_dir)
self.session = asks.Session(connections=8)
self.session.base_location = self.index_url
self.log.debug(f"Base URL: {self.session.base_location}")
self.api = LegacyApi(self.session, self.auth)
async def taskFinish(taskList, taskid, createtime):
errorinfo = []
def clearDIct(sDict):
# 清空dict里空的value,response也清掉
return {k: v for k, v in sDict.items() if v and k != 'response'}
for key, i in enumerate(taskList):
if i['error']:
errorinfo.append(key)
resultDict = {
'taskcount': len(taskList),
'runtime': round(time.time() - createtime, 2),
'errorcount': len(errorinfo),
'errorinfo': json.dumps(errorinfo),
'finished': 1
}
#先并发写taskresult子表数据,api批量写入太慢了
print('------taskResult save start-------')
s = asks.Session(connections=10)
_psotReslt = []
async def taskresultSave(k, jsonData):
_r = await s.post(f'{dbUrl}result', json=jsonData, auth=basicAuth)
try:
_psotReslt.append((k, _r.json()))
except:
_psotReslt.append((k, _r.text))
async with curio.TaskGroup() as g:
for k, i in enumerate(taskList):
await g.spawn(taskresultSave, k, {
'fid': taskid,
'taskid': k,
'result': json.dumps(clearDIct(i))
})
async for _ in g:
print(f"done with {_psotReslt[-1][0]}:{_psotReslt[-1][1]}")
#更新父表数据
_r = await asks.put(f'{dbUrl}/{taskid}', json=resultDict, auth=basicAuth)
return _r.json()
async def load_skyq_channel_data(self) -> None:
"""Load channel data into channel property.
This method fetches the channel list from ``/as/services`` endpoint and
loads the fetched channels into this ``EPG`` object. It then
concurrently fetches each channel's detail info from
``/as/services/details/<sid>`` and adds that to the channels.
Returns:
None
"""
async def _load_chan_detail(detail_url: str, channel: Channel) -> None:
"""Async function to fetch the details of each channel."""
LOGGER.debug(f'Fetching channel details from {detail_url}')
detail_payload_json = await sess.get(detail_url)
LOGGER.debug(f'Got channel details from {detail_url}')
detail_payload = detail_payload_json.json()
LOGGER.debug(f'Parsed JSON from {detail_url}')
channel.load_skyq_detail_data(detail_payload)
LOGGER.debug(f'Merged {detail_url} into {channel}...')
self._channels.append(channel)
LOGGER.debug(f'Added {channel} to EPG')
url = f'http://{self.host}:{self.rest_port}{REST_SERVICES_URL}'
LOGGER.debug(f'Fetching channel list from {url}')
sess = asks.Session(connections=50)
LOGGER.debug(f'About to fetch channel list from sky box.')
chan_payload_json = await sess.get(url)
LOGGER.debug(f'Got channel list from sky box.')
chan_payload = chan_payload_json.json()
LOGGER.debug(f'Parsed channel list JSON payload.')
with trio.move_on_after(90): # 90 sec timeout #TODO parametrise this.
async with trio.open_nursery() as nursery:
LOGGER.debug('Started nursery to collect channel details.')
for single_channel_payload in chan_payload['services']:
LOGGER.debug(f'Processing {single_channel_payload}')
channel = channel_from_skyq_service(single_channel_payload)
sid = channel.sid
detail_url = f'http://{self.host}:{self.rest_port}' + \
f'{REST_SERVICE_DETAIL_URL_PREFIX}{sid}'
nursery.start_soon(_load_chan_detail, detail_url, channel)
