async def test_function(): session = asks.Session() client = SlackAPICurio(session=session, token=token) response = await client._request( "POST", "https://slack.com/api//api.test", {}, {"token": token}) return response[0], response[1]
def get_http_session(connections=50): ssl_context = ssl.SSLContext() ssl_context.verify_mode = ssl.CERT_NONE ssl_context.check_hostname = False session = asks.Session(connections=connections, ssl_context=ssl_context) return session
async def ready(self): self.menu_session.headers = config.fake_header self.menu_session.headers.update({"User-Agent": self.random_ua()}) self.async_session = asks.Session(connections=config.max_connections) self.async_session.headers = config.fake_header self.main_url = f"https://{self.domain}/" self.url_list = { "citys": urljoin(self.main_url, "city/"), "old": lambda page: urljoin( self.main_url, f"/{self.current_city}/ershoufang/pg{page}/?_t=1"), "new": lambda page: urljoin( self.main_url, f"/{self.current_city}/loupan/pg{page}/?_t=1&source=index", ), "chuzu": lambda page: urljoin( self.main_url, f"/chuzu/{self.current_city}/zufang/pg{page}/?ajax=1"), "xiaoqu": lambda page: urljoin( self.main_url, f"/{self.current_city}/xiaoqu/pg{page}/?_t=1"), "zhuangxiu": lambda page: urljoin( self.main_url, f"/home/{self.current_city}/zhuangxiu/list/getCompanyInfoList?tagID=&page={page}&pageSize=10&timer={int(time.time())}", ), } list(map(init_path, [config.debug_file, config.datas_file]))
def __init__( self, config: dict, ): # use 2 connections per streaming endpoint (stocks, opts) # TODO: when we have more then one account key then this should scale # linearly with that. self._sess = asks.Session(connections=4) self.api = _API(self) self._conf = config self._is_practice = _use_practice_account or (config['questrade'].get( 'is_practice', False)) self._auth_ep = _refresh_token_ep.format( 'practice' if self._is_practice else '') self.access_data = {} self._reload_config(config=config) self._symbol_cache: Dict[str, int] = {} self._optids2contractinfo = {} self._contract2ids = {} # for blocking during token refresh self._has_access = trio.Event() self._has_access.set() self._request_not_in_progress = trio.Event() self._request_not_in_progress.set() self._mutex = trio.StrictFIFOLock()
async def main(self, agents): self.session = asks.Session( connections=CONCURRENT_REQUESTS, headers={"Authorization": f"Token {settings.CALLHUB_API_KEY}"}, base_location=settings.CALLHUB_API_DOMAIN, endpoint="/v1/", ) # Callhub limite à 13 requêtes par seconde, on prend une mini-marge. self.bucket = TrioTokenBucket(5, RATE_LIMIT) self.conn_capacity = trio.CapacityLimiter(CONCURRENT_REQUESTS) equipes = await self.recuperer_equipes() for a in agents: if a["team"] not in equipes: logger.info(f"Équipe {a['team']} inexistante.") agents = [a for a in agents if a["team"] in equipes] existants = await self.recuperer_agents_existants() a_ajouter = [a for a in agents if a["email"] not in existants] logger.info( f"{len(a_ajouter)} agents à ajouter. Délai estimé {self.temps_estime(len(a_ajouter))}." ) async with trio.open_nursery() as nursery: for a in a_ajouter: await nursery.start(self.creer_agent, a)
def __init__(self) -> None: self._sesh = asks.Session(connections=4) self._sesh.base_location = _url self._sesh.headers.update({ 'User-Agent': 'krakenex/2.1.0 (+https://github.com/veox/python3-krakenex)' })
async def test_session_stateful(server): s = asks.Session(server.http_test_url, persist_cookies=True) await s.get(cookies={"Test-Cookie": "Test Cookie Value"}) assert ":".join( str(x) for x in _TEST_LOC) in s._cookie_tracker.domain_dict.keys() assert (s._cookie_tracker.domain_dict[":".join( str(x) for x in _TEST_LOC)][0].value == '"Test Cookie Value"')
def __init__(self, tasks, sessionConnections=10): ''' :param taskList: taskList dict, {'public':{ mothed:get|post cookies:cookies字典 headers:headers字典 responseType:text|json|stream 仅在为stream时result保存的是stream的临时文件,否则保存返回的text json可以为用来标示结果是否为json } 'taskList':[{ url:'http://www.qq.com',必填 params:get参数字典 data:post数据,字典或是str #结果数据 responseType:text|json|stream 仅在为stream时result保存的是stream的临时文件,否则保存返回的text json可以为用来标示结果是否为json requestOk:1或0,返回的状态码为200即成功。 callbackOk:1,0,如果callback函数成功执行即为1 result:如果respnseType 为 text或 json,结果就是返回的html或json串 如果 reonsetype 为 stream 结果为 stream保存的文件名 如果 responseType 为 空,结果保存在 response里,否则response为空 response:asks.response,只果请求成功,都会有response,否由为None retryTime:重试次数 error:最后一次重试出错的错误信息 }]} ''' publicArgs = tasks['public'] self.method = publicArgs['method'] self.cookies = publicArgs[ 'cookies'] if 'cookies' in publicArgs else None self.responseType = publicArgs[ 'responseType'] if 'resonseType' in publicArgs else 'text' self.taskList = tasks['taskList'] self.s = asks.Session(connections=sessionConnections) self.s.headers.update({ 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', 'Accept-Encoding': 'gzip', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36', 'Cache-Control': 'no-cache', 'Connection': 'keep-alive' }) self.s.headers.update(publicArgs['headers']) for i in self.taskList: i.update({ 'requestOk': 0, 'result': '', 'retryTime': 0, 'response': None, 'error': '', 'responseType': self.responseType }) if 'params' not in i: i.update({'params': None}) if 'data' not in i: i.update({'data': None})
def __init__(self, account, private_key, *args, max_connections=None, **kwargs): super().__init__(*args, **kwargs) if max_connections is None: max_connections = 30 self._session = asks.Session(connections=max_connections) self._auth = _BigQueryAuthentication(self._session, account, private_key)
async def test_function(): delay = 1 start = datetime.datetime.now() session = asks.Session() client = SlackAPICurio(session=session, token=token) await client.sleep(delay) stop = datetime.datetime.now() return datetime.timedelta(seconds=delay + 1) > (stop - start) > datetime.timedelta(seconds=delay)
async def run(self): async with self._daemon() as server, asks.Session() as session, connect( port=TCP_PORT ) as client: self._server = server self._session = session self._client = client yield self
async def test_session_smallpool(server): async def worker(s): r = await s.get(path="/get") assert r.status_code == 200 s = asks.Session(server.http_test_url, connections=2) async with create_task_group() as g: for _ in range(10): await g.spawn(worker, s)
def __init__(self, version=APPINFO['version_code']): super(SignUtil, self).__init__() self.version = version self.sign = { "expired": arrow.Arrow(2000, 1, 1, 0, 0, 0), "common_params": None, "token": None } self.s = asks.Session(_API, connections=5)
async def worker(tasks: List[Tuple[int, float]]) -> None: count = 0 start = now = time.time() s = asks.Session() while (now - start < duration): await s.get(url) count += 1 now = time.time() tasks.append((count, now - start))
def __init__(self, username='', password='', auth=None): if auth is None: if username or password: auth = BasicAuthenticator(None, username, password) elif username or password: raise RuntimeError("Conflicting authentication:" " use user+pass or auth, not both") self.authenticator = auth self.websockets = set() self.session = asks.Session(connections=3)
async def test_session_unknown_kwargs(): session = asks.Session("https://httpbin.org/get") try: await session.request("GET", ko=7, foo=0, bar=3, s***e=3) except TypeError as e: # yes, i chose "ko" to make the line 79 characters :D assert e.args == ( "request() got an unexpected keyword argument 'ko'", ) else: pytest.fail("Passing unknown kwargs does not raise TypeError")
def __init__(self, token: str, **kwargs): self._token = token self._session = kwargs.get('session', asks.Session()) self.rate_limiter = RateLimiter() self._responses = [] self.headers = { 'User-Agent': self.user_agent, 'Authorization': kwargs.get('app', 'Bot') + ' ' + self._token, }
async def init_session(self): self.session = asks.Session(connections=config.conns) self.session.header = { "User-Agent": random.choice(config.ua_list), "Accept": "text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01", "Accept-Encoding": "gzip, deflate, sdch", "Accept-Language": "zh-CN,zh;q=0.8", "Referer": "http://www.gpsspg.com", }
def __init__(self, auth, connections=DEFAULT_CONNECTIONS, session=None, rate_limiter=True, rate=RATE, user=None, logging=False): self.auth = auth if session: self.session = session else: self.session = asks.Session(connections=connections, persist_cookies=True) self.session.base_location = API_BASE if user: self.as_user(user) self.session.headers.update( {'Authorization': 'Bearer ' + self.auth.token}) self.rate_limiter(RATE, MAX_TOKENS)
async def test_connection_httpbin(api): url_to_mock = 'http://httpbin.org/get' Entry.single_register(Entry.GET, url_to_mock, body=json.dumps({"args": {}}), headers={'content-type': 'application/json'}) with Mocketizer(): session = asks.Session() resp = await session.get(url_to_mock, ) assert "args" in resp.json() and resp["args"] == {}
async def init_session(self): self.session = asks.Session() self.session.headers = { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", "Cache-Control": "no-cache", "Host": "www.ceic.ac.cn", "Pragma": "no-cache", "Proxy-Connection": "keep-alive", "Referer": "http://news.ceic.ac.cn/index.html", "Upgrade-Insecure-Requests": "1", "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36", } # 定义访问类header
def run(): url = "http://example.com" replies.add(replies.GET, url, body=b"test") calls = [0] class DummyAdapter(asks.adapters.HTTPAdapter): def send(self, *a, **k): calls[0] += 1 return super(DummyAdapter, self).send(*a, **k) # Test that the adapter is actually used session = asks.Session() session.mount("http://", DummyAdapter()) resp = session.get(url, allow_redirects=False) assert calls[0] == 1 # Test that the response is still correctly emulated session = asks.Session() session.mount("http://", DummyAdapter()) resp = session.get(url) assert_response(resp, "test")
async def run(count: int = 1, batch: int = 1) -> None: sema = asyncio.Semaphore(batch) async def get(s: asks.Session) -> bool: async with sema: resp = await s.get(URL) return resp.status_code == 200 and resp.text == 'OK\n' fails = 0 s = asks.Session(connections=batch) tasks = [get(s) for _ in range(count)] for task in asyncio.as_completed(tasks): fails += not await task assert fails == 0
def _make_session(lib, loop=None): if lib == 'asyncio': try: import aiohttp except ImportError: raise ImportError( "To use tokage in asyncio mode, it requires the `aiohttp` module." ) return aiohttp.ClientSession(loop=loop) try: import asks except ImportError: raise ImportError( "To use tokage in curio/trio mode, it requires the `asks` module." ) return asks.Session()
def __init__( self, name: str = '', api_key: str = '', secret: str = '' ) -> None: self._sesh = asks.Session(connections=4) self._sesh.base_location = _url self._sesh.headers.update({ 'User-Agent': 'krakenex/2.1.0 (+https://github.com/veox/python3-krakenex)' }) self._pairs: list[str] = [] self._name = name self._api_key = api_key self._secret = secret
async def curl(self, url, params, data=None, headers=IPHONE_HEADER, method='GET', retries=2, timeout=CURL_TIMEOUT): '''抖音的签名请求函数''' if retries <= 0: logging.error( f"curl {url} with method={method} failed, return None!") return None try: # print(params) s_params = await self.get_signed_params(params) # print(s_params) if method.upper() == 'GET': resp = await self.s.get(url, params=s_params, data=data, headers=IPHONE_HEADER, verify=False, timeout=timeout) elif method.upper() == 'POST': resp = await self.s.post(url, params=s_params, data=data, headers=IPHONE_HEADER, verify=False, timeout=timeout) else: logging.error(f"undefined method={method} for url={url}") return None logging.debug( f"curl response from {url} is {resp} with body: {trim(resp.text)}" ) return resp except Exception as e: logging.warning( f"curl {url} with method={method} failed, retry with new signed params!" ) await self.get_sign_params(True) # force fresh self.sign self.s = asks.Session(_API, connections=5) # restart session return await self.curl(url, params, data, headers, method, retries - 1)
def __init__(self, url: str, code: str) -> None: # 5 is a guess, we know we need more than 1 to support the SSE # subscription but no idea what the real optimum is self.session = asks.Session(connections=5, persist_cookies=True) self.uid = f"{int(time.time())}-{secrets.token_hex(3)}" self.last_event = 0 self.url = url if code.startswith('~'): code = code[1:] # I could validate this for being real @p but I don't want to if not re.match(r"^([a-z]{6}-){3}[a-z]{6}$", code): raise ValueError("invalid code format") self.code = code self.channel_url = f"{self.url}/~/channel/{self.uid}" self.subscriptions: Dict[int, Subscription] = {} self._stream_rv = None self.deleted = False self.verbose = True
def __init__(self, index_url, target_dir): asks.init("trio") self.index_url = index_url if "@" in index_url: from asks import BasicAuth, DigestAuth index_url = index_url[8:] parts = index_url.partition("@") user, pw = parts[0].split(":") self.auth = BasicAuth((user, pw)) # TODO: use the original protocol instead of hardcoded https self.index_url = "https://" + parts[2] else: self.auth = None self.target_dir = trio.Path(target_dir) self.session = asks.Session(connections=8) self.session.base_location = self.index_url self.log.debug(f"Base URL: {self.session.base_location}") self.api = LegacyApi(self.session, self.auth)
async def taskFinish(taskList, taskid, createtime): errorinfo = [] def clearDIct(sDict): # 清空dict里空的value,response也清掉 return {k: v for k, v in sDict.items() if v and k != 'response'} for key, i in enumerate(taskList): if i['error']: errorinfo.append(key) resultDict = { 'taskcount': len(taskList), 'runtime': round(time.time() - createtime, 2), 'errorcount': len(errorinfo), 'errorinfo': json.dumps(errorinfo), 'finished': 1 } #先并发写taskresult子表数据,api批量写入太慢了 print('------taskResult save start-------') s = asks.Session(connections=10) _psotReslt = [] async def taskresultSave(k, jsonData): _r = await s.post(f'{dbUrl}result', json=jsonData, auth=basicAuth) try: _psotReslt.append((k, _r.json())) except: _psotReslt.append((k, _r.text)) async with curio.TaskGroup() as g: for k, i in enumerate(taskList): await g.spawn(taskresultSave, k, { 'fid': taskid, 'taskid': k, 'result': json.dumps(clearDIct(i)) }) async for _ in g: print(f"done with {_psotReslt[-1][0]}:{_psotReslt[-1][1]}") #更新父表数据 _r = await asks.put(f'{dbUrl}/{taskid}', json=resultDict, auth=basicAuth) return _r.json()
async def load_skyq_channel_data(self) -> None: """Load channel data into channel property. This method fetches the channel list from ``/as/services`` endpoint and loads the fetched channels into this ``EPG`` object. It then concurrently fetches each channel's detail info from ``/as/services/details/<sid>`` and adds that to the channels. Returns: None """ async def _load_chan_detail(detail_url: str, channel: Channel) -> None: """Async function to fetch the details of each channel.""" LOGGER.debug(f'Fetching channel details from {detail_url}') detail_payload_json = await sess.get(detail_url) LOGGER.debug(f'Got channel details from {detail_url}') detail_payload = detail_payload_json.json() LOGGER.debug(f'Parsed JSON from {detail_url}') channel.load_skyq_detail_data(detail_payload) LOGGER.debug(f'Merged {detail_url} into {channel}...') self._channels.append(channel) LOGGER.debug(f'Added {channel} to EPG') url = f'http://{self.host}:{self.rest_port}{REST_SERVICES_URL}' LOGGER.debug(f'Fetching channel list from {url}') sess = asks.Session(connections=50) LOGGER.debug(f'About to fetch channel list from sky box.') chan_payload_json = await sess.get(url) LOGGER.debug(f'Got channel list from sky box.') chan_payload = chan_payload_json.json() LOGGER.debug(f'Parsed channel list JSON payload.') with trio.move_on_after(90): # 90 sec timeout #TODO parametrise this. async with trio.open_nursery() as nursery: LOGGER.debug('Started nursery to collect channel details.') for single_channel_payload in chan_payload['services']: LOGGER.debug(f'Processing {single_channel_payload}') channel = channel_from_skyq_service(single_channel_payload) sid = channel.sid detail_url = f'http://{self.host}:{self.rest_port}' + \ f'{REST_SERVICE_DETAIL_URL_PREFIX}{sid}' nursery.start_soon(_load_chan_detail, detail_url, channel)