async def test_patch_aiohttp_session(*, execution):
patch_aiohttp_session()
from aiohttp.client import ClientSession
trace_configs = []
async with ClientSession(trace_configs=trace_configs):
assert trace_configs
config = trace_configs[-1]
on_request_start = config.on_request_start[-1]
session = Mock()
ctx = SimpleNamespace()
params = Mock()
params.headers = {}
assert current_test_stack.top is None
await on_request_start(session, ctx, params)
with current_test_stack.push(execution):
await on_request_start(session, ctx, params)
assert params.headers
assert 'LiveCheck-Test-Id' in params.headers
async with ClientSession(trace_configs=None):
pass
async def start(heliotrope: Heliotrope, loop: AbstractEventLoop):
heliotrope.ctx.mongo = (mongo(heliotrope_app.config.MONGO_DB_URL,
io_loop=loop).hitomi.info if mongo else None)
heliotrope.ctx.hitomi_requester = HitomiRequester(ClientSession(loop=loop))
heliotrope.ctx.mirroring_manager = Mirroring(ClientSession(loop=loop),
heliotrope.ctx.mongo)
heliotrope.add_task(heliotrope.ctx.mirroring_manager.mirroring_task(3600))
def _session(self) -> ClientSession:
return ClientSession(connector=ProxyConnector.from_url(
self.http_proxy),
raise_for_status=True,
timeout=ClientTimeout(total=60,
connect=30,
sock_read=10))
async def opensips_cmd(cmd: str) -> dict:
data = {"jsonrpc": "2.0", "method": cmd, "id": opensips_cmd_seq.id}
async with ClientSession() as session:
async with session.post(OSIPS_MI_URL, json=data) as resp:
if resp.status == 200:
return await resp.json()
raise Exception(await resp.text())
async def _make_request(self, context: FContext, payload):
sem = asyncio.Semaphore(200)
conn = aiohttp.TCPConnector(use_dns_cache=True,
loop=self.loop,
limit=0)
async with sem:
async with ClientSession(connector=conn) as session:
try:
if self._timeout > 0:
with async_timeout.timeout(self._timeout / 1000):
async with session.post(self._url,
data=payload,
headers=self._headers) \
as response:
return response.status, await response.content.read(
)
else:
async with session.post(
self._url, data=payload,
headers=self._headers) as response:
return response.status, await response.content.read(
)
except asyncio.TimeoutError:
raise TTransportException(
type=TTransportExceptionType.TIMED_OUT,
message='request timed out')
async def get_client(klass) -> ClientSession:
sess = getattr(klass, '_session')
if sess:
return sess
client = ClientSession()
setattr(klass, '_session', client)
return client
async def get_by_id(cls, acc_id) -> Account:
async with ClientSession() as session:
async with session.get(
f"{settings.ACCOUNT_SERVICE_ADDRESS}/accounts/{acc_id}"
) as resp:
data = await resp.json()
return Account(**data)
async def get_policy_data_config(self,
url: str = None) -> DataSourceConfig:
"""
Get the configuration for
Args:
url: the URL to query for the config, Defaults to self._data_sources_config_url
Returns:
DataSourceConfig: the data sources config
"""
if url is None:
url = self._data_sources_config_url
logger.info("Getting data-sources configuration from '{source}'",
source=url)
try:
async with ClientSession(headers=self._extra_headers) as session:
response = await session.get(url)
if response.status == 200:
return DataSourceConfig.parse_obj(await response.json())
else:
error_details = await response.json()
raise ClientError(
f"Fetch data sources failed with status code {response.status}, error: {error_details}"
)
except:
logger.exception(f"Failed to load data sources config")
raise
async def _make_request(self, context: FContext, payload):
"""
Helper method to make a request over the network.
Args:
payload: The data to be sent over the network.
Return:
The status code and body of the response.
Throws:
TTransportException if the request timed out.
"""
# construct headers for request
request_headers = {}
if self._get_request_headers is not None:
request_headers = self._get_request_headers(context)
# apply the default headers so their values cannot be modified
request_headers.update(self._headers)
timeout = ClientTimeout(total=context.timeout / 1000)
async with ClientSession(timeout=timeout) as session:
try:
async with session.post(self._url,
data=payload,
headers=request_headers) \
as response:
return response.status, await response.content.read()
except ServerTimeoutError:
raise TTransportException(
type=TTransportExceptionType.TIMED_OUT,
message='request timed out')
class ProxyBlobMixin:
"""Provides methods upload and download.
Proxies request to other HTTP service"""
http = ClientSession()
def __init__(self, base, http_session=None):
self.base = base
if http_session:
self.http = http_session or ClientSession()
async def upload(self, stream, content_type=None):
headers = {}
if content_type is not None:
headers['content-type'] = content_type
async with self.http.post(f"{self.base}{self.id}",
data=stream,
headers=headers) as r:
return await r.json()
async def download(self, blobId):
async with self.http.get(f"{self.base}{self.id}/{blobId}") as res:
if res.status == 200:
return await res.read()
elif res.status // 100 == 5:
raise errors.serverFail()
raise errors.notFound(f'Blob {blobId} not found')
async def _check_fresh(self,
dev_db: bool = False,
site_db: bool = False,
template_db: bool = False,
group_db: bool = False):
update_funcs = []
if dev_db:
update_funcs += [self.update_dev_db]
if site_db:
update_funcs += [self.update_site_db]
if template_db:
update_funcs += [self.update_template_db]
if group_db:
update_funcs += [self.update_group_db]
async with ClientSession() as self.central.aio_session:
if update_funcs:
if await update_funcs[0]():
if len(update_funcs) > 1:
await asyncio.gather(*[f() for f in update_funcs[1:]])
# update groups first so template update can use the result, and to trigger token_refresh if necessary
elif await self.update_group_db():
await asyncio.gather(self.update_dev_db(),
self.update_site_db(),
self.update_template_db())
def build_session(self,
apply_for: Optional[ClientSession] = None,
*args,
**kwargs) -> ClientSession:
trace_config = TraceConfig()
trace_config.on_request_start.append(self.on_request_start)
trace_config.on_request_end.append(self.on_request_end)
session = apply_for or ClientSession(
trace_configs=[trace_config], *args, **kwargs)
if apply_for is not None:
session.trace_configs.append(trace_config)
trace_config.freeze()
def hook_method(method):
def hooked_wrapper(*args, **kwargs):
return method(
*args, **{
**kwargs, "trace_request_ctx": (args, kwargs)
})
return hooked_wrapper
session.get = hook_method(session.get)
session.post = hook_method(session.post)
return session
def __init__(
self,
base_url: str,
*,
api_key: Optional[str] = None,
admin_insecure: Optional[bool] = False,
tenant_jwt: Optional[str] = None,
):
if not api_key and not admin_insecure:
raise Exception(
"api_key property is missing. Use admin_insecure=True if you want"
" to use the controller without authentication.")
headers = {}
if api_key:
headers["x-api-key"] = api_key
if tenant_jwt:
headers["authorization"] = f"Bearer {tenant_jwt}"
client_session = ClientSession(
headers=headers,
raise_for_status=True,
)
super().__init__(
base_url,
client=client_session,
extra_service_params={"converter": PydanticConverter()},
)
async def listen_to_runner(runner_url,
artifact_manager,
dput_host,
debsign_keyid: Optional[str] = None,
distributions: Optional[List[str]] = None,
source_only: bool = False):
from aiohttp.client import ClientSession
import urllib.parse
url = urllib.parse.urljoin(runner_url, "ws/result")
async with ClientSession() as session:
async for result in pubsub_reader(session, url):
if result["code"] != "success":
continue
if not result['target']:
continue
if result['target']['name'] != 'debian':
continue
if not distributions or result['target']['details'][
'build_distribution'] in distributions:
await upload_build_result(result['log_id'],
artifact_manager,
dput_host,
debsign_keyid=debsign_keyid,
source_only=source_only)
async def main():
forms: Dict[str, Form] = await get_forms()
for task_code in task2_data:
parsed_task_code: str = parse_task_code(task_code)
if parsed_task_code not in forms:
print(f'Error: task code {parsed_task_code} does not exist')
else:
form: Form = forms[parsed_task_code]
req_range: range = get_range(task2_data[task_code])
exist_range: range = get_range([form.min_year, form.max_year])
no_pdf_set: Set[int] = set(req_range) - set(exist_range)
yes_pdf_set: Set[int] = set(req_range) & set(exist_range)
if len(no_pdf_set):
print(f'{task_code}. No PDFs for year(s): {no_pdf_set}')
if not len(yes_pdf_set):
print('{task_code}. No years to load')
else:
for year in yes_pdf_set:
async with ClientSession() as PdfPage.session:
pdf_page: PdfPage = PdfPage(form.url_year(year))
content: bytes = await pdf_page.content()
folder: str = os.path.normpath(
os.path.join(TASK2_DIR, task_code))
if not os.path.isdir(folder):
os.mkdir(folder)
filename: str = f'{task_code} - {year}.pdf'
file_path: str = os.path.normpath(
os.path.join(folder, filename))
async with aiofiles.open(file_path, mode='wb') as f:
await f.write(content)
print(f'{filename} successfully created')
async def test_borrow_connector_loop(connector: Any, create_session: Any,
loop: Any) -> None:
session = ClientSession(connector=connector)
try:
assert session._loop, loop
finally:
await session.close()
def test_del(self):
conn = self.make_open_connector()
session = ClientSession(loop=self.loop, connector=conn)
with self.assertWarns(ResourceWarning):
del session
gc.collect()
def test_close(self):
conn = self.make_open_connector()
session = ClientSession(loop=self.loop, connector=conn)
session.close()
self.assertIsNone(session.connector)
self.assertTrue(conn.closed)
async def test_client_session_timeout_zero() -> None:
timeout = client.ClientTimeout(total=10, connect=0, sock_connect=0, sock_read=0)
try:
async with ClientSession(timeout=timeout) as session:
await session.get("http://example.com")
except asyncio.TimeoutError:
pytest.fail("0 should disable timeout.")
async def on_message(self, target, source, message):
if not message.startswith(self.nickname + ": "):
return
message = message[len(self.nickname + ": "):]
m = re.match("reschedule (.*)", message)
if m:
await self.message(target, "Rescheduling %s" % m.group(1))
return
if message == "status":
if self._runner_status:
status_strs = [
"%s (%s) since %s" %
(item["package"], item["suite"], item["start_time"])
for item in self._runner_status["processing"]
]
await self.message(
target,
"Currently processing: " + ", ".join(status_strs) + ".")
else:
await self.message(target, "Current runner status unknown.")
if message == "scan":
url = urljoin(self.publisher_url, "scan")
async with ClientSession() as session, session.post(url) as resp:
if resp.status in (200, 202):
await self.message(target, "Merge proposal scan started.")
else:
await self.message(
target,
"Merge proposal scan failed: %d." % resp.status)
async def get_urls_filtered() -> List[str]:
urls_filtered: List[str]
urls_cache: str = os.path.join(CACHE_DIR, 'urls_cache.txt')
if os.path.isfile(urls_cache):
with open(urls_cache, 'r') as f:
urls_filtered = f.readlines()
else:
async with ClientSession() as Page.session:
root_sitemap: RootSitemap = RootSitemap()
sitemap_urls: List[str] = await root_sitemap.urls()
assert sitemap_urls # be sure that root sitemap loaded correctly
sitemaps: List[Sitemap] = []
tasks: List[FetchFuncListStr] = []
for sitemap_url in sitemap_urls:
sitemap: Sitemap = Sitemap(sitemap_url)
sitemaps.append(sitemap)
tasks.append(sitemap.urls())
urls: List[str] = [] # all urls from sitemaps
urls_list: Tuple[List[str]] = await asyncio.gather(*tasks)
for url_list_item in urls_list:
urls += url_list_item
urls_filtered: List[str] = list(filter(url_filter, urls))
with open(urls_cache, 'w') as f:
f.write('\n'.join(urls_filtered))
return urls_filtered
async def invoke_async(self, runs, url, headers, secrets):
results = RunList()
tasks = []
async with ClientSession() as session:
for run in runs:
self.store_run(run)
run.spec.secret_sources = secrets or []
tasks.append(
asyncio.ensure_future(
submit(session, url, run.to_dict(), headers), ))
for status, resp, logs in await asyncio.gather(*tasks):
if status != 200:
logger.error("failed to access {} - {}".format(url, resp))
else:
results.append(json.loads(resp))
if logs:
parsed = parse_logs(logs)
if parsed:
print(parsed, '----------')
return results
def test_init_cookies_with_list_of_tuples(self):
session = ClientSession(cookies=[("c1", "cookie1"), ("c2", "cookie2")],
loop=self.loop)
self.assertEqual(set(session.cookies), {'c1', 'c2'})
self.assertEqual(session.cookies['c1'].value, 'cookie1')
self.assertEqual(session.cookies['c2'].value, 'cookie2')
session.close()
async def _invoke_async(self, tasks, url, headers, secrets, generator):
results = RunList()
runs = []
num_errors = 0
stop = False
parallel_runs = generator.options.parallel_runs or 1
semaphore = asyncio.Semaphore(parallel_runs)
async with ClientSession() as session:
for task in tasks:
# TODO: store run using async calls to improve performance
self.store_run(task)
task.spec.secret_sources = secrets or []
resp = submit(session, url, task, semaphore, headers=headers)
runs.append(asyncio.ensure_future(resp, ))
for result in asyncio.as_completed(runs):
status, resp, logs, task = await result
if status != 200:
err_message = f"failed to access {url} - {resp}"
# TODO: store logs using async calls to improve performance
log_std(
self._db_conn,
task,
parse_logs(logs) if logs else None,
err_message,
silent=True,
)
# TODO: update run using async calls to improve performance
results.append(
self._update_run_state(task=task, err=err_message))
num_errors += 1
else:
if logs:
log_std(self._db_conn, task, parse_logs(logs))
resp = self._update_run_state(json.loads(resp))
state = get_in(resp, "status.state", "")
if state == "error":
num_errors += 1
results.append(resp)
run_results = get_in(resp, "status.results", {})
stop = generator.eval_stop_condition(run_results)
if stop:
logger.info(
f"reached early stop condition ({generator.options.stop_condition}), stopping iterations!"
)
break
if num_errors > generator.max_errors:
logger.error("max errors reached, stopping iterations!")
stop = True
break
if stop:
for task in runs:
task.cancel()
return results
async def query_db():
await Tortoise.init(
db_url=os.environ["DB_URL"],
modules={
"models": [
"heliotrope.database.models.hitomi",
"heliotrope.database.models.requestcount",
]
},
)
await Tortoise.generate_schemas()
hitomi = HitomiRequester(ClientSession())
galleryinfo = await hitomi.get_galleryinfo(1536576)
galleyinfo_orm_object = await GalleryInfo.create(
language_localname=galleryinfo.language_localname,
language=galleryinfo.language,
date=galleryinfo.date,
japanese_title=galleryinfo.japanese_title,
title=galleryinfo.title,
id=galleryinfo.galleryid,
type=galleryinfo.hitomi_type,
)
if galleryinfo.files:
file_orm_object_list = []
for file_info in galleryinfo.files:
file_orm_object = File(
index_id=galleryinfo.galleryid,
width=file_info.get("width"),
hash=file_info.get("hash"),
haswebp=file_info.get("haswebp"),
name=file_info.get("name"),
height=file_info.get("height"),
)
await file_orm_object.save()
file_orm_object_list.append(file_orm_object)
await galleyinfo_orm_object.files.add(*file_orm_object_list)
if galleryinfo.tags:
tag_orm_object_list = []
for tag_info in galleryinfo.tags:
tag_orm_object = Tag(
index_id=galleryinfo.galleryid,
male=tag_info.get("male"),
female=tag_info.get("female"),
url=tag_info.get("url"),
)
await tag_orm_object.save()
tag_orm_object_list.append(tag_orm_object)
await galleyinfo_orm_object.tags.add(*tag_orm_object_list)
await Index.create(index_id="1536576")
info = await hitomi.get_info_using_index(1536576)
mongo = AsyncIOMotorClient(os.environ["MONGO_DB_URL"]).hitomi.info
await mongo.insert_one(info)
await hitomi.session.close()
def test_del(connector, loop, warning):
# N.B. don't use session fixture, it stores extra reference internally
session = ClientSession(connector=connector, loop=loop)
loop.set_exception_handler(lambda loop, ctx: None)
with warning(ResourceWarning):
del session
gc.collect()
def test_connector_loop(self):
loop = asyncio.new_event_loop()
connector = TCPConnector(loop=loop)
with self.assertRaisesRegex(ValueError,
"loop argument must agree with connector"):
ClientSession(connector=connector, loop=self.loop)
connector.close()
loop.close()
def test_del(self):
conn = self.make_open_connector()
session = ClientSession(loop=self.loop, connector=conn)
self.loop.set_exception_handler(lambda loop, ctx: None)
with self.assertWarns(ResourceWarning):
del session
gc.collect()
def __init__(self, *args: Any, **kwargs: Any) -> None:
self.proxy_url = kwargs.pop("proxy_url")
self.permanent_headers = {
"Proxy-Authorization-Key": kwargs.pop("authorization"),
"Accept": "application/json",
}
self._session = ClientSession(*args, **kwargs)
async def get_weather(lat, lon):
weather_api_key = os.environ['WEATHER_API_KEY']
url = 'https://api.openweathermap.org/data/2.5/weather?lat={}&lon={}&appid={}'.format(lat, lon, weather_api_key)
print(url)
async with ClientSession() as session:
async with session.get(url) as response:
result = await response.json()
return result
