async def get_presigned_s3_url(self,
session: aiohttp.ClientSession) -> None:
"""Method to make a request to the object service and pre-sign an S3 PUT
Args:
session: The current aiohttp session
Returns:
None
"""
# Get the object id from the object path
_, obj_id = self.object_details.object_path.rsplit('/', 1)
async with session.put(
f"{self.service_root}/{obj_id}",
headers=self.object_service_headers) as response:
if response.status == 200:
# Successfully signed the request
response_data = await response.json()
self.presigned_s3_url = response_data.get("presigned_url")
self.set_s3_headers(response_data.get("key_id"))
elif response.status == 403:
# Forbidden indicates Object already exists, don't need to re-push since we deduplicate so mark it skip
self.skip_object = True
else:
# Something when wrong while trying to pre-sign the URL.
body = await response.json()
raise IOError(
f"Failed to get pre-signed URL for PUT at {self.object_details.dataset_path}:{obj_id}."
f" Status: {response.status}. Response: {body}")
async def update_status(status: str,
task_id: str,
session: aiohttp.ClientSession,
type: str,
run_endpoint: str,
uuid: Optional[str] = None) -> Any:
"""Updates status of `type` via the orchest-api.
Args:
type: One of ``['pipeline', 'step']``.
"""
data = {'status': status}
if data['status'] == 'STARTED':
data['started_time'] = datetime.utcnow().isoformat()
elif data['status'] in ['SUCCESS', 'FAILURE']:
data['finished_time'] = datetime.utcnow().isoformat()
base_url = f'{CONFIG_CLASS.ORCHEST_API_ADDRESS}/{run_endpoint}/{task_id}'
if type == 'step':
url = f'{base_url}/{uuid}'
elif type == 'pipeline':
url = base_url
async with session.put(url, json=data) as response:
return await response.json()
class HTTP:
def __init__(self):
self.sess = ClientSession(headers={'Authorization': f'Bot {TOKEN}'})
async def init(self):
if self.sess.closed:
self.sess = ClientSession(
headers={'Authorization': f'Bot {TOKEN}'})
async def pong(self, url: str):
async with self.sess.get(url) as resp:
return str(resp.status).startswith('2')
async def send_message(self,
channel: str,
message: str,
allowed_mentions: dict = {}):
async with self.sess.post(BASE + f"/channels/{channel}/messages",
json={
"content": message,
**allowed_mentions
}) as resp:
return await resp.json()
async def add_role(self, guild, user, role):
async with self.sess.put(
BASE + f"/guilds/{guild}/members/{user}/roles/{role}") as resp:
return resp.status == 204
async def del_role(self, guild, user, role):
async with self.sess.delete(
BASE + f"/guilds/{guild}/members/{user}/roles/{role}") as resp:
return resp.status == 204
async def update_status(
status: str,
task_id: str,
session: aiohttp.ClientSession,
type: str,
run_endpoint: str,
uuid: Optional[str] = None,
) -> Any:
"""Updates status of `type` via the orchest-api.
Args:
type: One of ``['pipeline', 'step']``.
"""
data = {"status": status}
if data["status"] == "STARTED":
data["started_time"] = datetime.utcnow().isoformat()
elif data["status"] in ["SUCCESS", "FAILURE"]:
data["finished_time"] = datetime.utcnow().isoformat()
base_url = f"{CONFIG_CLASS.ORCHEST_API_ADDRESS}/{run_endpoint}/{task_id}"
if type == "step":
url = f"{base_url}/{uuid}"
elif type == "pipeline":
url = base_url
async with session.put(url, json=data) as response:
return await response.json()
async def create_shot(session=None, loop=None, keywords=None):
"""
Create/upload a new Page Shot shot.
"""
if session is None:
session = ClientSession(cookies=_COOKIES, loop=loop)
fresh_session = True
else:
fresh_session = False
try:
path = "data/{}/test.com".format(make_random_id())
if path not in _SHOTS:
_SHOTS.append(path)
path_screenshots = urljoin(SERVER_URL, path)
data = make_example_shot(keywords=keywords)
headers = {'content-type': 'application/json'}
async with session.put(path_screenshots, data=json.dumps(data),
headers=headers) as r:
r.path = path
r.session = session
return r
finally:
if fresh_session:
session.close()
async def upload_file_via_put(http: aiohttp.ClientSession,
url: str,
path: Path,
format_: VirtoolFileFormat = None,
params: dict = None):
if not params:
params = {"name": path.name}
if format_ is not None:
params.update(format=format_)
with path.open('rb') as binary:
async with http.put(url, data={"file": binary},
params=params) as response:
async with raising_errors_by_status_code(
response) as response_json:
return VirtoolFile(
id=response_json["id"],
name=response_json["name"],
name_on_disk=(response_json["name_on_disk"]
if "name_on_disk" in response_json else
response_json["name"]),
size=response_json["size"],
uploaded_at=dateutil.parser.isoparse(
response_json["uploaded_at"]),
format=response_json["format"]
if "format" in response_json else "fastq",
)
async def put(session: ClientSession) -> None:
async with session.put(
f"http://{self._host}{path}",
**self._api_kwargs,
json=action.argument
) as response:
response.raise_for_status()
async def _upload_file_to_link(session: ClientSession, url: URL,
file_path: Path):
log.debug("Uploading from %s to %s", file_path, url)
async with session.put(url, data=file_path.open("rb")) as resp:
if resp.status > 299:
response_text = await resp.text()
raise exceptions.S3TransferError(
"Could not upload file {}:{}".format(file_path, response_text))
async def step3(session: ClientSession, token: str, request_id: str,
meme_id: int) -> None:
""" Request to begin processing on the uploaded image """
url = f"https://api.wombo.ai/mobile-app/mashups/{request_id}"
headers = {"Authorization": f"Basic {token}"}
json = {"meme_id": str(meme_id), "premium": False}
async with session.put(url, headers=headers, json=json) as response:
response.raise_for_status()
async def async_submit(sess: aiohttp.ClientSession,
lang,
problem_id,
code=None) -> str:
'''
submit asynchronously `problem_id` with language `lang`
if `code` is "", use default source decided by `lang`
Args:
code: the code path
'''
logging.debug(
f"sending async submission with lang id: {lang} , problem id :{problem_id}"
)
API_BASE = get_api_base()
logging.debug('===submission===')
langs = ['c', 'cpp', 'py', 'hw']
# create submission
async with sess.post(f'{API_BASE}/submission',
json={
'languageType': lang,
'problemId': problem_id
}) as resp:
rj = await resp.json()
rc = resp.status
logging.debug(f"create submission return code:{rc}")
logging.debug(rj)
rj = rj['data']
assert rc == 200
# open code file
if code is "":
# use default
code = open(f'{langs[lang]}-code.zip', 'rb')
else:
# check zip
if not is_zipfile(code):
logging.warning('you are submitting a non-zip file.')
# if it is the path string
if 'read' not in code:
code = open(code, 'rb')
form = aiohttp.FormData(quote_fields=False)
form.add_field("code", code, content_type="multipart/form-data")
# upload source
async with sess.put(f'{API_BASE}/submission/{rj["submissionId"]}',
data=form) as resp2:
status_code = resp2.status
status_text = await resp2.text()
logging.debug(status_code)
logging.debug(status_text)
assert resp2.status == 200, resp2.status
logging.debug('===end===')
return rj["submissionId"]
async def set_execution_status(self, session: aiohttp.ClientSession, execution_id, status_id, attempts=5):
if not attempts:
return f"Status {status_id} was not added to execution {execution_id} in the cycle {self._cycle_id}"
data = {"status": status_id, "changeAssignee": "false"}
async with session.put(
url=urls.set_execution_status(execution_id), data=json.dumps(data), headers=AUTH_HEADERS) as response:
log.info(f"url -> {urls.set_execution_status(execution_id)}, data -> {data}, resp -> {response.status}")
if response.status != 200:
attempts -= 1
return await self.set_execution_status(session, execution_id, status_id, attempts)
return
async def add_group_member(
self,
id_: str,
localpart: str,
*,
session: aiohttp.ClientSession,
) -> None:
async with session.put(
self._admin_v1_endpoint("/groups/{}/members/{}".format(
id_, localpart)), ) as resp:
self._raise_error_from_response(resp)
class Client:
_client = None
def __init__(self, loop, url=None):
self._client = ClientSession(loop=loop)
self._url = url
@property
def cli(self):
return self._client
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc_value, traceback):
pass
def handler_url(self, url):
if url.startswith("http"):
return url
if self._url:
return "{}{}".format(self._url, url)
return url
def request(self, method, url, *args, **kwargs):
return self._client.request(method, self.handler_url(url), *args,
**kwargs)
def get(self, url, allow_redirects=True, **kwargs):
return self._client.get(self.handler_url(url),
allow_redirects=True,
**kwargs)
def post(self, url, data=None, **kwargs):
return self._client.post(self.handler_url(url), data=data, **kwargs)
def put(self, url, data=None, **kwargs):
return self._client.put(self.handler_url(url), data=data, **kwargs)
def delete(self, url, **kwargs):
return self._client.delete(self.handler_url(url), **kwargs)
def head(self, url, allow_redirects=False, **kwargs):
return self._client.head(self.handler_url(url),
allow_redirects=allow_redirects,
**kwargs)
def options(self, url, allow_redirects=True, **kwargs):
return self._client.options(self.handler_url(url),
allow_redirects=allow_redirects,
**kwargs)
def close(self):
self._client.close()
async def send_mojang_request(s: aiohttp.ClientSession, bearer: str,
name: str) -> None:
headers = {
"Content-type": "application/json",
"Authorization": "Bearer " + bearer
}
async with s.put(
f"https://api.minecraftservices.com/minecraft/profile/name/{name}",
headers=headers) as r:
print(f"Response received @ {datetime.now()}"
f" with the status {r.status}")
end.append(datetime.now())
async def subscribe(session: aiohttp.ClientSession) -> str:
headers = {
'X-MPBX-API-AUTH-TOKEN': API_TOKEN,
'Content-Type': 'application/json'
}
data = {"pattern": "*****@*****.**",
"expires": 3600,
"subscriptionType": "BASIC_CALL",
"url": f"http://{IP}/subscription"}
url = 'https://cloudpbx.beeline.ru/apis/portal/subscription'
async with session.put(url=url,
headers=headers,
data=json.dumps(data)
) as response:
data = await response.json()
return data['subscriptionId']
async def _upload_file_to_link(session: aiohttp.ClientSession, url: URL,
file_path: Path):
log.debug("Uploading from %s to %s", file_path, url)
# TODO: PC->SAN. what to do with this?
# with aiohttp.MultipartWriter() as writer:
# writer.append(await aiofiles.open(file_path, 'rb'))
# async with session.put(url, data=writer) as resp:
# if resp.status > 299:
# response_text = await resp.text()
# raise exceptions.S3TransferError("Could not upload file {}:{}".format(file_path, response_text))
async with session.put(url, data=file_path.open('rb')) as resp:
if resp.status > 299:
response_text = await resp.text()
raise exceptions.S3TransferError(
"Could not upload file {}:{}".format(file_path, response_text))
async def update_group(
self,
id_: str,
*,
new_name: typing.Optional[str] = None,
session: aiohttp.ClientSession,
) -> None:
payload = {}
if new_name is not None:
payload["name"] = new_name
async with session.put(
self._admin_v1_endpoint("/groups/{}".format(id_)),
json=payload,
) as resp:
self._raise_error_from_response(resp)
class Qiwi:
def __init__(self, private_key: str):
self.__private_key = private_key
self._session = ClientSession()
self._headers = {
'Authorization': f'Bearer {self.__private_key}',
'Content-Type': 'application/json',
'Accept': 'application/json',
}
async def create_bill(self, value: Union[int, float], currency: str, expiration_date_time: datetime,
customer_phone: Optional[str] = None, customer_email: Optional[str] = None,
customer_account: Optional[str] = None, comment: Optional[str] = None,
custom_fields: Optional[dict] = None) -> CreateBillResponse:
data = CreateBillRequest(
amount=Amount(
value=prepare_amount_value(value),
currency=currency
),
expirationDateTime=expiration_date_time.strftime('%Y-%m-%dT%H:%M:%S+00:00'),
comment=comment,
customer=Customer(
phone=customer_phone,
email=customer_email,
account=customer_account
),
customFields=custom_fields
)
async with self._session.put(f"{ApiMethods.CREATE_BILL}{uuid4().hex}", json=data.dict(),
headers=self._headers) as resp:
result = process_result(CreateBillResponse, await resp.json(), data.dict())
return result
async def check_bill(self, bill_id):
async with self._session.get(f"{ApiMethods.CHECK_BILL}{bill_id}", headers=self._headers) as resp:
result = CreateBillResponse(**(await resp.json()))
return result
async def close_session(self):
await self._session.close()
await asyncio.sleep(.25)
logger.info('QIWI session closed.')
async def get_presigned_s3_url(self, session: aiohttp.ClientSession) -> None:
"""Method to make a request to the object service and pre-sign an S3 PUT
Args:
session: The current aiohttp session
Returns:
None
"""
if self.is_multipart:
if not self.current_part:
raise ValueError("No parts remain to get presigned URL.")
part_num = self.current_part.part_number
url = f"{self.service_root}/{self.object_id}/multipart/{self.multipart_upload_id}/part/{part_num}"
else:
url = f"{self.service_root}/{self.object_id}"
try_count = 0
error_status = None
error_msg = None
while try_count < 3:
async with session.put(url, headers=self.object_service_headers) as response:
if response.status == 200:
# Successfully signed the request
response_data = await response.json()
self.presigned_s3_url = response_data.get("presigned_url")
self.set_s3_headers(response_data.get("key_id"))
return
elif response.status == 403:
# Forbidden indicates Object already exists,
# don't need to re-push since we deduplicate so mark it skip
self.skip_object = True
return
else:
# Something when wrong while trying to pre-sign the URL.
error_msg = await response.json()
error_status = response.status
await asyncio.sleep(try_count ** 2)
try_count += 1
raise IOError(f"Failed to get pre-signed URL for PUT at "
f"{self.object_details.dataset_path}:{self.object_id}."
f" Status: {error_status}. Response: {error_msg}")
async def put_object(self, session: aiohttp.ClientSession) -> None:
"""Method to put the object in S3 after the pre-signed URL has been obtained
Args:
session: The current aiohttp session
Returns:
None
"""
# Set the Content-Length of the PUT explicitly since it won't happen automatically due to streaming IO
headers = copy.deepcopy(self.s3_headers)
# Detect file type if possible
_, object_id = self.object_details.object_path.rsplit("/", 1)
temp_compressed_path = os.path.join(tempfile.gettempdir(), object_id)
# gzip object before upload
try:
with open(self.object_details.object_path, "rb") as src_file:
with open(temp_compressed_path, "wb") as compressed_file:
snappy.stream_compress(src_file, compressed_file)
headers['Content-Length'] = str(
os.path.getsize(temp_compressed_path))
# Stream the file up to S3
async with session.put(
self.presigned_s3_url,
headers=headers,
data=self._file_loader(
filename=temp_compressed_path)) as response:
if response.status != 200:
# An error occurred
body = await response.text()
raise IOError(
f"Failed to push {self.object_details.dataset_path} to storage backend."
f" Status: {response.status}. Response: {body}")
finally:
try:
os.remove(temp_compressed_path)
except FileNotFoundError:
# Temp file never got created
pass
async def put_object(self, session: aiohttp.ClientSession, progress_update_fn: Callable) -> str:
"""Method to put the object in S3 after the pre-signed URL has been obtained
Args:
session: The current aiohttp session
progress_update_fn: A callable with arg "completed_bytes" (int) indicating how many bytes have been
uploaded in since last called
Returns:
None
"""
# Set the Content-Length of the PUT explicitly since it won't happen automatically due to streaming IO
headers = copy.deepcopy(self.s3_headers)
# compress object before upload
if self.is_multipart:
if not self.current_part:
raise ValueError(f"Failed to put_object part. No parts remain for {self.object_id}")
headers['Content-Length'] = str(self.current_part.end_byte - self.current_part.start_byte)
else:
headers['Content-Length'] = str(self.compressed_object_size)
# Stream the file up to S3
try_count = 0
error_msg = None
error_status = None
while try_count < 3:
async with session.put(self.presigned_s3_url, headers=headers,
data=self._file_loader(filename=self.compressed_object_path,
progress_update_fn=progress_update_fn)) as response:
if response.status != 200:
# An error occurred, retry
error_msg = await response.text()
error_status = response.status
await asyncio.sleep(try_count ** 2)
try_count += 1
else:
return response.headers['Etag']
raise IOError(f"Failed to push {self.object_details.dataset_path} to storage backend."
f" Status: {error_status}. Response: {error_msg}")
async def upload_file_to_storage(
app: web.Application,
link_and_path: LinkAndPath2,
user_id: int,
session: ClientSession,
) -> Tuple[LinkAndPath2, ETag]:
try:
upload_url = await get_file_upload_url(
app=app,
location_id=str(link_and_path.storage_type),
fileId=str(link_and_path.relative_path_to_file),
user_id=user_id,
)
except Exception as e:
raise ExporterException(
f"While requesting upload for {str(link_and_path.relative_path_to_file)}"
f"the following error occurred {str(e)}"
) from e
log.debug(">>> upload url >>> %s", upload_url)
async def file_sender(file_name=None):
async with aiofiles.open(file_name, "rb") as f:
chunk = await f.read(64 * 1024)
while chunk:
yield chunk
chunk = await f.read(64 * 1024)
data_provider = file_sender(file_name=link_and_path.storage_path_to_file)
content_size = await path_getsize(link_and_path.storage_path_to_file)
headers = {"Content-Length": str(content_size)}
async with session.put(upload_url, data=data_provider, headers=headers) as resp:
upload_result = await resp.text()
if resp.status != 200:
raise ExporterException(
f"Client replied with status={resp.status} and body '{upload_result}'"
)
e_tag = json.loads(resp.headers.get("Etag", None))
log.debug(
"Upload status=%s, result: '%s', Etag %s", resp.status, upload_result, e_tag
)
return (link_and_path, e_tag)
async def update_user(
self,
localpart: str,
*,
display_name: typing.Optional[str],
roles: typing.Optional[typing.Collection[str]],
session: aiohttp.ClientSession,
) -> None:
payload: typing.Dict[str, typing.Any] = {
"username": localpart,
}
if display_name is not None:
payload["display_name"] = display_name
if roles is not None:
payload["roles"] = list(roles)
async with session.put(
self._admin_v1_endpoint("/users/{}".format(localpart)),
json=payload,
) as resp:
self._raise_error_from_response(resp)
async def _upload_file_to_link(session: ClientSession, url: URL,
file_path: Path) -> Optional[ETag]:
log.debug("Uploading from %s to %s", file_path, url)
file_size = file_path.stat().st_size
async def file_sender(file_name: Path):
with tqdm(
desc=f"uploading {file_path} [{file_size} bytes]",
total=file_size,
unit="byte",
unit_scale=True,
) as pbar:
async with aiofiles.open(file_name, "rb") as f:
chunk = await f.read(CHUNK_SIZE)
while chunk:
pbar.update(len(chunk))
yield chunk
chunk = await f.read(CHUNK_SIZE)
data_provider = file_sender(file_path)
headers = {"Content-Length": f"{file_size}"}
async with session.put(url, data=data_provider, headers=headers) as resp:
if resp.status > 299:
response_text = await resp.text()
raise exceptions.S3TransferError(
"Could not upload file {}:{}".format(file_path, response_text))
if resp.status != 200:
response_text = await resp.text()
raise exceptions.S3TransferError(
"Issue when uploading file {}:{}".format(
file_path, response_text))
# get the S3 etag from the headers
e_tag = json.loads(resp.headers.get("Etag", ""))
log.debug("Uploaded %s to %s, received Etag %s", file_path, url, e_tag)
return e_tag
async def get_upload_file_presigned_link(session: ClientSession, file_id: str,
location_id: str,
user_id: UserID) -> AnyUrl:
if (not isinstance(file_id, str) or not isinstance(location_id, str)
or not isinstance(user_id, int)):
raise exceptions.StorageInvalidCall(
f"invalid call: user_id '{user_id}', location_id '{location_id}', file_id '{file_id}' are invalid",
)
if file_id is None or location_id is None or user_id is None:
raise exceptions.StorageInvalidCall(
f"invalid call: user_id '{user_id}', location_id '{location_id}', file_id '{file_id}' are not allowed to be empty",
)
async with session.put(
f"{_base_url()}/locations/{location_id}/files/{quote(file_id, safe='')}",
params={"user_id": f"{user_id}"},
) as response:
response.raise_for_status()
presigned_link_enveloped = Envelope[PresignedLink].parse_obj(
await response.json())
if presigned_link_enveloped.data is None:
raise exceptions.StorageServerIssue(
"Storage server is not reponding")
return presigned_link_enveloped.data.link
async def _ping(self, session: aiohttp.ClientSession):
async with session.put(self.url, json=self.body_content) as resp:
resp.raise_for_status()
logger.debug("Successfully pinged heartbeat server.")
class ApiInstance:
def __init__(self,
base_url: str,
timeout: int = 5000,
headers: dict = {},
*,
logger: logging.Logger):
self.base_url = base_url
self.headers = headers
self.timeout = timeout
self.session = ClientSession(timeout=ClientTimeout(total=timeout))
self.logger = logger
async def __aenter__(self) -> "ApiInstance":
return self
async def __aexit__(
self,
exc_type: Optional[Type[BaseException]],
exc_val: Optional[BaseException],
exc_tb: Optional[TracebackType],
) -> None:
await self.close()
@staticmethod
def check_result(res: Any) -> Any:
if res == "":
return None
return json_loads_attrs(res)
async def call(self,
method: str,
prefix: str,
data: Any,
headers: dict = {}):
new_headers = CIMultiDict()
new_headers.update(self.headers)
new_headers.update(headers)
data_binary = json.dumps(data).encode("utf-8")
# TODO: this is too much code duplication but I cannot think of
# a way outside macros that could abstract async with block
# and sadly there are no macro in python
if method == "get":
async with self.session.get(self.base_url + "/" + prefix,
headers=new_headers) as resp:
await check_response(resp, self.logger)
res = await resp.text()
return self.check_result(res)
elif method == "post":
async with self.session.post(self.base_url + "/" + prefix,
data=data_binary,
headers=new_headers) as resp:
await check_response(resp, self.logger)
res = await resp.text()
return self.check_result(res)
elif method == "put":
async with self.session.put(self.base_url + "/" + prefix,
data=data_binary,
headers=new_headers) as resp:
await check_response(resp, self.logger)
res = await resp.text()
return self.check_result(res)
elif method == "patch":
async with self.session.patch(self.base_url + "/" + prefix,
data=data_binary,
headers=new_headers) as resp:
await check_response(resp, self.logger)
res = await resp.text()
return self.check_result(res)
elif method == "delete":
async with self.session.delete(self.base_url + "/" + prefix,
headers=new_headers) as resp:
await check_response(resp, self.logger)
res = await resp.text()
return self.check_result(res)
async def make_request(self, method: str, prefix: str, data: Any,
headers: dict):
try:
return await self.call(method, prefix, data, headers)
except (ConflictingEntityException, EntityNotFoundException,
PermissionDeniedException, ProcedureInvocationException,
UnauthorizedException, ValidationException,
BadRequestException, PapieaServerException, ApiException):
raise
except:
self.logger.debug("RENEWING SESSION")
await self.renew_session()
return await self.call(method, prefix, data, headers)
async def post(self, prefix: str, data: Any, headers: dict = {}) -> Any:
return await self.make_request("post", prefix, data, headers)
async def put(self, prefix: str, data: Any, headers: dict = {}) -> Any:
return await self.make_request("put", prefix, data, headers)
async def patch(self, prefix: str, data: Any, headers: dict = {}) -> Any:
return await self.make_request("patch", prefix, data, headers)
async def get(self, prefix: str, headers: dict = {}) -> Any:
return await self.make_request("get", prefix, {}, headers)
async def delete(self, prefix: str, headers: dict = {}) -> Any:
return await self.make_request("delete", prefix, {}, headers)
async def close(self):
await self.session.close()
async def renew_session(self):
await self.close()
self.session = ClientSession(timeout=ClientTimeout(total=self.timeout))
class AsyncUTMTestSession:
"""
Requests Asyncio client session that provides additional functionality for running DSS concurrency tests:
* Adds a prefix to URLs that start with a '/'.
* Automatically applies authorization according to adapter, when present
"""
def __init__(self, prefix_url: str, auth_adapter: Optional[AuthAdapter] = None):
self._client = None
loop = asyncio.get_event_loop()
loop.run_until_complete(self.build_session())
self._prefix_url = prefix_url[0:-1] if prefix_url[-1] == '/' else prefix_url
self.auth_adapter = auth_adapter
self.default_scopes = None
async def build_session(self):
self._client = ClientSession()
def close(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._client.close())
def adjust_request_kwargs(self, url, method, kwargs):
if self.auth_adapter:
scopes = None
if 'scopes' in kwargs:
scopes = kwargs['scopes']
del kwargs['scopes']
if 'scope' in kwargs:
scopes = [kwargs['scope']]
del kwargs['scope']
if scopes is None:
scopes = self.default_scopes
if not scopes:
raise ValueError('All tests must specify auth scope for all session requests. Either specify as an argument for each individual HTTP call, or decorate the test with @default_scope.')
headers = {}
for k, v in self.auth_adapter.get_headers(url, scopes).items():
headers[k] = v
kwargs['headers'] = headers
if method == 'PUT' and kwargs.get('data'):
kwargs['json'] = kwargs['data']
del kwargs['data']
return kwargs
async def put(self, url, **kwargs):
url = self._prefix_url + url
if 'auth' not in kwargs:
kwargs = self.adjust_request_kwargs(url, 'PUT', kwargs)
async with self._client.put(url, **kwargs) as response:
return response.status, await response.json()
async def get(self, url, **kwargs):
url = self._prefix_url + url
if 'auth' not in kwargs:
kwargs = self.adjust_request_kwargs(url, 'GET', kwargs)
async with self._client.get(url, **kwargs) as response:
return response.status, await response.json()
async def post(self, url, **kwargs):
url = self._prefix_url + url
if 'auth' not in kwargs:
kwargs = self.adjust_request_kwargs(url, 'POST', kwargs)
async with self._client.post(url, **kwargs) as response:
return response.status, await response.json()
async def delete(self, url, **kwargs):
url = self._prefix_url + url
if 'auth' not in kwargs:
kwargs = self.adjust_request_kwargs(url, 'DELETE', kwargs)
async with self._client.delete(url, **kwargs) as response:
return response.status, await response.json()
async def notify_mattermost_header(
mattermost_config: Dict,
app_session: ClientSession,
state: State,
status_message: str,
):
if mattermost_config["enabled"]:
status_emoji = ":+1: "
if state is State.FAILED:
status_emoji = ":x:"
elif state is State.PAUSED:
status_emoji = ":x:"
header_unique_name = mattermost_config["header_unique_name"]
date = datetime.datetime.utcnow().isoformat(timespec="seconds")
message = f"{header_unique_name} {status_emoji} {status_message} - {date} |"
personal_token = mattermost_config["personal_token"]
channel_id = mattermost_config["channel_id"]
headers = {"Authorization": f"Bearer {personal_token}"}
# get the current header to update it
url = URL(mattermost_config["url"]).with_path(
f"api/v4/channels/{channel_id}")
current_header = ""
async with app_session.get(url, headers=headers) as resp:
log.debug("requested channel description: received with code %s",
resp.status)
if resp.status == 404:
log.error("could not find route in %s", url)
raise ConfigurationError(
"Could not find channel within Mattermost app in {}:\n {}".
format(url, await resp.text()))
if not resp.status == 200:
log.error("Unknown error")
raise AutoDeployAgentException(
"Unknown error while accessing Mattermost app in {}:\n {}".
format(url, await resp.text()))
data = await resp.json()
log.debug("received data: %s", data)
current_header = data["header"]
new_header = message
start_index = current_header.find(header_unique_name)
if start_index != -1:
# update the message instead
lastindex = current_header.find("|", start_index)
new_header = "{}{}{}".format(current_header[0:start_index],
message,
current_header[lastindex + 1:])
url = URL(mattermost_config["url"]).with_path(
f"api/v4/channels/{channel_id}/patch")
async with app_session.put(url,
headers=headers,
data=json.dumps({"header":
new_header})) as resp:
log.debug(
"requested patch channel description: response received with code %s",
resp.status,
)
if resp.status == 200:
data = await resp.json()
return data
if resp.status == 404:
log.error("could not find route in %s", url)
raise ConfigurationError(
"Could not find channel within Mattermost app in {}:\n {}".
format(url, await resp.text()))
log.error("Unknown error")
raise AutoDeployAgentException(
"Unknown error while accessing Mattermost app in {}:\n {}".
format(url, await resp.text()))
class AioHttpClient(HttpClient):
def __init__(self,
*,
connector=None,
loop=None,
cookies=None,
headers=None,
skip_auto_headers=None,
auth=None,
json_serialize=json.dumps,
request_class=ClientRequest,
response_class=ClientResponse,
ws_response_class=ClientWebSocketResponse,
version=http.HttpVersion11,
cookie_jar=None,
connector_owner=True,
raise_for_status=False,
read_timeout=sentinel,
conn_timeout=None,
auto_decompress=True,
trust_env=False,
**kwargs):
"""
The class packaging a class ClientSession to perform HTTP request and manager that these HTTP connection.
For details of the params: http://aiohttp.readthedocs.io/en/stable/client_advanced.html#client-session
"""
super(AioHttpClient, self).__init__(**kwargs)
self.client = ClientSession(connector=connector,
loop=loop,
cookies=cookies,
headers=headers,
skip_auto_headers=skip_auto_headers,
auth=auth,
json_serialize=json_serialize,
request_class=request_class,
response_class=response_class,
ws_response_class=ws_response_class,
version=version,
cookie_jar=cookie_jar,
connector_owner=connector_owner,
raise_for_status=raise_for_status,
read_timeout=read_timeout,
conn_timeout=conn_timeout,
auto_decompress=auto_decompress,
trust_env=trust_env)
def request(self, method, url, *args, **kwargs):
return self.client.request(method=method, url=url, **kwargs)
def get(self, url, *args, **kwargs):
return self.client.get(url=url, **kwargs)
def post(self, url, *args, data=None, **kwargs):
return self.client.post(url=url, data=data, **kwargs)
def put(self, url, *args, data=None, **kwargs):
return self.client.put(url=url, data=data, **kwargs)
def delete(self, url, *args, **kwargs):
return self.client.delete(url=url, **kwargs)
def options(self, url, *args, **kwargs):
return self.client.options(url=url, **kwargs)
def head(self, url, *args, **kwargs):
return self.client.head(url=url, **kwargs)
def patch(self, url, *args, data=None, **kwargs):
return self.client.patch(url=url, data=data, **kwargs)
async def close(self):
await self.client.close()
async def get_response(self, response):
text = await response.text()
return Response(url=response.url,
status=response.status,
charset=response.charset,
content_type=response.content_type,
content_length=response.content_length,
reason=response.reason,
headers=response.headers,
text=text,
selector=etree.HTML(text))
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
await self.close()