def test_multipart_invalid_value(value):
client = Client(dispatch=MockDispatch())
data = {"text": value}
files = {"file": io.BytesIO(b"<file content>")}
with pytest.raises(TypeError) as e:
client.post("http://127.0.0.1:8000/", data=data, files=files)
assert "Invalid type for value" in str(e.value)
def _get_token(self, client: Client, concourse_url: str) -> str:
concourse_login = f"{concourse_url}/sky/login"
r = client.get(concourse_login, follow_redirects=True)
ldap_url = expect(
re.search(_LDAP_URL_REGEX, r.text),
"BUG: no ldap url found"
).group(0)
ldap_login_url = f"{concourse_url}{ldap_url}"
print("Concourse login")
username = input("Username: "******"login": username, "password": password}
r = client.post(ldap_login_url, data=data, follow_redirects=True)
token = expect(
re.search(_BEARER_REGEX, r.text),
"BUG: no bearer found"
).group(1)
return token
def get_slots(
entityId,
medicalStaffId,
reasonId,
start_date,
end_date,
client: httpx.Client = DEFAULT_CLIENT,
request: ScraperRequest = None,
):
base_url = ORDOCLIC_API.get("slots")
payload = {
"entityId": entityId,
"medicalStaffId": medicalStaffId,
"reasonId": reasonId,
"dateEnd": f"{end_date}T00:00:00.000Z",
"dateStart": f"{start_date}T23:59:59.000Z",
}
headers = {"Content-type": "application/json", "Accept": "text/plain"}
if request:
request.increase_request_count("slots")
try:
r = client.post(base_url, data=json.dumps(payload), headers=headers)
r.raise_for_status()
except httpx.TimeoutException as hex:
logger.warning(f"request timed out for center: {base_url}")
return False
except httpx.HTTPStatusError as hex:
logger.warning(f"{base_url} returned error {hex.response.status_code}")
return None
return r.json()
def send_message_to_channel(text: str,
service_id: str,
client: httpx.Client = None):
"""Huey task to send message to specific payload for specific slack service
Args:
payload (dict): payload for serivce
service_id (str): service id of the service (ex of service: Webhook)
client (httpx.Client, optional): client to send requests. Defaults to None.
Raises:
e: httpx.HTTPStatusError
"""
if settings.SLACK_NOTIF_IS_ACTIVE:
if not client:
client = get_default_client()
try:
data = {"text": text}
response = client.post(f"{BASE_URL}{service_id}", json=data)
response.raise_for_status()
logger.info("send message to slack")
logger.info(response.json())
return response.json()
except httpx.HTTPStatusError as e:
logger.error("Error while fetching `%s`: %s", e.request.url, e)
raise e
def _send_chunk(client: httpx.Client, config: Config, file: ResumableFile,
chunk: FileChunk) -> bool:
"""Upload the chunk to the server.
Returns
-------
bool
True if the upload was successful
Raises
------
ResumableError
If the server responded with an error code indicating permanent failure
"""
try:
response = client.post(
config.target,
data=_build_query(file, chunk, config.additional_query_params),
files={"file": chunk.read()},
)
except httpx.TransportError:
# The request itself failed. The calling function will retry.
return False
if response.status_code in config.permanent_errors:
raise ResumableError(f"{response.status_code} Error: {response.text}")
return response.status_code in [200, 201]
def refresh_grant(client: httpx.Client,
token_uri: typing.Union["URL", str, RawURL],
refresh_token: str,
scopes: str = None):
payload = {
"refresh_token": refresh_token,
}
# if scopes:
# payload["scope"] = scopes
logger.debug(f"HTTP Request: POST {token_uri} {client.headers} {payload}")
response = client.post(url=token_uri, json=payload)
response_data = response.json()
try:
access_token = response_data["access_token"]
except KeyError as caught_exc:
new_exc = exceptions.RefreshError("No access token in response.",
response_data)
raise new_exc from caught_exc
refresh_token = response_data.get("refresh_token", refresh_token)
expiry = _parse_expiry(response_data)
return access_token, refresh_token, expiry, response_data
def get_slots(entityId,
medicalStaffId,
reasonId,
start_date,
end_date,
client: httpx.Client = DEFAULT_CLIENT):
base_url = "https://api.ordoclic.fr/v1/solar/slots/availableSlots"
payload = {
"entityId": entityId,
"medicalStaffId": medicalStaffId,
"reasonId": reasonId,
"dateEnd": f"{end_date}T00:00:00.000Z",
"dateStart": f"{start_date}T23:59:59.000Z",
}
headers = {"Content-type": "application/json", "Accept": "text/plain"}
try:
r = client.post(base_url, data=json.dumps(payload), headers=headers)
r.raise_for_status()
except httpx.TimeoutException as hex:
logger.warning(f"request timed out for center: {base_url}")
return False
except httpx.HTTPStatusError as hex:
logger.warning(f"{base_url} returned error {hex.response.status_code}")
return None
return r.json()
def _submit2(self, json, client: httpx.Client):
if self.message_id is None:
message = client.post(f'{self.webhook}?wait=true', json=json)
self.message_id = message.json()['id']
else:
client.patch(f'{self.webhook}/messages/{self.message_id}',
json=json)
def get_availabilities_week(
reason_id: int, organization_id: int, start_date: datetime, client: httpx.Client = DEFAULT_CLIENT
) -> Optional[list]:
url = AVECMONDOC_API.get("availabilities_per_day", "")
payload = {
"consultationReasonId": reason_id,
"disabledPeriods": [],
"fullDisplay": True,
"organizationId": organization_id,
"periodEnd": (start_date + timedelta(days=(AVECMONDOC_DAYS_PER_PAGE - 1))).strftime("%Y-%m-%dT%H:%M:%S.000Z"),
"periodStart": start_date.strftime("%Y-%m-%dT%H:%M:%S.000Z"),
"type": "inOffice",
}
headers = {"Content-type": "application/json", "Accept": "application/json"}
try:
r = client.post(url, data=json.dumps(payload), headers=headers)
r.raise_for_status()
except httpx.TimeoutException as hex:
logger.warning(f"request timed out for center: {url}")
return None
except httpx.HTTPStatusError as hex:
logger.warning(f"{url} returned error {hex.response.status_code}")
logger.warning(payload)
return None
return r.json()
class HTTPClient:
def __init__(
self,
base_url: str,
default_headers: Optional[dict] = None,
default_params: Optional[dict] = None,
):
self.base_url = base_url
self.default_headers = default_headers or {}
self.default_params = default_params or {}
self.http_client = Client(
base_url=self.base_url, headers=default_headers, params=self.default_params
)
def get(self, url: str, params: dict, headers: dict = None):
custom_headers = headers or {}
if not params.get("_rticket"):
params["_rticket"] = int(round(time() * 1000))
response = self.http_client.get(url=url, params=params, headers=custom_headers)
return response
def post(self, url: str, data: dict, headers: dict = None):
custom_headers = headers or {}
rticket = int(round(time() * 1000))
response = self.http_client.post(
url=url, params={"_rticket": rticket}, data=data, headers=custom_headers
)
return response
def commitPushedDate(
client: httpx.Client, org: str, commit: GitHubRepoCommit
) -> Optional[str]:
ql = """query ($org: String!, $repo: String!, $hash: String!) {
repository(owner: $org, name: $repo) {
object(expression: $hash) {
... on Commit {
pushedDate
}
}
}
}"""
result = client.post(
QL_ENDPOINT,
json={
"query": ql,
"variables": {"org": org, "repo": commit.name, "hash": commit.commit_hash,},
},
)
if result.status_code == 200:
body: Dict = result.json()
if body.get("errors", None) is not None:
print(body["errors"])
return None
else:
commit_obj: Dict[str, Any] = body["data"]["repository"]["object"]
return commit_obj["pushedDate"]
def admin_login(protocol: str,
host: str,
port: str,
username: str = None,
password: str = None,
suppress_exception: bool = False,
no_admin: bool = False) -> str:
"""Login and get an access token
Args:
protocol (str): "http://" or "https://". Defaults to None. # noqa: E501
host (str): homeserver address. Defaults to None.
port (int): homeserver listening port. Defaults to None.
username (str, optional): just username. Defaults to None.
password (str, optional): just password. Defaults to None.
suppress_exception (bool, optional): suppress exception or not, if not return False and the error in dict. Defaults to False. # noqa: E501
Returns:
str: access token
"""
if username is None:
username = input("Enter a username: "******"identifier": {
"type": "m.id.user",
"user": username
},
"type": "m.login.password",
"password": password,
"initial_device_display_name": "matrix-synapse-admin"
}
http = Client()
base_url = f"{protocol}{host}:{port}"
while True:
resp = http.post(f"{base_url}{ClientAPI.BASE_PATH}/login",
json=login_data)
data = resp.json()
if "errcode" in data and data["errcode"] == "M_LIMIT_EXCEEDED":
time.sleep(data["retry_after_ms"] / 1000)
continue
if resp.status_code == 200:
access_token = data["access_token"]
if not no_admin:
resp = User(host, port, access_token,
protocol).query(username)
if "errcode" not in resp:
return data["access_token"]
else:
data = resp
else:
return access_token
if suppress_exception:
return False, data
else:
raise SynapseException(data["errcode"], data["error"])
def test_remove_path():
with WebServer('test').start() as server:
client = Client(base_url=server.local_url())
@http_endpoint('POST', '/{a:int}/foo')
async def get_foo(request):
a = request.path_params['a']
b = await request.json()
return JSONResponse(a * b)
assert client.post('/12/foo',
json=15).status_code == HTTP_404_NOT_FOUND
assert client.post('/12/bar',
json=15).status_code == HTTP_404_NOT_FOUND
with server.patch_http_endpoint(get_foo):
resp = client.post('/12/foo', json=15)
resp.raise_for_status()
assert resp.json() == 15 * 12
assert client.post('/12/foo',
json=15).status_code == HTTP_404_NOT_FOUND
assert client.post('/12/bar',
json=15).status_code == HTTP_404_NOT_FOUND
with raises(HTTPError):
client.post('/12/foo', json=15)
def getSlots(entityId, medicalStaffId, reasonId, start_date, end_date, client: httpx.Client = DEFAULT_CLIENT):
base_url = 'https://api.ordoclic.fr/v1/solar/slots/availableSlots'
payload = {"entityId": entityId,
"medicalStaffId": medicalStaffId,
"reasonId": reasonId,
"dateEnd": f"{end_date}T00:00:00.000Z",
"dateStart": f"{start_date}T23:59:59.000Z"}
headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
r = client.post(base_url, data=json.dumps(payload), headers=headers)
r.raise_for_status()
return r.json()
def viewerIsOrgMember(client: httpx.Client, org: str) -> Optional[bool]:
ql = """query($org: String!){
organization(login:$org){
viewerIsAMember
}
}"""
result = client.post(QL_ENDPOINT, json={"query": ql, "variables": {"org": org}})
if result.status_code == 200:
body: Dict = result.json()
if body.get("errors", None) is not None:
print(body["errors"])
return False
else:
org: Dict[str, Any] = body["data"]["organization"]
return org["viewerIsAMember"]
return None
def request_new_grant_with_post_kbank_special(
self, url: str, data, grant_name: str,
client: httpx.Client) -> Tuple[str, int]:
with client:
header = {"Content-Type": "application/x-www-form-urlencoded"}
response = client.post(url, data=data, headers=header)
if response.is_error:
# As described in https://tools.ietf.org/html/rfc6749#section-5.2
raise InvalidGrantRequest(response)
content = response.json()
token = content.get(grant_name)
if not token:
raise GrantNotProvided(grant_name, content)
return token, content.get("expires_in")
async def test_tutor_signup_with_email(client: httpx.Client):
signup_func = lambda: client.post(
"/signup",
json={
"full_name": "Danny::Novak",
"email": "*****@*****.**",
"gender": "M",
"dob": "1991-12-16", # year-month-day
"password": "******",
# sent in addition to signup info to skip email verification.
"signup_info": {
"verification": None
},
},
)
async with models.User.database:
await clean_db()
response = await signup_func()
assert response.status_code == 200
result: dict = response.json()
access_token = result["data"]["access_token"]
# decode jwt token
user_data = jwt.decode(access_token, verify=False)
user_info = {
"email": "*****@*****.**",
"gender": "M",
"full_name": "Danny::Novak",
"first_name": "Danny",
"birthday": "1991-12-16",
}
for key in user_info.keys():
assert user_data[key] == user_info[key]
record = await models.User.objects.get(email=user_data["email"])
assert record.email == "*****@*****.**"
assert record.gender.value == "M"
assert not record.email_verified
# second attempt to signup should result in an error.
response = await signup_func()
assert response.status_code == 400
assert response.json() == {
"status": False,
"errors": {
"email": ["value_error.duplicate"]
},
}
class Requests:
"""Client for requests"""
def __init__(self):
self._client = Client()
def post(
self,
*,
url: str,
params: Optional[Dict] = None,
body: Optional[Dict] = None,
data: Optional[Dict] = None,
files: Optional[Dict] = None,
) -> Response:
"""Send post request"""
return self._client.post(url=url,
params=params,
json=body,
data=data,
files=files)
def request_new_grant_with_post_scb_special(
self, url: str, data, grant_name: str,
client: httpx.Client) -> Tuple[str, int]:
with client:
header = {
"Content-Type": "application/json",
"resourceOwnerId": self.client_id,
"requestUId": uuid.uuid4().hex,
"accept-language": "EN",
}
response = client.post(url, json=data, headers=header)
if response.is_error:
# As described in https://tools.ietf.org/html/rfc6749#section-5.2
raise InvalidGrantRequest(response)
content = response.json().get("data")
token = content.get(grant_name)
if not token:
raise GrantNotProvided(grant_name, content)
return token, content.get("expiresIn")
def post_method(url: str,
postdata=None,
postjson=None,
headers: dict = None,
timeout=5,
max_retries=5,
c: httpx.Client = None):
"""
timeout: 超时时间, 单位秒(s), 默认为 5 秒, 为 `None` 时禁用
max_retries: 最大尝试次数, 默认为 5 次, 为 0 时禁用
"""
k = 1
while (k <= max_retries) or (max_retries == 0):
try:
if c is not None:
res = c.post(url,
data=postdata,
json=postjson,
headers=headers,
timeout=timeout)
else:
res = httpx.post(url,
data=postdata,
json=postjson,
headers=headers,
timeout=timeout)
except Exception as e:
k = k + 1
print(sys._getframe().f_code.co_name + ": " + str(e))
time.sleep(1)
continue
else:
break
try:
return res
except Exception:
sys.exit(sys._getframe().f_code.co_name + ": " +
"Max retries exceeded")
def test_multipart_file_tuple():
client = Client(dispatch=MockDispatch())
# Test with a list of values 'data' argument, and a tuple style 'files' argument.
data = {"text": ["abc"]}
files = {"file": ("name.txt", io.BytesIO(b"<file content>"))}
response = client.post("http://127.0.0.1:8000/", data=data, files=files)
assert response.status_code == 200
# We're using the cgi module to verify the behavior here, which is a
# bit grungy, but sufficient just for our testing purposes.
boundary = response.request.headers["Content-Type"].split("boundary=")[-1]
content_length = response.request.headers["Content-Length"]
pdict = {
"boundary": boundary.encode("ascii"),
"CONTENT-LENGTH": content_length
}
multipart = cgi.parse_multipart(io.BytesIO(response.content), pdict)
# Note that the expected return type for text fields
# appears to differs from 3.6 to 3.7+
assert multipart["text"] == ["abc"] or multipart["text"] == [b"abc"]
assert multipart["file"] == [b"<file content>"]
def install_component(component_file: str = Argument(
..., help="The path to the component file containing the data")):
"""Function to install a component:
- Takes as input a YAML file
- Parses it as a dictionary through the Pydantic model
TunableComponentsModel
- Send it as a POST request to /components
"""
api_client = Client(
base_url=f"http://{api_settings.api_host}:{api_settings.api_port}",
proxies={},
)
component = TunableComponentsModel.from_yaml(component_file)
logger.debug(f"Sending component data {component.dict()}"
"to endpoint"
f"http: // {api_settings.api_host}: {api_settings.api_port}"
f"{api_settings.component_endpoint}")
request = api_client.post(api_settings.component_endpoint,
json=component.dict())
logger.info("Successfully registered components.")
if not 200 <= request.status_code < 400:
raise Exception("Could not create component with status code"
f"{request.status_code}")
class RegisterCommands(object):
"""This object contains methods to help registering a command to Discord.
While you shouldn't need to import this directly, it's still accessible if you
prefer not to initalize a Dispike object.
Important to remember all methods are not async.
"""
def __init__(self, application_id: str, bot_token: str):
"""Initalize object provided with application_id and a bot token
Args:
application_id (str): Client ID
bot_token (str): Bot user Token
"""
self.__bot_token = bot_token
self._application_id = application_id
self._client = Client(
base_url=f"https://discord.com/api/v8/applications/{self._application_id}/",
event_hooks={
"response": [dispike_httpx_event_hook_incoming_request],
"request": [dispike_httpx_event_hook_outgoing_request],
},
)
@logger.catch(reraise=True, message="Issue with bulk overrwriting commands")
def bulk_overwrite_commands(
self,
commands: typing.List[DiscordCommand],
guild_only: bool = False,
guild_to_target: int = None,
):
"""Bulk OVERWRITE commands to specific guilds or globally.
Args:
commands (typing.List[DiscordCommand]): List of new commands (these commands will be overwritten)
guild_only (bool, optional): Default to set global mode (True). Set to False to let the function know to expect a guild_id
guild_to_target (int, optional): A guild Id if guild_only is set to True.
"""
if guild_only == True:
if guild_to_target is None:
raise TypeError(
"if guild_only is set to true, a guild id must be provided."
)
logger.info(f"Targeting a specific guild -> {guild_to_target}")
_request_url = f"guilds/{guild_to_target}/commands"
else:
_request_url = f"commands"
_commands_to_json = [command.dict(exclude_none=True) for command in commands]
_send_request = self._client.put(
url=_request_url, json=_commands_to_json, headers=self.request_headers
)
if _send_request.status_code == 200:
logger.info(
f"Overwritten {len(_send_request.json())} commands.. Recieved ({len(_commands_to_json)}"
)
return True
else:
logger.debug(
f"BULK Overwrite failed ({guild_only} = {guild_to_target}): Body: {_commands_to_json}.. Status code: {_send_request.status_code}"
)
raise DiscordAPIError(_send_request.status_code, _send_request.text)
def register(
self, command: DiscordCommand, guild_only=False, guild_to_target: int = None
):
"""Register a completed `DiscordCommand` model to Discord API.
Args:
command (DiscordCommand): A properly configured DiscordCommand
guild_only (bool, optional): Default to set global mode (True). Set to False to let the function know to expect a guild_id
guild_to_target (int, optional): A guild Id if guild_only is set to True.
"""
if guild_only == True:
if guild_to_target is None:
raise TypeError(
"if guild_only is set to true, a guild id must be provided."
)
logger.info(f"Targeting a specific guild -> {guild_to_target}")
_request_url = f"guilds/{guild_to_target}/commands"
else:
_request_url = f"commands"
try:
_command_to_json = command.dict(exclude_none=True)
_send_request = self._client.post(
_request_url, headers=self.request_headers, json=_command_to_json
)
if _send_request.status_code in [200, 201]:
return True
raise DiscordAPIError(_send_request.status_code, _send_request.text)
except Exception:
raise
@property
def request_headers(self):
"""Return a valid header for authorization
Returns:
dict: a valid header for authorization
"""
return {"Authorization": f"Bot {self.__bot_token}"}
@property
def bot_token(self):
"""You cannot view the bot_token directly, but you can still 'update' it.
Raises:
PermissionError: If you attempt to view the bot token without a new value
"""
raise PermissionError("You cannot view the bot token directly.")
@bot_token.setter
def bot_token(self, new_bot_token):
if new_bot_token == self.__bot_token:
raise TypeError("Bot token already set to that.")
self.__bot_token = new_bot_token
class DevApi:
"""
A class to access the dev.to API
"""
BASE_URL = "https://dev.to/api"
def __init__(self, api_key: Text):
"""
Creates the HTTP client
"""
self.api_key = api_key
self.client = Client(headers=[("Api-Key", self.api_key)])
def url(self, path: Text) -> Text:
"""
Generates an URL, be careful not to put any slash at the start or the
end.
"""
return f"{self.BASE_URL}/{path}"
def get_my_articles(self, publication: Text = "all") -> Iterator[Dict]:
"""
Returns an iterator over all the articles corresponding to the
publication filter.
:param publication: Publication status. Allowed values are "published",
"unpublished" and "all"
:return: An iterator of all selected articles
"""
assert publication in {"published", "unpublished", "all"}
url = self.url(f"articles/me/{publication}")
class NoMorePages(Exception):
"""
A way to communicate that there is no more page coming up from the
API and that the polling of pages should stop now.
"""
def get_page(page: int):
"""
Returns a given page. Pages are 1-indexed.
"""
r = self.client.get(url, params={"page": page})
stop = True
for article in r.json():
stop = False
yield article
if stop:
raise NoMorePages
for i in range(1, 1000):
try:
yield from get_page(i)
except NoMorePages:
return
def find_article(self, key: DevKey) -> Optional[Dict]:
"""
Finds the first article matching they key. Let's take a moment to note
that this method is really approximate but since we can't retrofit the
API ID into the Markdown file it's the only decent way to go.
"""
for article in self.get_my_articles():
if article[key.name] == key.value:
return article
def create_article(self, parser: DevParser) -> None:
"""
Creates an article based on the parsed file.
"""
url = self.url("articles")
if "title" not in parser.front_matter:
raise DevApiError(
"Cannot create an article with no `title` in the front matter")
r = self.client.post(
url,
json={
"title": parser.front_matter["title"],
"body_markdown": parser.file_content,
},
)
r.raise_for_status()
def update_article(self, parser: DevParser, article_id: int) -> None:
"""
Updates an article based on the parsed file and an existing ID.
"""
url = self.url(f"articles/{article_id}")
r = self.client.put(url, json={"body_markdown": parser.file_content})
r.raise_for_status()
class Request:
"""Start a requests session and provide helper methods for HTTP verbs.
"""
def __init__(self):
self.session = Client(timeout=10)
self.headers = self.client.AuthHeader
def InjectAttrs(self, **kwargs):
"""Add custom headers if requested, default authorisation header included in __init__.
Args:
headers (dict): Key value pairs for headers
"""
if kwargs.get("headers"):
self.headers = {**self.headers, **kwargs.get("headers")}
if kwargs.get("start"):
self.body.update({"start": kwargs.get("start")})
@retry(
retry_on_exceptions=(RetryException),
max_calls_total=3,
retry_window_after_first_call_in_seconds=10,
)
@basicauth
def GET(self, url, params={}, headers: dict = {}) -> Response:
"""HTTP GET Request
Args:
url (str): URL to connect to.
headers (dict, optional): Key Value pairs for additional headers. (default: None)
Returns:
Response -- HTTPX Response object
"""
self.InjectAttrs(headers=headers)
try:
response = self.session.get(url,
params=params,
headers=self.headers)
return response
except _exceptions.TimeoutException:
raise RetryException
@retry(
retry_on_exceptions=(RetryException),
max_calls_total=3,
retry_window_after_first_call_in_seconds=10,
)
@contenttype
@basicauth
def PATCH(
self,
url: str,
body=None,
headers: dict = {},
idempotent: bool = False,
) -> Response:
"""HTTP PATCH Request
Args:
url (str): URL to connect to.
body (dict, optional): JSON payload for the request. (default: dict())
headers (dict, optional): Key Value pairs for additional headers. (default: None)
idempotent (bool, optional): Is this request idempotent? (default: False)
Returns:
Response -- HTTPX Response object
"""
self.InjectAttrs(headers=headers)
try:
response = self.session.patch(url, data=body, headers=self.headers)
return response
except _exceptions.TimeoutException:
if idempotent is False:
raise IdempotentTimeout(method="PATCH",
url=url,
data=body,
headers=self.headers)
else:
print("retry")
raise RetryException
@retry(
retry_on_exceptions=(RetryException),
max_calls_total=3,
retry_window_after_first_call_in_seconds=10,
)
@contenttype
@basicauth
def PUT(self, url: str, body: dict = {}, headers: dict = {}) -> Response:
"""HTTP PUT Request
Args:
url (str): URL to connect to.
body (dict, optional): JSON payload for the request. (default: dict())
headers (dict, optional): Key Value pairs for additional headers. (default: None)
Returns:
Response -- HTTPX Response object
"""
self.InjectAttrs(headers=headers)
try:
response = self.session.put(url, data=body, headers=self.headers)
return response
except _exceptions.TimeoutException:
raise RetryException
@retry(
retry_on_exceptions=(RetryException),
max_calls_total=3,
retry_window_after_first_call_in_seconds=10,
)
@contenttype
@basicauth
def POST(
self,
url: str,
body: str = None,
params: dict = None,
headers: dict = {},
idempotent: bool = False,
) -> Response:
"""HTTP POST Request
Args:
url (str): URL to connect to.
body (dict, optional): JSON payload for the request. (default: dict())
params (dict, optional): Dictionary for path parameters. (default: None)
headers (dict, optional): Key Value pairs for additional headers. (default: None)
idempotent (bool, optional): Is this request idempotent? (default: False)
Returns:
Response -- HTTPX Response object
"""
self.InjectAttrs(headers=headers)
try:
response = self.session.post(url,
data=body,
params=params,
headers=self.headers)
return response
except _exceptions.TimeoutException:
if idempotent is False:
raise IdempotentTimeout(method="POST",
url=url,
body=body,
headers=self.headers)
else:
print("retry")
raise RetryException
@retry(
retry_on_exceptions=(RetryException),
max_calls_total=3,
retry_window_after_first_call_in_seconds=10,
)
@contenttype
@basicauth
def DELETE(self,
url: str,
body: dict = {},
headers: dict = {}) -> Response:
"""HTTP DELETE Request
Args:
url (str): URL to connect to.
body (dict, optional): JSON payload for the request. (default: dict())
headers (dict, optional): Key Value pairs for additional headers. (default: None)
Returns:
Response -- HTTPX Response object
"""
self.InjectAttrs(headers=headers)
try:
response = self.session.delete(url, headers=self.headers)
return response
except _exceptions.TimeoutException:
raise RetryException
class ZebrunnerAPI(metaclass=Singleton):
authenticated = False
def __init__(self, service_url: str = None, access_token: str = None):
if service_url and access_token:
self.service_url = service_url.rstrip("/")
self.access_token = access_token
self._client = Client()
self._auth_token = None
self.authenticated = False
def _sign_request(self, request: Request) -> Request:
request.headers["Authorization"] = f"Bearer {self._auth_token}"
return request
def auth(self) -> None:
if not self.access_token or not self.service_url:
return
url = self.service_url + "/api/iam/v1/auth/refresh"
try:
response = self._client.post(
url, json={"refreshToken": self.access_token})
except httpx.RequestError as e:
logger.warning("Error while sending request to zebrunner.",
exc_info=e)
return
if response.status_code != 200:
log_response(response, logging.ERROR)
return
self._auth_token = response.json()["authToken"]
self._client.auth = self._sign_request
self.authenticated = True
def start_test_run(self, project_key: str,
body: StartTestRunModel) -> Optional[int]:
url = self.service_url + "/api/reporting/v1/test-runs"
try:
response = self._client.post(url,
params={"projectKey": project_key},
json=body.dict(exclude_none=True,
by_alias=True))
except httpx.RequestError as e:
logger.warning("Error while sending request to zebrunner.",
exc_info=e)
return None
if response.status_code != 200:
log_response(response, logging.ERROR)
return None
return response.json()["id"]
def start_test(self, test_run_id: int,
body: StartTestModel) -> Optional[int]:
url = self.service_url + f"/api/reporting/v1/test-runs/{test_run_id}/tests"
try:
response = self._client.post(url,
json=body.dict(exclude_none=True,
by_alias=True))
except httpx.RequestError as e:
logger.warning("Error while sending request to zebrunner.",
exc_info=e)
return None
if response.status_code != 200:
log_response(response, logging.ERROR)
return None
return response.json()["id"]
def finish_test(self, test_run_id: int, test_id: int,
body: FinishTestModel) -> None:
url = self.service_url + f"/api/reporting/v1/test-runs/{test_run_id}/tests/{test_id}"
try:
response = self._client.put(url,
json=body.dict(exclude_none=True,
by_alias=True))
except httpx.RequestError as e:
logger.warning("Error while sending request to zebrunner.",
exc_info=e)
return
if response.status_code != 200:
log_response(response, logging.ERROR)
def finish_test_run(self, test_run_id: int) -> None:
url = self.service_url + f"/api/reporting/v1/test-runs/{test_run_id}"
try:
response = self._client.put(
url,
json={
"endedAt":
(datetime.utcnow().replace(tzinfo=timezone.utc) -
timedelta(seconds=1)).isoformat()
},
)
except httpx.RequestError as e:
logger.warning("Error while sending request to zebrunner.",
exc_info=e)
return
if response.status_code != 200:
log_response(response, logging.ERROR)
return
def send_logs(self, test_run_id: int, logs: List[LogRecordModel]) -> None:
url = self.service_url + f"/api/reporting/v1/test-runs/{test_run_id}/logs"
body = [x.dict(exclude_none=True, by_alias=True) for x in logs]
self._client.post(url, json=body)
def send_screenshot(self, test_run_id: int, test_id: int,
image_path: Union[str, Path]) -> None:
url = self.service_url + f"/api/reporting/v1/test-runs/{test_run_id}/tests/{test_id}/screenshots"
with open(image_path, "rb") as image:
self._client.post(
url,
content=image.read(),
headers={
"Content-Type": "image/png",
"x-zbr-screenshot-captured-at": round(time.time() * 1000)
},
)
def send_artifact(self,
filename: Union[str, Path],
test_run_id: int,
test_id: Optional[int] = None) -> None:
if test_id:
url = f"{self.service_url}/api/reporting/v1/test-runs/{test_run_id}/tests/{test_id}/artifacts"
else:
url = f"{self.service_url}/api/reporting/v1/test-runs/{test_run_id}/artifacts"
with open(filename, "rb") as file:
self._client.post(url, files={"file": file})
def send_artifact_references(self,
references: List[ArtifactReferenceModel],
test_run_id: int,
test_id: Optional[int] = None) -> None:
if test_id:
url = f"{self.service_url}/api/reporting/v1/test-runs/{test_run_id}/tests/{test_id}/artifact-references"
else:
url = f"{self.service_url}/api/reporting/v1/test-runs/{test_run_id}/artifact-references/"
json_items = [
item.dict(exclude_none=True, by_alias=True) for item in references
]
self._client.put(url, json={"items": json_items})
def send_labels(self,
labels: List[LabelModel],
test_run_id: int,
test_id: Optional[int] = None) -> None:
if test_id:
url = f"{self.service_url}/api/reporting/v1/test-runs/{test_run_id}/tests/{test_id}/labels"
else:
url = f"{self.service_url}/api/reporting/v1/test-runs/{test_run_id}/labels"
labels_json = [
label.dict(exclude_none=True, by_alias=True) for label in labels
]
self._client.put(url, json={"items": labels_json})
def start_test_session(self, test_run_id: int,
body: StartTestSessionModel) -> Optional[str]:
url = self.service_url + f"/api/reporting/v1/test-runs/{test_run_id}/test-sessions"
response = self._client.post(url,
json=body.dict(exclude_none=True,
by_alias=True))
if not response.status_code == 200:
log_response(response, logging.ERROR)
return None
return response.json().get("id")
def finish_test_session(self, test_run_id: int, zebrunner_id: str,
body: FinishTestSessionModel) -> None:
url = self.service_url + f"/api/reporting/v1/test-runs/{test_run_id}/test-sessions/{zebrunner_id}"
self._client.put(url, json=body.dict(exclude_none=True, by_alias=True))
def get_rerun_tests(self, run_context: str) -> RerunDataModel:
url = self.service_url + "/api/reporting/v1/run-context-exchanges"
run_context_dict = json.loads(run_context)
response = self._client.post(url, json=run_context_dict)
response_data = response.json()
for test in response_data["tests"]:
correlation_data = test["correlationData"]
if correlation_data is not None:
test["correlationData"] = json.loads(correlation_data)
return RerunDataModel(**response_data)
def close(self) -> None:
self._client.close()
class RegisterCommands(object):
"""This object contains methods to help registering a command to Discord.
While you shouldn't need to import this directly, it's still accessible if you
prefer not to initalize a Dispike object.
Important to remember all methods are not async.
"""
def __init__(self, application_id: str, bot_token: str):
"""Initalize object provided with application_id and a bot token
Args:
application_id (str): Client ID
bot_token (str): Bot user Token
"""
self.__bot_token = bot_token
self._application_id = application_id
self._client = Client(
base_url=
f"https://discord.com/api/v8/applications/{self._application_id}/")
def register(self,
command: DiscordCommand,
guild_only=False,
guild_to_target: int = None):
"""Register a completed `DiscordCommand` model to Discord API.
Args:
command (DiscordCommand): A properly configured DiscordCommand
guild_only (bool, optional): Default to set global mode (True). Set to False to let the function know to expect a guild_id
guild_to_target (int, optional): A guild Id if guild_only is set to True.
"""
if guild_only == True:
if guild_to_target is None:
raise TypeError(
"if guild_only is set to true, a guild id must be provided."
)
logger.info(f"Targeting a specific guild -> {guild_to_target}")
_request_url = f"guilds/{guild_to_target}/commands"
else:
_request_url = f"commands"
try:
_command_to_json = command.dict(exclude_none=True)
_send_request = self._client.post(_request_url,
headers=self.request_headers,
json=_command_to_json)
if _send_request.status_code in [200, 201]:
return True
raise DiscordAPIError(_send_request.status_code,
_send_request.text)
except Exception:
raise
@property
def request_headers(self):
"""Return a valid header for authorization
Returns:
dict: a valid header for authorization
"""
return {"Authorization": f"Bot {self.__bot_token}"}
@property
def bot_token(self):
"""You cannot view the bot_token directly, but you can still 'update' it.
Raises:
PermissionError: If you attempt to view the bot token without a new value
"""
raise PermissionError("You cannot view the bot token directly.")
@bot_token.setter
def bot_token(self, new_bot_token):
if new_bot_token == self.__bot_token:
raise TypeError("Bot token already set to that.")
self.__bot_token = new_bot_token
class SHAManExperiment:
"""This class represents an optimization experiment.
It allows to plan, launch and store the information corresponding to
an experiment.
"""
def __init__(
self,
component_name: str,
nbr_iteration: int,
sbatch_file: str,
experiment_name: str,
configuration_file: str,
sbatch_dir: str = None,
slurm_dir: str = None,
result_file: str = None,
) -> None:
"""Builds an object of class SHAManExperiment. This experiment is
defined by giving:
- The name of the component to tune.
- The number of allowed iterations for the optimization process.
- The name of the sbatch file to run.
- The name of the experiment.
- A configuration file to setup the experiment.
- Where the slurm outputs should be stored
(optional, if not specified, the outputs are deleted)
- The path to the result file (optional, if not specified,
no file is created).
Args:
component_name (str): The name of the component to use.
nbr_iteration (int): The number of iterations.
sbatch_file (str): The path to the sbatch file.
experiment_name (str): The name of the experiment.
sbatch_dir (str or Path): The working directory,
where the shaman sbatch will be stored.
If not specified, the directory where the command is called.
slurm_dir (str or Path): The directory where the slurm outputs
will be stored.
If set to None, all the slurm outputs are removed after the
end of the experiment.
result_file (str): The path to the result file.
If set to None, no such file is created and the results are
debuged to the screen.
configuration_file (str): The path to the configuration file.
Defaults to the configuration file present in the package
and called config.cfg.
"""
# The name of the component that will be tuned through the experiment
self.component_name = component_name
# The maximum number of iterations
self.nbr_iteration = nbr_iteration
# The name of the sbatch to use, after ensuring its existence
if Path(sbatch_file).exists():
self.sbatch_file = sbatch_file
else:
raise FileNotFoundError(f"Could not find sbatch {sbatch_file}.")
# Store information about the experiment
self.experiment_name = experiment_name
# Store the sbatch directory and convert to Path if not already
if sbatch_dir:
if isinstance(sbatch_dir, Path):
self.sbatch_dir = sbatch_dir
else:
self.sbatch_dir = Path(sbatch_dir)
else:
self.sbatch_dir = Path.cwd()
# Store the slurm directory and convert to Path if not already
if slurm_dir:
if isinstance(slurm_dir, Path):
self.slurm_dir = slurm_dir
else:
self.slurm_dir = Path(slurm_dir)
else:
self.slurm_dir = None
self.result_file = result_file
self.experiment_id = ""
# Parse the configuration
self.configuration = SHAManConfig.from_yaml(
configuration_file, self.component_name
)
# Create API client using the configuration information
self.api_client = Client(base_url=self.api_url, proxies={})
# Create the black box object using the informations
self.bb_wrapper = BBWrapper(
self.component_name,
self.configuration.component_parameter_names,
self.sbatch_file,
sbatch_dir=self.sbatch_dir,
component_configuration=self.api_url
+ "/"
+ api_settings.component_endpoint,
)
logger.debug(self.api_url + "/" + api_settings.component_endpoint)
# Setup black box optimizer using configuration information
self.bb_optimizer = self.setup_bb_optimizer()
# Compute the start of the experiment
self.experiment_start = \
datetime.datetime.utcnow().strftime("%y/%m/%d %H:%M:%S")
def setup_bb_optimizer(self) -> BBOptimizer:
"""Setups the black-box from the configuration."""
# Create the BBOptimizer object using the different options in the
# configuration
# If pruning is enabled, parse corresponding fields
if self.configuration.pruning:
pruning = True
max_step_cost = (
self.bb_wrapper.default_target_value
if self.configuration.pruning.max_step_duration == "default"
else self.configuration.pruning.max_step_duration
)
else:
max_step_cost = None
pruning = False
self.bb_optimizer = BBOptimizer(
black_box=self.bb_wrapper,
parameter_space=self.configuration.component_parameter_space,
max_iteration=self.nbr_iteration,
async_optim=pruning,
max_step_cost=max_step_cost,
**self.configuration.bbo_parameters,
)
return self.bb_optimizer
def launch(self) -> None:
"""Launches the tuning experiment."""
logger.debug("Creating experiment.")
# Create the experiment through API request
self.create_experiment()
if self.configuration.experiment.default_first:
# Launch a run using default parameterization of the component
logger.info("Running application with default parametrization.")
self.bb_wrapper.run_default()
# Setup the optimizer
logger.debug("Initializing black box optimizer.")
self.setup_bb_optimizer()
# Launch the optimization
logger.debug("Launching optimization.")
self.bb_optimizer.optimize(callbacks=[self.update_history])
# Summarize the optimization
# If there is a result file, write the output in it
if self.result_file:
orig_stdout = sys.stdout
file_ = open(self.result_file, "w")
sys.stdout = file_
self.summarize()
sys.stdout = orig_stdout
file_.close()
logger.debug(f"Summary of experiment: {self.summarize()}")
def clean(self) -> None:
"""Cleans the experiment by removing the sbatch generated by Shaman.
If there is no value specified for the slurm outputs folder,
removes them.
"""
for file_ in Path(__CURRENT_DIR__).glob("slurm*.out"):
if self.slurm_dir:
# Create if it doesn't already exist the folder
self.slurm_dir.mkdir(exist_ok=True)
# Move the output
file_.rename(self.slurm_dir / file_.name)
else:
# Remove the output
file_.unlink()
# Clean up the sbatch file in the current dir where the experiment runs
# Else keep it
for file_ in Path(__CURRENT_DIR__).glob("*_shaman*.sbatch"):
(Path(__CURRENT_DIR__) / file_).unlink()
@property
def api_url(self) -> str:
"""Returns as a string the URL to the API using the data in the
configuration file."""
return f"http://{api_settings.api_host}:{api_settings.api_port}"
@property
def start_experiment_dict(self) -> Dict:
"""Creates a dictionnary describing the experiment from its start."""
return InitExperiment(
**{
"experiment_name": self.experiment_name,
"experiment_start": self.experiment_start,
"experiment_budget": self.nbr_iteration,
"component": self.component_name,
"experiment_parameters": dict(self.configuration.bbo),
"noise_reduction_strategy":
dict(self.configuration.noise_reduction)
if self.configuration.noise_reduction
else dict(),
"pruning_strategy": {
"pruning_strategy": str(
self.configuration.pruning.max_step_duration
)
if self.configuration.pruning
else self.configuration.pruning
},
"sbatch": open(self.sbatch_file, "r").read(),
}
).dict()
def create_experiment(self) -> None:
"""Create the experiment upon initialization."""
request = self.api_client.post(
"experiments", json=self.start_experiment_dict)
if not 200 <= request.status_code < 400:
raise Exception(
"Could not create experiment with status code"
f"{request.status_code}"
)
self.experiment_id = request.json()["id"]
@property
def best_performance(self) -> float:
"""Returns the current best performance.
Returns:
float: The best time.
"""
return self.bb_optimizer._get_best_performance()
@property
def improvement_default(self) -> float:
"""Computes the improvement compared to the default parametrization.
Returns:
float: The improvement compared to the default parametrization.
"""
_, best_time = self.best_performance
return float(
round(
(self.bb_wrapper.default_target_value - best_time)
/ self.bb_wrapper.default_target_value,
2,
)
* 100
)
@property
def average_noise(self) -> float:
"""Computes the average noise within the tested parametrization.
Returns:
float: The average measured noise.
"""
return float(np.mean(self.bb_optimizer.measured_noise))
def _updated_dict(self, history: Dict) -> Dict:
"""Builds the updated dictionary that will be sent at each iteration to
the API.
Args:
history (Dict): The BBO history from the optimizer
Returns:
Dict: The updated dict to add to the POST request.
"""
logger.debug(f"Current optimization history: {history}")
best_parameters, best_fitness = self.best_performance
logger.debug(f"Best parameters so far: {best_parameters}")
logger.debug(f"Best performance so far: {best_fitness}")
return IntermediateResult(
**{
"jobids": self.bb_wrapper.component.submitted_jobids[-1],
"fitness": list(history["fitness"])[-1],
"parameters": self.build_parameter_dict(
self.configuration.component_parameter_names,
history["parameters"].tolist(),
)[-1],
"truncated": bool(list(history["truncated"])[-1]),
"resampled": bool(list(history["resampled"])[-1]),
"initialization": bool(list(history["initialization"])[-1]),
"improvement_default": self.improvement_default,
"average_noise": self.average_noise,
"explored_space":
float(self.bb_optimizer.size_explored_space[0]),
"best_parameters": self.build_parameter_dict(
self.configuration.component_parameter_names,
best_parameters.tolist(),
),
"best_fitness": best_fitness,
}
).dict()
def update_history(self, history: Dict) -> None:
"""Update the optimization history at each BBO step and force
conversion from numpy types.
Args:
history (dict): The BBO history
"""
logger.debug(
f"Writing update dictionary {self._updated_dict(history)}")
request = self.api_client.put(
f"experiments/{self.experiment_id}/update",
json=self._updated_dict(history)
)
if not 200 <= request.status_code < 400:
self.fail()
raise Exception(
"Could not finish experiment with status code"
f"{request.status_code}"
)
@property
def end_dict(self) -> Dict:
"""Comptues the dictionary sent to the API at the end of the
experiment.
Returns:
Dict: The dictionary to send to the API.
"""
best_parameters, best_fitness = self.best_performance
return FinalResult(
**{
"averaged_fitness": self.bb_optimizer.averaged_fitness,
"min_fitness": self.bb_optimizer.min_fitness,
"max_fitness": self.bb_optimizer.max_fitness,
"std_fitness": self.bb_optimizer.measured_noise,
"resampled_nbr": self.bb_optimizer.resampled_nbr,
"improvement_default": self.improvement_default,
"elapsed_time": self.bb_optimizer.elapsed_time,
"default_run": {
"fitness": self.bb_wrapper.default_target_value,
"job_id": self.bb_wrapper.default_jobid,
"parameters": self.bb_wrapper.default_parameters,
},
"average_noise": self.average_noise,
"explored_space":
float(self.bb_optimizer.size_explored_space[0]),
"best_fitness": best_fitness,
"best_parameters": self.build_parameter_dict(
self.configuration.component_parameter_names,
best_parameters.tolist(),
),
}
).dict()
def end(self):
"""End the experiment once it is over."""
request = self.api_client.put(
f"experiments/{self.experiment_id}/finish", json=self.end_dict
)
if not 200 <= request.status_code < 400:
self.fail()
raise Exception(
"Could not finish experiment with status code"
f"{request.status_code}"
)
def fail(self):
"""Fail the experiment.
If there is an experiment id, sends a request at the endpoint
/fail. Else, does not do anything (could be a good idea to
perform some logging).
"""
if self.experiment_id:
request = self.api_client.put(
f"experiments/{self.experiment_id}/fail")
if not 200 <= request.status_code < 400:
raise Exception(
"Could not fail experiment with status code"
f"{request.status_code}"
)
def stop(self):
"""Stop the experiment."""
request = self.api_client.put(f"experiments/{self.experiment_id}/stop")
if not 200 <= request.status_code < 400:
raise Exception(
"Could not fail experiment with status code"
f"{request.status_code}"
)
def summarize(self):
"""Summarize the experiment by printing out the best parametrization,
the associated fitness, and the BBO summary."""
# Compute the optimal parametrization from the bb optimizer object
optimal_param = self.build_parameter_dict(
self.configuration.component_parameter_names,
[self.bb_optimizer.best_parameters_in_grid],
)
print(f"Optimal time: {self.bb_optimizer.best_fitness}")
print(
f"Improvement compared to default:" f"{self.improvement_default}%")
print(f"Optimal parametrization: {optimal_param}")
print(
"Number of early stops:"
f"{sum(self.bb_optimizer.history['truncated'])}")
print(
"Average noise within each parametrization:"
f"{self.bb_optimizer.measured_noise}"
)
self.bb_optimizer.summarize()
@staticmethod
def build_parameter_dict(
parameter_names: List[str], parameter_values: List[List[float]]
) -> List[Dict]:
"""Static method to build the dictionary associated with the name of
the parameters and their associated values.
Args:
parameter_names (list of str): The name of the parameters to use.
parameter_values (list of lists of float):
The values of the parameters
Returns:
list of dict: the dicts of the parameters with
corresponding values.
"""
# The list that will contain the dictionaries
list_dict = []
# Check if there is at least two nested elements in the dictionary
if not isinstance(parameter_values[0], list):
parameter_values = [parameter_values]
for value in parameter_values:
list_dict.append(dict(zip(parameter_names, value)))
return list_dict
class HTTPClient:
def __init__(
self,
base_url: str,
default_headers: Optional[dict] = None,
default_params: Optional[dict] = None,
history_len: int = 30,
):
self.base_url = base_url
self.default_headers = default_headers or {}
self.default_params = default_params or {}
self.history: Deque[Response] = deque(maxlen=history_len)
self.http_client = Client(
base_url=self.base_url, headers=default_headers, params=self.default_params
)
def get(self, url: str, params: dict, headers: Optional[dict] = None):
custom_headers = headers or {}
all_params = {**self._generate_params(), **params}
logger.debug(f"Sending request to {url}", params=all_params, custom_headers=custom_headers)
response = self.http_client.get(url=url, params=all_params, headers=custom_headers)
self.history.append(response)
body = response.text or "is empty!"
logger.debug(f"Response return status_code: {response.status_code}, body: {body}")
for cookie_name, cookie_data in response.cookies.items():
self.http_client.cookies.set(cookie_name, cookie_data)
logger.debug(f"New cookies: {dict(response.cookies)}")
return response
def post(
self, url: str, data: dict, headers: Optional[dict] = None, params: Optional[dict] = None
):
custom_headers = headers or {}
custom_params = params or {}
# merge parameters
all_params = {**self._generate_params(), **custom_params}
logger.debug(
f"Sending request to {url}", params=all_params, custom_headers=custom_headers, data=data
)
response = self.http_client.post(
url=url, params=all_params, data=data, headers=custom_headers,
)
self.history.append(response)
body = response.text or "is empty!"
logger.debug(f"Response return status_code: {response.status_code}, body: {body}")
for cookie_name, cookie_data in response.cookies.items():
self.http_client.cookies.set(cookie_name, cookie_data)
logger.debug(f"New cookies: {dict(response.cookies)}")
return response
def _generate_params(self):
now = str(int(round(time() * 1000)))
params = {
"_rticket": now,
"ts": now,
"mas": generate_mas(now),
"as": generate_as(now),
"cp": generate_cp(now),
"idfa": str(uuid4()).upper(),
}
return params