async def generate_config(server_url: URL, ports, cleandir):
return Config.withOverrides({
Setting.DRIVE_URL: str(server_url),
Setting.SUPERVISOR_URL: str(server_url) + "/",
Setting.AUTHORIZATION_HOST: str(server_url),
Setting.TOKEN_SERVER_HOSTS: str(server_url),
Setting.DRIVE_REFRESH_URL: str(server_url.with_path("/oauth2/v4/token")),
Setting.DRIVE_AUTHORIZE_URL: str(server_url.with_path("/o/oauth2/v2/auth")),
Setting.DRIVE_TOKEN_URL: str(server_url.with_path("/token")),
Setting.SUPERVISOR_TOKEN: "test_header",
Setting.SECRETS_FILE_PATH: "secrets.yaml",
Setting.CREDENTIALS_FILE_PATH: "credentials.dat",
Setting.FOLDER_FILE_PATH: "folder.dat",
Setting.RETAINED_FILE_PATH: "retained.json",
Setting.ID_FILE_PATH: "id.json",
Setting.DATA_CACHE_FILE_PATH: "data_cache.json",
Setting.STOP_ADDON_STATE_PATH: "stop_addon.json",
Setting.INGRESS_TOKEN_FILE_PATH: "ingress.dat",
Setting.DEFAULT_DRIVE_CLIENT_ID: "test_client_id",
Setting.DEFAULT_DRIVE_CLIENT_SECRET: "test_client_secret",
Setting.BACKUP_DIRECTORY_PATH: cleandir,
Setting.PORT: ports.ui,
Setting.INGRESS_PORT: ports.ingress,
Setting.BACKUP_STARTUP_DELAY_MINUTES: 0,
})
async def main():
port = 56153
base = URL("http://localhost").with_port(port)
config = Config.withOverrides({
Setting.DRIVE_AUTHORIZE_URL:
str(base.with_path("o/oauth2/v2/auth")),
Setting.AUTHENTICATE_URL:
str(base.with_path("drive/authorize")),
Setting.DRIVE_TOKEN_URL:
str(base.with_path("token")),
Setting.DRIVE_REFRESH_URL:
str(base.with_path("oauth2/v4/token")),
Setting.INGRESS_PORT:
56152
})
injector = Injector(SimServerModule(config))
server = injector.get(SimulationServer)
# start the server
runner = aiohttp.web.AppRunner(server.createApp())
await runner.setup()
site = aiohttp.web.TCPSite(runner, "0.0.0.0", port=port)
await site.start()
print("Server started on port " + str(port))
print("Open a browser at http://localhost:" + str(port))
async def main():
port = 56153
base = URL("http://localhost").with_port(port)
config = Config.withOverrides({
Setting.DRIVE_AUTHORIZE_URL:
str(base.with_path("o/oauth2/v2/auth")),
Setting.AUTHENTICATE_URL:
str(base.with_path("drive/authorize")),
Setting.DRIVE_TOKEN_URL:
str(base.with_path("token")),
Setting.DRIVE_REFRESH_URL:
str(base.with_path("oauth2/v4/token"))
})
injector = Injector(BaseModule(config, override_dns=False))
server = injector.get(
ClassAssistedBuilder[SimulationServer]).build(port=port)
await server.reset({
'snapshot_min_size': 1024 * 1024 * 3,
'snapshot_max_size': 1024 * 1024 * 5,
"drive_refresh_token": "test_refresh_token",
"drive_upload_sleep": 0,
"snapshot_wait_time": 0,
"hassio_header": "test_header"
})
# start the server
runner = aiohttp.web.AppRunner(server.createApp())
await runner.setup()
site = aiohttp.web.TCPSite(runner, "0.0.0.0", port=port)
await site.start()
print("Server started on port " + str(port))
class Browser:
"""
Communicate with Google Chrome through its DevTools protocol.
Asynchronous context manager that creates a new Tab when entering.
Destroyed upon exit.
"""
__slots__ = ('session', 'url', 'tab')
def __init__(self, url):
self.url = URL(url)
self.session = None
self.tab = None
async def __aiter__(self):
""" List all tabs """
async with aiohttp.ClientSession() as session:
async with session.get(self.url.with_path('/json/list')) as r:
resp = await r.json()
for tab in resp:
if tab['type'] == 'page':
yield tab
async def __aenter__(self):
""" Create tab """
assert self.tab is None
assert self.session is None
self.session = aiohttp.ClientSession()
async with self.session.get(self.url.with_path('/json/new')) as r:
resp = await r.json()
self.tab = await Tab.create(**resp)
return self.tab
async def __aexit__(self, excType, excValue, traceback):
assert self.tab is not None
assert self.session is not None
await self.tab.close()
try:
async with self.session.get(
self.url.with_path(f'/json/close/{self.tab.id}')) as r:
resp = await r.text()
assert resp == 'Target is closing'
except aiohttp.client_exceptions.ClientConnectorError:
# oh boy, the whole browser crashed instead
if excType is Crashed:
# exception is reraised by `return False`
pass
else:
# this one is more important
raise
self.tab = None
await self.session.close()
self.session = None
return False
def test_images_complete_lifecycle(
helper: Helper,
image: str,
tag: str,
loop: asyncio.AbstractEventLoop,
docker: aiodocker.Docker,
) -> None:
# Let`s push image
captured = helper.run_cli(["image", "push", image])
# stderr has "Used image ..." lines
# assert not captured.err
image_full_str = f"image://{helper.cluster_name}/{helper.username}/{image}"
assert captured.out.endswith(image_full_str)
image_url = URL(image_full_str)
# Check if image available on registry
captured = helper.run_cli(["image", "ls", "--full-uri"])
image_urls = [URL(line) for line in captured.out.splitlines() if line]
for url in image_urls:
assert url.scheme == "image"
image_url_without_tag = image_url.with_path(
image_url.path.replace(f":{tag}", ""))
assert image_url_without_tag in image_urls
# delete local
loop.run_until_complete(docker.images.delete(image, force=True))
docker_ls_output = loop.run_until_complete(docker.images.list())
local_images = parse_docker_ls_output(docker_ls_output)
assert image not in local_images
# Pull image as with another tag
captured = helper.run_cli(["image", "pull", f"image:{image}"])
# stderr has "Used image ..." lines
# assert not captured.err
assert captured.out.endswith(image)
# check pulled locally, delete for cleanup
docker_ls_output = loop.run_until_complete(docker.images.list())
local_images = parse_docker_ls_output(docker_ls_output)
assert image in local_images
# Execute image and check result
captured = helper.run_cli([
"-q",
"submit",
*JOB_TINY_CONTAINER_PARAMS,
"--non-preemptible",
"--no-wait-start",
str(image_url),
])
assert not captured.err
job_id = captured.out
assert job_id.startswith("job-")
helper.wait_job_change_state_to(job_id, JobStatus.SUCCEEDED,
JobStatus.FAILED)
helper.check_job_output(job_id, re.escape(tag))
async def test_old_auth_method(server: SimulationServer,
session: ClientSession, server_url: URL):
start_auth = server_url.with_path("drive/authorize").with_query(
{"redirectbacktoken": "http://example.com"})
# Verify the redirect to Drive's oauthv2 endpoint
async with session.get(start_auth, data={}, allow_redirects=False) as r:
assert r.status == 303
redirect = URL(r.headers[hdrs.LOCATION])
assert redirect.path == "/o/oauth2/v2/auth"
assert redirect.host == "localhost"
# Verify the redirect back to the server's oauth page
async with session.get(redirect, data={}, allow_redirects=False) as r:
assert r.status == 303
redirect = URL(r.headers[hdrs.LOCATION])
assert redirect.path == "/drive/authorize"
assert redirect.host == "localhost"
# Verify we gte redirected back to the addon (example.com) with creds
async with session.get(redirect, data={}, allow_redirects=False) as r:
assert r.status == 303
redirect = URL(r.headers[hdrs.LOCATION])
assert redirect.query.get("creds") is not None
assert redirect.host == "example.com"
def inject(self, url: yarl.URL,
payload: Payload) -> Iterable[Tuple[yarl.URL, Payload]]:
if payload:
i: str
for i in payload:
# поддержки вложенных массивов и объектов для JSON пока нет
if not isinstance(payload[i], (bool, float, int, str)):
continue
copy: Dict[str, Any] = payload.copy()
if isinstance(copy[i], str):
copy[i] += QUOTE_CHARS
else:
# None -> null
copy[i] = json.dumps(copy[i]) + QUOTE_CHARS
yield url, copy
# проводить дальнейшие проверки нет смысла
return
# Проверяем параметры URL в обратном порядке
# TypeError: 'multidict._multidict.MultiDictProxy' object is not reversible
i: str
for i in reversed(list(url.query)):
# как правильно тип описывать?
copy: multidict.MultiDict[str] = url.query.copy()
copy[i] += QUOTE_CHARS
yield url.with_query(copy), payload
# В последнюю очередь проверяем сегменты пути
parts: List[str] = url.path.split('/')
# так же в обратном порядке проверяем
i: int
for i in range(len(parts) - 1, -1, -1):
if not parts[i]:
continue
copy: List[str] = parts.copy()
copy[i] += quote(QUOTE_CHARS)
yield url.with_path('/'.join(copy)), payload
def extract_valid_links(url: yarl.URL, soup: bs4.BeautifulSoup):
link_urls = set(
url.with_path(link.attrs.get("href"))
for link in soup.find_all("a")
if link_is_valid(link)
)
return link_urls
async def test_refresh_unknown_error(server: SimulationServer,
session: ClientSession, config: Config,
server_url: URL):
async with session.post(server_url.with_path("drive/refresh"),
data={}) as r:
assert r.status == 500
assert len((await r.json())["error"]) > 0
async def wait_till_storage_responsive(storage_endpoint: URL):
async with aiohttp.ClientSession() as session:
async with session.get(storage_endpoint.with_path("/v0/")) as resp:
assert resp.status == 200
data = await resp.json()
assert "data" in data
assert data["data"] is not None
def http_server(tmp_path_factory: TempPathFactory) -> Iterable[List[URL]]:
faker = Faker()
files = ["file_1", "file_2", "file_3"]
directory_path = tmp_path_factory.mktemp("http_server")
assert directory_path.exists()
for fn in files:
with (directory_path / fn).open("wt") as f:
f.write(f"This file is named: {fn}\n")
for s in faker.sentences():
f.write(f"{s}\n")
cmd = [sys.executable, "-m", "http.server", "8999"]
base_url = URL("http://localhost:8999")
with subprocess.Popen(cmd, cwd=directory_path) as p:
timeout = 10
while True:
try:
requests.get(f"{base_url}")
break
except requests.exceptions.ConnectionError as e:
time.sleep(0.1)
timeout -= 0.1
if timeout < 0:
raise RuntimeError("Server did not appear") from e
# the server must be up
yield [base_url.with_path(f) for f in files]
# cleanup now, sometimes it hangs
p.kill()
async def wait_till_webserver_responsive(webserver_endpoint: URL):
async with aiohttp.ClientSession() as session:
async with session.get(webserver_endpoint.with_path("/v0/")) as resp:
# NOTE: Health-check endpoint require only a
# status code 200 (see e.g. services/web/server/docker/healthcheck.py)
# regardless of the payload content
assert resp.status == 200
async def post_new_stack(
base_url: URL,
app_session: ClientSession,
bearer_code: str,
swarm_id: str,
endpoint_id: int,
stack_name: str,
stack_cfg: Dict,
): # pylint: disable=too-many-arguments
log.debug("creating new stack %s", base_url)
if endpoint_id < 0:
endpoint_id = await get_first_endpoint_id(base_url, app_session, bearer_code)
log.debug("Determined the following endpoint id: %i", endpoint_id)
headers = {"Authorization": "Bearer {}".format(bearer_code)}
body_data = {
"Name": stack_name,
"SwarmID": swarm_id,
"StackFileContent": json.dumps(stack_cfg, indent=2),
}
url = base_url.with_path("api/stacks").with_query(
{"type": 1, "method": "string", "endpointId": endpoint_id}
)
log.debug("Assuming URL: %s", url)
log.debug("Assuming headers:")
log.debug(json.dumps(headers, indent=2))
log.debug("Assuming data:")
log.debug(json.dumps(body_data, indent=2))
log.debug("Sending POST request....")
data = await _portainer_request(
url, app_session, "POST", headers=headers, json=body_data
)
log.debug("created new stack: %s", data)
class Browser:
"""
Communicate with Google Chrome through its DevTools protocol.
Asynchronous context manager that creates a new Tab when entering.
Destroyed upon exit.
"""
__slots__ = ('session', 'url', 'tab', 'loop')
def __init__(self, url, loop=None):
self.url = URL(url)
self.session = None
self.tab = None
self.loop = loop
async def __aiter__(self):
""" List all tabs """
async with aiohttp.ClientSession(loop=self.loop) as session:
async with session.get(self.url.with_path('/json/list')) as r:
resp = await r.json()
for tab in resp:
if tab['type'] == 'page':
yield tab
async def __aenter__(self):
""" Create tab """
assert self.tab is None
assert self.session is None
self.session = aiohttp.ClientSession(loop=self.loop)
async with self.session.get(self.url.with_path('/json/new')) as r:
resp = await r.json()
self.tab = await Tab.create(**resp)
return self.tab
async def __aexit__(self, *args):
assert self.tab is not None
assert self.session is not None
await self.tab.close()
async with self.session.get(
self.url.with_path(f'/json/close/{self.tab.id}')) as r:
resp = await r.text()
assert resp == 'Target is closing'
self.tab = None
await self.session.close()
self.session = None
return False
def append_base_url_slash(value):
value = URL(value)
if not value.path.endswith("/"):
warnings.warn(
"StudIP API %s must end with a slash. Appending '/' to make path concatenation work correctly.",
repr(value))
value = value.with_path(value.path + "/")
return value
async def get_stacks_list(base_url: URL, app_session: ClientSession,
bearer_code: str) -> List[Dict]:
log.debug("getting stacks list %s", base_url)
headers = {"Authorization": "Bearer {}".format(bearer_code)}
url = base_url.with_path("api/stacks")
data = await _portainer_request(url, app_session, "GET", headers=headers)
log.debug("received list of stacks: %s", data)
return data
async def wait_till_webserver_responsive(webserver_endpoint: URL):
async with aiohttp.ClientSession() as session:
async with session.get(webserver_endpoint.with_path("/v0/")) as resp:
assert resp.status == 200
data = await resp.json()
assert "data" in data
assert "status" in data["data"]
assert data["data"]["status"] == "SERVICE_RUNNING"
def tmp_database(db_url: URL, suffix: str = '', **kwargs):
tmp_db_name = '.'.join([uuid.uuid4().hex, project_name, suffix])
tmp_db_url = str(db_url.with_path(tmp_db_name))
create_database(tmp_db_url, **kwargs)
try:
yield tmp_db_url
finally:
drop_database(tmp_db_url)
def to_backend_service(rel_url: URL, origin: URL, version_prefix: str) -> URL:
""" Translates relative url to backend catalog service url
E.g. https://osparc.io/v0/catalog/dags -> http://catalog:8080/v0/dags
"""
assert not rel_url.is_absolute() # nosec
new_path = rel_url.path.replace(f"/{api_version_prefix}/catalog",
f"/{version_prefix}")
return origin.with_path(new_path).with_query(rel_url.query)
def rel_url(self):
# special case for path started with `//`
# if path starts with // it is valid host, but in case of web server
# liklyhood of it beein malformed path is much higher
url = URL(self._message.path)
if self._message.path.startswith('//'):
return url.with_path(self._message.path.split('?')[0])
return URL(self._message.path)
def parse_path(request: web.Request) -> URL:
domain, *path = request.path.strip('/').split('/', 1)
if not path:
path = '/'
else:
path = path[0]
url = URL(domain)
if not url.scheme:
url = URL(f'{request.scheme}://{domain}')
return url.with_path(path)
def swap_url(
ignore_query: bool = True,
old_url: URL = URL(""), # noqa: B008
new_url: URL = URL(""), # noqa: B008
url: URL = URL(""), # noqa: B008
) -> URL:
"""Swap any instances of the old url with the new url. Will not replace query info.
Args:
ignore_query (bool): Whether the url.query should be ignored. Defaults to True.
old_url (URL): Old url to find and replace. If there is any additional path, it will be added to the new_url.
new_url (URL): New url to replace.
url (URL): url to modify
"""
if isinstance(old_url, str):
old_url = URL(old_url)
if isinstance(new_url, str):
new_url = URL(new_url)
if isinstance(url, str):
url = URL(url)
old_url_string: Text = str(old_url.with_query({}))
new_url_string: Text = str(new_url.with_query({}))
old_query: MultiDict = url.query
url_string = str(url.with_query({}))
# ensure both paths end with "/" if one of them does
if (new_url_string and new_url_string[-1] == "/" and old_url_string
and old_url_string[-1] != "/"):
old_url_string += "/"
elif (old_url_string and old_url_string[-1] == "/" and new_url_string
and new_url_string[-1] != "/"):
new_url_string += "/"
if ignore_query:
result = URL(url_string.replace(old_url_string, new_url_string))
# clean up any // in path
return result.with_path(result.path.replace("//",
"/")).with_query(old_query)
new_query = {}
for key, value in old_query.items():
if value:
new_query[key] = value.replace(old_url_string, new_url_string)
result = URL(url_string.replace(old_url_string, new_url_string))
return result.with_path(result.path.replace("//",
"/")).update_query(new_query)
async def wait_till_traefik_responsive(api_endpoint: URL):
async with aiohttp.ClientSession() as session:
async with session.get(
api_endpoint.with_path("/api/http/routers")) as resp:
assert resp.status == 200
data = await resp.json()
for proxied_service in data:
assert "service" in proxied_service
if "webserver" in proxied_service["service"]:
assert proxied_service["status"] == "enabled"
async def test_refresh_known_error(server: SimulationServer,
session: ClientSession, config: Config,
server_url: URL):
async with session.post(server_url.with_path("drive/refresh"),
json={"blah": "blah"}) as r:
assert r.status == 503
assert await r.json() == {
'error':
"Required key 'refresh_token' was missing from the request payload"
}
async def get_first_endpoint_id(base_url: URL, app_session: ClientSession,
bearer_code: str) -> int:
log.debug("getting first endpoint id %s", base_url)
headers = {"Authorization": "Bearer {}".format(bearer_code)}
url = base_url.with_path(f"api/endpoints")
data = await _portainer_request(url, app_session, "GET", headers=headers)
log.debug("received list of endpoints: %s", data)
if not data:
raise ConfigurationError("portainer does not provide any endpoint")
return data[0]["Id"]
async def wait_till_service_responsive(endpoint: URL):
async with aiohttp.ClientSession() as session:
async with session.get(endpoint.with_path("/v0/")) as resp:
assert resp.status == 200
data = await resp.json()
# aiohttp based services are like this:
assert "data" in data or ":-)" in data
if "data" in data:
assert "status" in data["data"]
assert data["data"]["status"] == "SERVICE_RUNNING"
async def decrease_steps(self, xsrf):
logging.info('start steps decreasing')
await self.api.myaccount_page()
await self._resources(self.api.MYACCOUNT_URL)
html = await self.api.one_size_accessories_page()
product_url = find_rand_product_url(html)
product_url = ZalandoAPI.INDEX_URL.join(URL(product_url))
await self._resources(self.api.ACCESSORIES_URL)
html = await self.api.product_page(product_url)
product_id, silhouette, version, uid_hash = find_product_params(html)
await self._resources(product_url)
await self.api.api_sizereco(xsrf, product_url, product_id, silhouette,
version, uid_hash)
await sleep(Delay.API)
await self.api.api_check_wishlist(xsrf, product_url, product_id)
await sleep(Delay.API)
await self.api.api_preference_brands(xsrf, product_url)
await sleep(Delay.CART)
await self.api.api_cart(xsrf, product_url, product_id)
await sleep(Delay.API)
await self.api.api_cart_count(xsrf, product_url)
await sleep(Delay.API)
await self.api.api_cart_details(xsrf, product_url)
await sleep(Delay.API)
html = await self.api.cart_page(product_url)
cart_id, flow_id = find_redeem_params(html)
await self._resources(self.api.CART_URL)
# discount may be expired/invalid, or product already has it
await self.api.api_redeem(xsrf, cart_id, flow_id)
await sleep(Delay.API)
html = await self.api.checkout_confirm_page() # 302 /checkout/address
address_id = find_address_id(html)
await self._resources(self.api.CHK_ADDRESS_URL)
await self.api.api_checkout_address_def(xsrf, address_id)
payment_session_url = URL(await self.api.api_next_step(xsrf))
await sleep(Delay.API)
# 307,303, 302: payment.domain/selection, payment.domain/payment-complete, /checkout/confirm
location = await self.api.payment_session(payment_session_url)
payment_selection_url = payment_session_url.with_path(location)
await sleep(Delay.PAYMENT)
location = await self.api.payment_selection(payment_selection_url)
await sleep(Delay.PAYMENT)
await self.api.payment_complete(location)
await self._resources(self.api.CHK_CONFIRM_URL)
await self.api.api_remove_item(xsrf, product_id)
await sleep(Delay.PAGE)
await self.api.cart_page(self.api.CHK_CONFIRM_URL)
await self._resources(self.api.CART_URL)
logging.info('steps decreasing is finished')
class AidboxSession(ClientSession):
def __init__(self, *args, base_url=None, **kwargs):
self.base_url = URL(base_url or os.environ.get("AIDBOX_BASE_URL"))
super().__init__(*args, **kwargs)
def make_url(self, path):
return self.base_url.with_path(path)
async def _request(self, method, path, *args, **kwargs):
url = self.make_url(path)
return await super()._request(method, url, *args, **kwargs)
async def test_fail_401(time: Time, session: TracingSession, config: Config,
server: SimulationServer,
drive_requests: DriveRequests,
interceptor: RequestInterceptor, server_url: URL):
session.record = True
interceptor.setError("^/drive/refresh$", 401)
with pytest.raises(GoogleCredentialsExpired):
await drive_requests.exchanger.refresh(drive_requests.creds)
# Verify both hosts were checked
session._records[0]['url'] == server_url.with_path("/drive/refresh")
async def get_swarm_id(base_url: URL, app_session: ClientSession,
bearer_code: str, endpoint_id: int) -> str:
log.debug("getting swarm id %s", base_url)
headers = {"Authorization": "Bearer {}".format(bearer_code)}
if endpoint_id < 0:
endpoint_id = await get_first_endpoint_id(base_url, app_session,
bearer_code)
url = base_url.with_path(f"api/endpoints/{endpoint_id}/docker/swarm")
data = await _portainer_request(url, app_session, "GET", headers=headers)
log.debug("received swarm details: %s", data)
swarm_id = data["ID"]
return swarm_id
def test_with_path_encoded():
url = URL("http://example.com")
assert str(url.with_path("/test", encoded=True)) == "http://example.com/test"
def test_with_path_leading_slash():
url = URL("http://example.com")
assert url.with_path("test").path == "/test"
def test_with_path_empty():
url = URL("http://example.com/test")
assert str(url.with_path("")) == "http://example.com"
def test_with_path_fragment():
url = URL("http://example.com#frag")
assert str(url.with_path("/test")) == "http://example.com/test"
def test_with_path_query():
url = URL("http://example.com?a=b")
assert str(url.with_path("/test")) == "http://example.com/test"
def test_with_path_relative():
url = URL("/path")
assert str(url.with_path("/new")) == "/new"
def test_with_path_dots():
url = URL("http://example.com")
assert str(url.with_path("/test/.")) == "http://example.com/test/"