async def generate_config(server_url: URL, ports, cleandir): return Config.withOverrides({ Setting.DRIVE_URL: str(server_url), Setting.SUPERVISOR_URL: str(server_url) + "/", Setting.AUTHORIZATION_HOST: str(server_url), Setting.TOKEN_SERVER_HOSTS: str(server_url), Setting.DRIVE_REFRESH_URL: str(server_url.with_path("/oauth2/v4/token")), Setting.DRIVE_AUTHORIZE_URL: str(server_url.with_path("/o/oauth2/v2/auth")), Setting.DRIVE_TOKEN_URL: str(server_url.with_path("/token")), Setting.SUPERVISOR_TOKEN: "test_header", Setting.SECRETS_FILE_PATH: "secrets.yaml", Setting.CREDENTIALS_FILE_PATH: "credentials.dat", Setting.FOLDER_FILE_PATH: "folder.dat", Setting.RETAINED_FILE_PATH: "retained.json", Setting.ID_FILE_PATH: "id.json", Setting.DATA_CACHE_FILE_PATH: "data_cache.json", Setting.STOP_ADDON_STATE_PATH: "stop_addon.json", Setting.INGRESS_TOKEN_FILE_PATH: "ingress.dat", Setting.DEFAULT_DRIVE_CLIENT_ID: "test_client_id", Setting.DEFAULT_DRIVE_CLIENT_SECRET: "test_client_secret", Setting.BACKUP_DIRECTORY_PATH: cleandir, Setting.PORT: ports.ui, Setting.INGRESS_PORT: ports.ingress, Setting.BACKUP_STARTUP_DELAY_MINUTES: 0, })
async def main(): port = 56153 base = URL("http://localhost").with_port(port) config = Config.withOverrides({ Setting.DRIVE_AUTHORIZE_URL: str(base.with_path("o/oauth2/v2/auth")), Setting.AUTHENTICATE_URL: str(base.with_path("drive/authorize")), Setting.DRIVE_TOKEN_URL: str(base.with_path("token")), Setting.DRIVE_REFRESH_URL: str(base.with_path("oauth2/v4/token")), Setting.INGRESS_PORT: 56152 }) injector = Injector(SimServerModule(config)) server = injector.get(SimulationServer) # start the server runner = aiohttp.web.AppRunner(server.createApp()) await runner.setup() site = aiohttp.web.TCPSite(runner, "0.0.0.0", port=port) await site.start() print("Server started on port " + str(port)) print("Open a browser at http://localhost:" + str(port))
async def main(): port = 56153 base = URL("http://localhost").with_port(port) config = Config.withOverrides({ Setting.DRIVE_AUTHORIZE_URL: str(base.with_path("o/oauth2/v2/auth")), Setting.AUTHENTICATE_URL: str(base.with_path("drive/authorize")), Setting.DRIVE_TOKEN_URL: str(base.with_path("token")), Setting.DRIVE_REFRESH_URL: str(base.with_path("oauth2/v4/token")) }) injector = Injector(BaseModule(config, override_dns=False)) server = injector.get( ClassAssistedBuilder[SimulationServer]).build(port=port) await server.reset({ 'snapshot_min_size': 1024 * 1024 * 3, 'snapshot_max_size': 1024 * 1024 * 5, "drive_refresh_token": "test_refresh_token", "drive_upload_sleep": 0, "snapshot_wait_time": 0, "hassio_header": "test_header" }) # start the server runner = aiohttp.web.AppRunner(server.createApp()) await runner.setup() site = aiohttp.web.TCPSite(runner, "0.0.0.0", port=port) await site.start() print("Server started on port " + str(port))
class Browser: """ Communicate with Google Chrome through its DevTools protocol. Asynchronous context manager that creates a new Tab when entering. Destroyed upon exit. """ __slots__ = ('session', 'url', 'tab') def __init__(self, url): self.url = URL(url) self.session = None self.tab = None async def __aiter__(self): """ List all tabs """ async with aiohttp.ClientSession() as session: async with session.get(self.url.with_path('/json/list')) as r: resp = await r.json() for tab in resp: if tab['type'] == 'page': yield tab async def __aenter__(self): """ Create tab """ assert self.tab is None assert self.session is None self.session = aiohttp.ClientSession() async with self.session.get(self.url.with_path('/json/new')) as r: resp = await r.json() self.tab = await Tab.create(**resp) return self.tab async def __aexit__(self, excType, excValue, traceback): assert self.tab is not None assert self.session is not None await self.tab.close() try: async with self.session.get( self.url.with_path(f'/json/close/{self.tab.id}')) as r: resp = await r.text() assert resp == 'Target is closing' except aiohttp.client_exceptions.ClientConnectorError: # oh boy, the whole browser crashed instead if excType is Crashed: # exception is reraised by `return False` pass else: # this one is more important raise self.tab = None await self.session.close() self.session = None return False
def test_images_complete_lifecycle( helper: Helper, image: str, tag: str, loop: asyncio.AbstractEventLoop, docker: aiodocker.Docker, ) -> None: # Let`s push image captured = helper.run_cli(["image", "push", image]) # stderr has "Used image ..." lines # assert not captured.err image_full_str = f"image://{helper.cluster_name}/{helper.username}/{image}" assert captured.out.endswith(image_full_str) image_url = URL(image_full_str) # Check if image available on registry captured = helper.run_cli(["image", "ls", "--full-uri"]) image_urls = [URL(line) for line in captured.out.splitlines() if line] for url in image_urls: assert url.scheme == "image" image_url_without_tag = image_url.with_path( image_url.path.replace(f":{tag}", "")) assert image_url_without_tag in image_urls # delete local loop.run_until_complete(docker.images.delete(image, force=True)) docker_ls_output = loop.run_until_complete(docker.images.list()) local_images = parse_docker_ls_output(docker_ls_output) assert image not in local_images # Pull image as with another tag captured = helper.run_cli(["image", "pull", f"image:{image}"]) # stderr has "Used image ..." lines # assert not captured.err assert captured.out.endswith(image) # check pulled locally, delete for cleanup docker_ls_output = loop.run_until_complete(docker.images.list()) local_images = parse_docker_ls_output(docker_ls_output) assert image in local_images # Execute image and check result captured = helper.run_cli([ "-q", "submit", *JOB_TINY_CONTAINER_PARAMS, "--non-preemptible", "--no-wait-start", str(image_url), ]) assert not captured.err job_id = captured.out assert job_id.startswith("job-") helper.wait_job_change_state_to(job_id, JobStatus.SUCCEEDED, JobStatus.FAILED) helper.check_job_output(job_id, re.escape(tag))
async def test_old_auth_method(server: SimulationServer, session: ClientSession, server_url: URL): start_auth = server_url.with_path("drive/authorize").with_query( {"redirectbacktoken": "http://example.com"}) # Verify the redirect to Drive's oauthv2 endpoint async with session.get(start_auth, data={}, allow_redirects=False) as r: assert r.status == 303 redirect = URL(r.headers[hdrs.LOCATION]) assert redirect.path == "/o/oauth2/v2/auth" assert redirect.host == "localhost" # Verify the redirect back to the server's oauth page async with session.get(redirect, data={}, allow_redirects=False) as r: assert r.status == 303 redirect = URL(r.headers[hdrs.LOCATION]) assert redirect.path == "/drive/authorize" assert redirect.host == "localhost" # Verify we gte redirected back to the addon (example.com) with creds async with session.get(redirect, data={}, allow_redirects=False) as r: assert r.status == 303 redirect = URL(r.headers[hdrs.LOCATION]) assert redirect.query.get("creds") is not None assert redirect.host == "example.com"
def inject(self, url: yarl.URL, payload: Payload) -> Iterable[Tuple[yarl.URL, Payload]]: if payload: i: str for i in payload: # поддержки вложенных массивов и объектов для JSON пока нет if not isinstance(payload[i], (bool, float, int, str)): continue copy: Dict[str, Any] = payload.copy() if isinstance(copy[i], str): copy[i] += QUOTE_CHARS else: # None -> null copy[i] = json.dumps(copy[i]) + QUOTE_CHARS yield url, copy # проводить дальнейшие проверки нет смысла return # Проверяем параметры URL в обратном порядке # TypeError: 'multidict._multidict.MultiDictProxy' object is not reversible i: str for i in reversed(list(url.query)): # как правильно тип описывать? copy: multidict.MultiDict[str] = url.query.copy() copy[i] += QUOTE_CHARS yield url.with_query(copy), payload # В последнюю очередь проверяем сегменты пути parts: List[str] = url.path.split('/') # так же в обратном порядке проверяем i: int for i in range(len(parts) - 1, -1, -1): if not parts[i]: continue copy: List[str] = parts.copy() copy[i] += quote(QUOTE_CHARS) yield url.with_path('/'.join(copy)), payload
def extract_valid_links(url: yarl.URL, soup: bs4.BeautifulSoup): link_urls = set( url.with_path(link.attrs.get("href")) for link in soup.find_all("a") if link_is_valid(link) ) return link_urls
async def test_refresh_unknown_error(server: SimulationServer, session: ClientSession, config: Config, server_url: URL): async with session.post(server_url.with_path("drive/refresh"), data={}) as r: assert r.status == 500 assert len((await r.json())["error"]) > 0
async def wait_till_storage_responsive(storage_endpoint: URL): async with aiohttp.ClientSession() as session: async with session.get(storage_endpoint.with_path("/v0/")) as resp: assert resp.status == 200 data = await resp.json() assert "data" in data assert data["data"] is not None
def http_server(tmp_path_factory: TempPathFactory) -> Iterable[List[URL]]: faker = Faker() files = ["file_1", "file_2", "file_3"] directory_path = tmp_path_factory.mktemp("http_server") assert directory_path.exists() for fn in files: with (directory_path / fn).open("wt") as f: f.write(f"This file is named: {fn}\n") for s in faker.sentences(): f.write(f"{s}\n") cmd = [sys.executable, "-m", "http.server", "8999"] base_url = URL("http://localhost:8999") with subprocess.Popen(cmd, cwd=directory_path) as p: timeout = 10 while True: try: requests.get(f"{base_url}") break except requests.exceptions.ConnectionError as e: time.sleep(0.1) timeout -= 0.1 if timeout < 0: raise RuntimeError("Server did not appear") from e # the server must be up yield [base_url.with_path(f) for f in files] # cleanup now, sometimes it hangs p.kill()
async def wait_till_webserver_responsive(webserver_endpoint: URL): async with aiohttp.ClientSession() as session: async with session.get(webserver_endpoint.with_path("/v0/")) as resp: # NOTE: Health-check endpoint require only a # status code 200 (see e.g. services/web/server/docker/healthcheck.py) # regardless of the payload content assert resp.status == 200
async def post_new_stack( base_url: URL, app_session: ClientSession, bearer_code: str, swarm_id: str, endpoint_id: int, stack_name: str, stack_cfg: Dict, ): # pylint: disable=too-many-arguments log.debug("creating new stack %s", base_url) if endpoint_id < 0: endpoint_id = await get_first_endpoint_id(base_url, app_session, bearer_code) log.debug("Determined the following endpoint id: %i", endpoint_id) headers = {"Authorization": "Bearer {}".format(bearer_code)} body_data = { "Name": stack_name, "SwarmID": swarm_id, "StackFileContent": json.dumps(stack_cfg, indent=2), } url = base_url.with_path("api/stacks").with_query( {"type": 1, "method": "string", "endpointId": endpoint_id} ) log.debug("Assuming URL: %s", url) log.debug("Assuming headers:") log.debug(json.dumps(headers, indent=2)) log.debug("Assuming data:") log.debug(json.dumps(body_data, indent=2)) log.debug("Sending POST request....") data = await _portainer_request( url, app_session, "POST", headers=headers, json=body_data ) log.debug("created new stack: %s", data)
class Browser: """ Communicate with Google Chrome through its DevTools protocol. Asynchronous context manager that creates a new Tab when entering. Destroyed upon exit. """ __slots__ = ('session', 'url', 'tab', 'loop') def __init__(self, url, loop=None): self.url = URL(url) self.session = None self.tab = None self.loop = loop async def __aiter__(self): """ List all tabs """ async with aiohttp.ClientSession(loop=self.loop) as session: async with session.get(self.url.with_path('/json/list')) as r: resp = await r.json() for tab in resp: if tab['type'] == 'page': yield tab async def __aenter__(self): """ Create tab """ assert self.tab is None assert self.session is None self.session = aiohttp.ClientSession(loop=self.loop) async with self.session.get(self.url.with_path('/json/new')) as r: resp = await r.json() self.tab = await Tab.create(**resp) return self.tab async def __aexit__(self, *args): assert self.tab is not None assert self.session is not None await self.tab.close() async with self.session.get( self.url.with_path(f'/json/close/{self.tab.id}')) as r: resp = await r.text() assert resp == 'Target is closing' self.tab = None await self.session.close() self.session = None return False
def append_base_url_slash(value): value = URL(value) if not value.path.endswith("/"): warnings.warn( "StudIP API %s must end with a slash. Appending '/' to make path concatenation work correctly.", repr(value)) value = value.with_path(value.path + "/") return value
async def get_stacks_list(base_url: URL, app_session: ClientSession, bearer_code: str) -> List[Dict]: log.debug("getting stacks list %s", base_url) headers = {"Authorization": "Bearer {}".format(bearer_code)} url = base_url.with_path("api/stacks") data = await _portainer_request(url, app_session, "GET", headers=headers) log.debug("received list of stacks: %s", data) return data
async def wait_till_webserver_responsive(webserver_endpoint: URL): async with aiohttp.ClientSession() as session: async with session.get(webserver_endpoint.with_path("/v0/")) as resp: assert resp.status == 200 data = await resp.json() assert "data" in data assert "status" in data["data"] assert data["data"]["status"] == "SERVICE_RUNNING"
def tmp_database(db_url: URL, suffix: str = '', **kwargs): tmp_db_name = '.'.join([uuid.uuid4().hex, project_name, suffix]) tmp_db_url = str(db_url.with_path(tmp_db_name)) create_database(tmp_db_url, **kwargs) try: yield tmp_db_url finally: drop_database(tmp_db_url)
def to_backend_service(rel_url: URL, origin: URL, version_prefix: str) -> URL: """ Translates relative url to backend catalog service url E.g. https://osparc.io/v0/catalog/dags -> http://catalog:8080/v0/dags """ assert not rel_url.is_absolute() # nosec new_path = rel_url.path.replace(f"/{api_version_prefix}/catalog", f"/{version_prefix}") return origin.with_path(new_path).with_query(rel_url.query)
def rel_url(self): # special case for path started with `//` # if path starts with // it is valid host, but in case of web server # liklyhood of it beein malformed path is much higher url = URL(self._message.path) if self._message.path.startswith('//'): return url.with_path(self._message.path.split('?')[0]) return URL(self._message.path)
def parse_path(request: web.Request) -> URL: domain, *path = request.path.strip('/').split('/', 1) if not path: path = '/' else: path = path[0] url = URL(domain) if not url.scheme: url = URL(f'{request.scheme}://{domain}') return url.with_path(path)
def swap_url( ignore_query: bool = True, old_url: URL = URL(""), # noqa: B008 new_url: URL = URL(""), # noqa: B008 url: URL = URL(""), # noqa: B008 ) -> URL: """Swap any instances of the old url with the new url. Will not replace query info. Args: ignore_query (bool): Whether the url.query should be ignored. Defaults to True. old_url (URL): Old url to find and replace. If there is any additional path, it will be added to the new_url. new_url (URL): New url to replace. url (URL): url to modify """ if isinstance(old_url, str): old_url = URL(old_url) if isinstance(new_url, str): new_url = URL(new_url) if isinstance(url, str): url = URL(url) old_url_string: Text = str(old_url.with_query({})) new_url_string: Text = str(new_url.with_query({})) old_query: MultiDict = url.query url_string = str(url.with_query({})) # ensure both paths end with "/" if one of them does if (new_url_string and new_url_string[-1] == "/" and old_url_string and old_url_string[-1] != "/"): old_url_string += "/" elif (old_url_string and old_url_string[-1] == "/" and new_url_string and new_url_string[-1] != "/"): new_url_string += "/" if ignore_query: result = URL(url_string.replace(old_url_string, new_url_string)) # clean up any // in path return result.with_path(result.path.replace("//", "/")).with_query(old_query) new_query = {} for key, value in old_query.items(): if value: new_query[key] = value.replace(old_url_string, new_url_string) result = URL(url_string.replace(old_url_string, new_url_string)) return result.with_path(result.path.replace("//", "/")).update_query(new_query)
async def wait_till_traefik_responsive(api_endpoint: URL): async with aiohttp.ClientSession() as session: async with session.get( api_endpoint.with_path("/api/http/routers")) as resp: assert resp.status == 200 data = await resp.json() for proxied_service in data: assert "service" in proxied_service if "webserver" in proxied_service["service"]: assert proxied_service["status"] == "enabled"
async def test_refresh_known_error(server: SimulationServer, session: ClientSession, config: Config, server_url: URL): async with session.post(server_url.with_path("drive/refresh"), json={"blah": "blah"}) as r: assert r.status == 503 assert await r.json() == { 'error': "Required key 'refresh_token' was missing from the request payload" }
async def get_first_endpoint_id(base_url: URL, app_session: ClientSession, bearer_code: str) -> int: log.debug("getting first endpoint id %s", base_url) headers = {"Authorization": "Bearer {}".format(bearer_code)} url = base_url.with_path(f"api/endpoints") data = await _portainer_request(url, app_session, "GET", headers=headers) log.debug("received list of endpoints: %s", data) if not data: raise ConfigurationError("portainer does not provide any endpoint") return data[0]["Id"]
async def wait_till_service_responsive(endpoint: URL): async with aiohttp.ClientSession() as session: async with session.get(endpoint.with_path("/v0/")) as resp: assert resp.status == 200 data = await resp.json() # aiohttp based services are like this: assert "data" in data or ":-)" in data if "data" in data: assert "status" in data["data"] assert data["data"]["status"] == "SERVICE_RUNNING"
async def decrease_steps(self, xsrf): logging.info('start steps decreasing') await self.api.myaccount_page() await self._resources(self.api.MYACCOUNT_URL) html = await self.api.one_size_accessories_page() product_url = find_rand_product_url(html) product_url = ZalandoAPI.INDEX_URL.join(URL(product_url)) await self._resources(self.api.ACCESSORIES_URL) html = await self.api.product_page(product_url) product_id, silhouette, version, uid_hash = find_product_params(html) await self._resources(product_url) await self.api.api_sizereco(xsrf, product_url, product_id, silhouette, version, uid_hash) await sleep(Delay.API) await self.api.api_check_wishlist(xsrf, product_url, product_id) await sleep(Delay.API) await self.api.api_preference_brands(xsrf, product_url) await sleep(Delay.CART) await self.api.api_cart(xsrf, product_url, product_id) await sleep(Delay.API) await self.api.api_cart_count(xsrf, product_url) await sleep(Delay.API) await self.api.api_cart_details(xsrf, product_url) await sleep(Delay.API) html = await self.api.cart_page(product_url) cart_id, flow_id = find_redeem_params(html) await self._resources(self.api.CART_URL) # discount may be expired/invalid, or product already has it await self.api.api_redeem(xsrf, cart_id, flow_id) await sleep(Delay.API) html = await self.api.checkout_confirm_page() # 302 /checkout/address address_id = find_address_id(html) await self._resources(self.api.CHK_ADDRESS_URL) await self.api.api_checkout_address_def(xsrf, address_id) payment_session_url = URL(await self.api.api_next_step(xsrf)) await sleep(Delay.API) # 307,303, 302: payment.domain/selection, payment.domain/payment-complete, /checkout/confirm location = await self.api.payment_session(payment_session_url) payment_selection_url = payment_session_url.with_path(location) await sleep(Delay.PAYMENT) location = await self.api.payment_selection(payment_selection_url) await sleep(Delay.PAYMENT) await self.api.payment_complete(location) await self._resources(self.api.CHK_CONFIRM_URL) await self.api.api_remove_item(xsrf, product_id) await sleep(Delay.PAGE) await self.api.cart_page(self.api.CHK_CONFIRM_URL) await self._resources(self.api.CART_URL) logging.info('steps decreasing is finished')
class AidboxSession(ClientSession): def __init__(self, *args, base_url=None, **kwargs): self.base_url = URL(base_url or os.environ.get("AIDBOX_BASE_URL")) super().__init__(*args, **kwargs) def make_url(self, path): return self.base_url.with_path(path) async def _request(self, method, path, *args, **kwargs): url = self.make_url(path) return await super()._request(method, url, *args, **kwargs)
async def test_fail_401(time: Time, session: TracingSession, config: Config, server: SimulationServer, drive_requests: DriveRequests, interceptor: RequestInterceptor, server_url: URL): session.record = True interceptor.setError("^/drive/refresh$", 401) with pytest.raises(GoogleCredentialsExpired): await drive_requests.exchanger.refresh(drive_requests.creds) # Verify both hosts were checked session._records[0]['url'] == server_url.with_path("/drive/refresh")
async def get_swarm_id(base_url: URL, app_session: ClientSession, bearer_code: str, endpoint_id: int) -> str: log.debug("getting swarm id %s", base_url) headers = {"Authorization": "Bearer {}".format(bearer_code)} if endpoint_id < 0: endpoint_id = await get_first_endpoint_id(base_url, app_session, bearer_code) url = base_url.with_path(f"api/endpoints/{endpoint_id}/docker/swarm") data = await _portainer_request(url, app_session, "GET", headers=headers) log.debug("received swarm details: %s", data) swarm_id = data["ID"] return swarm_id
def test_with_path_encoded(): url = URL("http://example.com") assert str(url.with_path("/test", encoded=True)) == "http://example.com/test"
def test_with_path_leading_slash(): url = URL("http://example.com") assert url.with_path("test").path == "/test"
def test_with_path_empty(): url = URL("http://example.com/test") assert str(url.with_path("")) == "http://example.com"
def test_with_path_fragment(): url = URL("http://example.com#frag") assert str(url.with_path("/test")) == "http://example.com/test"
def test_with_path_query(): url = URL("http://example.com?a=b") assert str(url.with_path("/test")) == "http://example.com/test"
def test_with_path_relative(): url = URL("/path") assert str(url.with_path("/new")) == "/new"
def test_with_path_dots(): url = URL("http://example.com") assert str(url.with_path("/test/.")) == "http://example.com/test/"