def run_in_loop(s_args, s_kwargs, c_args, c_kwargs): loop = asyncio.new_event_loop() asyncio.set_event_loop(None) server = aioftp.Server(*s_args, loop=loop, **s_kwargs) client = aioftp.Client(*c_args, loop=loop, **c_kwargs) coro = asyncio.coroutine(f) try: loop.run_until_complete(coro(loop, client, server)) finally: if hasattr(server, "server"): server.close() loop.run_until_complete(server.wait_closed()) if hasattr(client, "writer"): client.close() loop.close()
async def test_server_side_throttle(pair_factory, skip_sleep, times, users, throttle_direction, data_direction, throttle_level): async with pair_factory() as pair: names = [] for i in range(users): name = f"foo{i}" names.append(name) await pair.make_server_files(name, size=SIZE) throttle = reduce(getattr, [throttle_level, throttle_direction], pair.server) throttle.limit = SIZE / times clients = [] for name in names: c = aioftp.Client(path_io_factory=aioftp.MemoryPathIO) async with c.path_io.open(Path(name), "wb") as f: await f.write(b"-" * SIZE) await c.connect(pair.server.server_host, pair.server.server_port) await c.login() clients.append(c) coros = [getattr(c, data_direction)(n) for c, n in zip(clients, names)] await asyncio.gather(*coros) await asyncio.gather(*[c.quit() for c in clients]) throttled = {("read", "upload"), ("write", "download")} if (throttle_direction, data_direction) not in throttled: assert skip_sleep.is_close(0) else: t = times if throttle_level == "throttle": # global t *= users assert skip_sleep.is_close(t)
def run_in_loop(s_args, s_kwargs, c_args, c_kwargs): logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(name)s] %(message)s", datefmt="[%H:%M:%S]:", ) loop = asyncio.new_event_loop() asyncio.set_event_loop(None) server = aioftp.Server(*s_args, loop=loop, **s_kwargs) client = aioftp.Client(*c_args, loop=loop, **c_kwargs) try: loop.run_until_complete(f(loop, client, server)) finally: if hasattr(server, "server"): server.close() loop.run_until_complete(server.wait_closed()) if hasattr(client, "writer"): client.close() loop.close()
def test_user_manager_timeout(): loop = asyncio.new_event_loop() asyncio.set_event_loop(None) server = aioftp.Server( SlowUserManager(None, timeout=1, loop=loop), loop=loop) client = aioftp.Client(loop=loop) @asyncio.coroutine def coro(): try: yield from server.start(None, 8888) yield from client.connect("127.0.0.1", 8888) yield from client.login() finally: server.close() yield from server.wait_closed() loop.run_until_complete(coro())
def __init__(self, github_auth, log_channel, testing=False, debug=False, *args, **kwargs): self.github_auth = aiohttp.BasicAuth(github_auth[0], github_auth[1]) self.testing = testing self.launch_time = datetime.utcnow() self.ftp_client = aioftp.Client() super().__init__(*args, **kwargs) self.logs = self.get_channel(log_channel) self.database = cc.Database(self) extensions = sorted( [f"cogs.{ext[:-3]}" for ext in listdir("cogs") if ".py" in ext]) try: self.loop.add_signal_handler( SIGTERM, lambda: asyncio.ensure_future(self.close("SIGTERM Shutdown"))) except NotImplementedError: pass self.loop.set_debug(debug) self.load_extensions(extensions)
async def worker(fname): client = aioftp.Client(loop=loop) await client.connect("127.0.0.1", PORT) await client.login() await client.upload("tests/foo/foo.txt", str.format("tests/foo/{}", fname), write_into=True) await client.quit()
async def test_multiply_connections_server_limit_error(loop, client, server): clients = [aioftp.Client(loop=loop) for _ in range(5)] for client in clients: await client.connect("127.0.0.1", PORT) # await client.login("foo") for client in clients: await client.quit()
async def test_multiply_connections_limited_error(loop, client, server): clients = [aioftp.Client(loop=loop, ssl=client.ssl) for _ in range(5)] for client in clients: await client.connect("127.0.0.1", PORT) await client.login() for client in clients: await client.quit()
async def test_multiply_connections_no_limits(loop, client, server): clients = [aioftp.Client(loop=loop) for _ in range(4)] for client in clients: await client.connect("127.0.0.1", PORT) await client.login() for client in clients: await client.quit()
async def _update_database(self, server_id: str): info = self.settings[server_id] ftp = aioftp.Client() await ftp.connect(info["ftp_server"]) await ftp.login(info["ftp_username"], info["ftp_password"]) await ftp.download(info["ftp_dbpath"], "data/kz/{}/kztimer-sqlite.sq3".format(server_id), write_into=True) await ftp.quit()
async def test_multiply_connections_server_relogin_balanced(loop, client, server): clients = [aioftp.Client(loop=loop) for _ in range(5)] for client in clients[:-1]: await client.connect("127.0.0.1", PORT) await clients[0].quit() await clients[-1].connect("127.0.0.1", PORT) for client in clients[1:]: await client.quit()
def test_client_socket_timeout(): loop = asyncio.new_event_loop() asyncio.set_event_loop(None) server = SlowServer(loop=loop) client = aioftp.Client(loop=loop, socket_timeout=1) async def coro(): try: await server.start(None, 8888) await client.connect("127.0.0.1", 8888) await asyncio.sleep(10, loop=loop) finally: await server.close() loop.run_until_complete(coro())
async def main(): import aioftp import aiohttp session = aiohttp.ClientSession() client = aioftp.Client() # await client.connect("ftp.bom.gov.au", 21) # await client.login() # await download_ftp(client, "/anon/gen/fwo/IDA00009.gif") # await get_forecast(client, "anon/gen/fwo/" + FORECAST_XML[0]) # await get_forecasts(client) # await get_trivia_categories(session)) # await get_demotivators(session)) await session.close() client.close()
def test_client_list_windows(): test_str = textwrap.dedent("""\ 11/4/2018 9:09 PM <DIR> . 8/10/2018 1:02 PM <DIR> .. 9/23/2018 2:16 PM <DIR> bin 10/16/2018 10:25 PM <DIR> Desktop 11/4/2018 3:31 PM <DIR> dow 10/16/2018 8:21 PM <DIR> Downloads 10/14/2018 5:34 PM <DIR> msc 9/9/2018 9:32 AM <DIR> opt 10/3/2018 2:58 PM 34,359,738,368 win10.img 6/30/2018 8:36 AM 3,939,237,888 win10.iso 7/26/2018 1:11 PM 189 win10.sh 10/29/2018 11:46 AM 34,359,738,368 win7.img 6/30/2018 8:35 AM 3,319,791,616 win7.iso 10/29/2018 10:55 AM 219 win7.sh 6 files 75,978,506,648 bytes 3 directories 22,198,362,112 bytes free """) test_str = test_str.strip().split("\n") entities = {} parse = aioftp.Client(encoding="utf-8").parse_list_line_windows for x in test_str: with contextlib.suppress(ValueError): path, stat = parse(x.encode("utf-8")) entities[path] = stat dirs = ["bin", "Desktop", "dow", "Downloads", "msc", "opt"] files = [ "win10.img", "win10.iso", "win10.sh", "win7.img", "win7.iso", "win7.sh" ] assert len(entities) == len(dirs + files) for d in dirs: p = pathlib.PurePosixPath(d) assert p in entities assert entities[p]["type"] == "dir" for f in files: p = pathlib.PurePosixPath(f) assert p in entities assert entities[p]["type"] == "file" with pytest.raises(ValueError): parse(b" 10/3/2018 2:58 PM 34,35xxx38,368 win10.img")
async def __aenter__(self): await self._lock.acquire() client = self._client try: if client is None: client = aioftp.Client() await client.connect(self._settings.host, self._settings.port) await client.login(self._settings.user, self._settings.password, aioftp.DEFAULT_ACCOUNT) self._client = client if self._expire is not None: self._expire.cancel() self._expire = self._loop.call_later(CACHE_TIME, self._loop.create_task, self.close()) return client except Exception: client.close() self._lock.release() raise
async def download_ftp(host: str, filename: Path, output_dir: Path, semaphore: asyncio.Semaphore): async with semaphore: try: async with timeout(1): client = aioftp.Client() await client.connect(host) session = await client.login() print( f"started download {filename} for {host} in folder {output_dir}" ) stream = await session.download_stream(filename) async with aiofiles.open(output_dir / filename.name, mode="wb") as f: async for chunk in stream.iter_by_block(): await f.write(chunk) except asyncio.TimeoutError as e: print(f"timeout error {filename} for {host}") finally: client.close() session.close()
def test_parse_list_line_unix(): lines = { "file": [ "-rw-rw-r-- 1 poh poh 6595 Feb 27 04:14 history.rst", "lrwxrwxrwx 1 poh poh 6 Mar 23 05:46 link-tmp.py -> tmp.py", ], "dir": [ "drw-rw-r-- 1 poh poh 6595 Feb 27 04:14 history.rst", ], "unknown": [ "Erw-rw-r-- 1 poh poh 6595 Feb 27 04:14 history.rst", ] } p = aioftp.Client(encoding="utf-8").parse_list_line_unix for t, stack in lines.items(): for line in stack: _, parsed = p(line.encode("utf-8")) assert parsed["type"] == t with pytest.raises(ValueError): p(b"-rw-rw-r-- 1 poh poh 6xx5 Feb 27 04:14 history.rst") with pytest.raises(ValueError): p(b"-rw-rw-r-- xx poh poh 6595 Feb 27 04:14 history.rst")
def ftp_login(self): self.ftp_object = aioftp.Client() self.ftp_object.connect(self.ftp_uri) self.ftp_object.login() print('logged into gutenberg ftp mirror')
def __init__(self, database_settings: DatabaseSettings, stripe_settings: StripeSettings, smtp_settings: SmtpSettings, friendly_url: str, frontend_url: str, root_steam_id: str, system_email: str, upload_settings: Tuple[B2UploadSettings, LocalUploadSettings] = None, map_images: Dict[str, str] = MAP_IMAGES, upload_delay: float = 0.00001, free_upload_size: float = 30.0, max_upload_size: float = 100.0, timestamp_format: str = "%m/%d/%Y-%H:%M:%S", community_types: List[str] = COMMUNITY_TYPES, webhook_settings: WebhookSettings = WebhookSettings(), match_max_length: timedelta = timedelta(hours=3), demo_expires: timedelta = timedelta(weeks=20), subscription_length: timedelta = timedelta(days=31), clear_cache: bool = True, **kwargs) -> None: """SQLMatches API. Parameters ---------- database_settings : DatabaseSettings stripe_settings : StripeSettings friendly_url : str frontend_url : str root_steam_id : str system_email : str upload_settings : [B2UploadSettings, LocalUploadSettings], optional by default None map_images : Dict[str, str], optional by default MAP_IMAGES upload_delay : float, optional by default 0.00001 free_upload_size : float, optional by default 50.0 max_upload_size : float, optional by default 100.0 timestamp_format : str, optional by default "%m/%d/%Y-%H:%M:%S" community_types : List[str], optional by default COMMUNITY_TYPES webhook_settings : WebhookSettings, optional by default WebhookSettings() match_max_length : timedelta, optional by default timedelta(hours=3) clear_cache : bool, optional by default True demo_expires : timedelta, optional by default timedelta(weeks=20) """ startup_tasks = [self._startup] shutdown_tasks = [self._shutdown] if "on_startup" in kwargs: startup_tasks = startup_tasks + kwargs["on_startup"] if "on_shutdown" in kwargs: shutdown_tasks = shutdown_tasks + kwargs["on_shutdown"] middlewares = [ Middleware(SessionMiddleware, secret_key=KeyLoader(name="session").load()), Middleware(AuthenticationMiddleware, backend=APIAuthentication(), on_error=auth_error), Middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["GET", "POST", "DELETE", "OPTIONS"]) ] if "middleware" in kwargs: middlewares = middlewares + kwargs["middleware"] if "routes" in kwargs: routes = kwargs["routes"] + ROUTES else: routes = ROUTES if "exception_handlers" in kwargs: exception_handlers = kwargs["exception_handlers"] + ERROR_HANDLERS else: exception_handlers = ERROR_HANDLERS if friendly_url[:1] != "/": friendly_url += "/" if frontend_url[:1] != "/": frontend_url += "/" Config.url = friendly_url Config.map_images = map_images Config.upload_delay = upload_delay Config.free_upload_size = free_upload_size Config.max_upload_size = max_upload_size Config.timestamp_format = timestamp_format Config.root_steam_id_hashed = bcrypt.hashpw(root_steam_id.encode(), bcrypt.gensalt()) Config.root_webhook_key_hashed = bcrypt.hashpw( (KeyLoader("webhook").load()).encode(), bcrypt.gensalt()) Config.webhook_timeout = webhook_settings.timeout Config.webhook_match_end = webhook_settings.match_end Config.webhook_match_start = webhook_settings.match_start Config.webhook_round_end = webhook_settings.round_end Config.webhook_key = webhook_settings.key Config.match_max_length = match_max_length Config.system_email = system_email Config.frontend_url = frontend_url Config.demo_expires = demo_expires Config.price_id = stripe_settings.price_id Config.subscription_length = subscription_length self.community_types = community_types self.clear_cache = clear_cache database_url = "://{}:{}@{}:{}/{}?charset=utf8mb4".format( database_settings.username, database_settings.password, database_settings.server, database_settings.port, database_settings.database) Sessions.database = Database(database_settings.engine + database_url) create_tables("{}+{}{}".format(database_settings.engine, database_settings.alchemy_engine, database_url)) Sessions.smtp = SMTP(hostname=smtp_settings.hostname, port=smtp_settings.port, use_tls=smtp_settings.use_tls, password=smtp_settings.password, username=smtp_settings.username) Sessions.stripe = Stripe(stripe_settings.api_key, stripe_settings.testing) Sessions.ftp = aioftp.Client() if upload_settings: Config.demo_pathway = upload_settings.pathway Config.demo_extension = upload_settings.extension if isinstance(upload_settings, B2UploadSettings): Config.cdn_url = upload_settings.cdn_url Config.upload_type = B2UploadSettings self.b2 = backblaze.Awaiting(upload_settings.key_id, upload_settings.application_key) Sessions.bucket = self.b2.bucket(upload_settings.bucket_id) elif isinstance(upload_settings, LocalUploadSettings): Config.cdn_url = None Config.upload_type = LocalUploadSettings # Dynamically adding mount if local storage. # Name attribute for Mount isn't working correctly here, # so please don't change '/demos/'. for mount in routes: if type(mount) == Mount and mount.name == "api": mount.app.routes.append( Mount("/demos/", StaticFiles(directory=Config.demo_pathway))) break logger.warning( "Using local storage for demos, use b2 for production.") else: Config.upload_type = None super().__init__(routes=routes, exception_handlers=exception_handlers, middleware=middlewares, on_startup=startup_tasks, on_shutdown=shutdown_tasks, **kwargs)
def test_parse_list_line_failed(): with pytest.raises(ValueError): aioftp.Client(encoding="utf-8").parse_list_line(b"what a hell?!")
def __init__(self, config: Config): self.client = aioftp.Client() self.files = config.get('files', []) self.config = config