def run_in_loop(s_args, s_kwargs, c_args, c_kwargs):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
server = aioftp.Server(*s_args, loop=loop, **s_kwargs)
client = aioftp.Client(*c_args, loop=loop, **c_kwargs)
coro = asyncio.coroutine(f)
try:
loop.run_until_complete(coro(loop, client, server))
finally:
if hasattr(server, "server"):
server.close()
loop.run_until_complete(server.wait_closed())
if hasattr(client, "writer"):
client.close()
loop.close()
async def test_server_side_throttle(pair_factory, skip_sleep, times, users,
throttle_direction, data_direction,
throttle_level):
async with pair_factory() as pair:
names = []
for i in range(users):
name = f"foo{i}"
names.append(name)
await pair.make_server_files(name, size=SIZE)
throttle = reduce(getattr, [throttle_level, throttle_direction],
pair.server)
throttle.limit = SIZE / times
clients = []
for name in names:
c = aioftp.Client(path_io_factory=aioftp.MemoryPathIO)
async with c.path_io.open(Path(name), "wb") as f:
await f.write(b"-" * SIZE)
await c.connect(pair.server.server_host, pair.server.server_port)
await c.login()
clients.append(c)
coros = [getattr(c, data_direction)(n) for c, n in zip(clients, names)]
await asyncio.gather(*coros)
await asyncio.gather(*[c.quit() for c in clients])
throttled = {("read", "upload"), ("write", "download")}
if (throttle_direction, data_direction) not in throttled:
assert skip_sleep.is_close(0)
else:
t = times
if throttle_level == "throttle": # global
t *= users
assert skip_sleep.is_close(t)
def run_in_loop(s_args, s_kwargs, c_args, c_kwargs):
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(name)s] %(message)s",
datefmt="[%H:%M:%S]:",
)
loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
server = aioftp.Server(*s_args, loop=loop, **s_kwargs)
client = aioftp.Client(*c_args, loop=loop, **c_kwargs)
try:
loop.run_until_complete(f(loop, client, server))
finally:
if hasattr(server, "server"):
server.close()
loop.run_until_complete(server.wait_closed())
if hasattr(client, "writer"):
client.close()
loop.close()
def test_user_manager_timeout():
loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
server = aioftp.Server(
SlowUserManager(None, timeout=1, loop=loop),
loop=loop)
client = aioftp.Client(loop=loop)
@asyncio.coroutine
def coro():
try:
yield from server.start(None, 8888)
yield from client.connect("127.0.0.1", 8888)
yield from client.login()
finally:
server.close()
yield from server.wait_closed()
loop.run_until_complete(coro())
def __init__(self,
github_auth,
log_channel,
testing=False,
debug=False,
*args,
**kwargs):
self.github_auth = aiohttp.BasicAuth(github_auth[0], github_auth[1])
self.testing = testing
self.launch_time = datetime.utcnow()
self.ftp_client = aioftp.Client()
super().__init__(*args, **kwargs)
self.logs = self.get_channel(log_channel)
self.database = cc.Database(self)
extensions = sorted(
[f"cogs.{ext[:-3]}" for ext in listdir("cogs") if ".py" in ext])
try:
self.loop.add_signal_handler(
SIGTERM,
lambda: asyncio.ensure_future(self.close("SIGTERM Shutdown")))
except NotImplementedError:
pass
self.loop.set_debug(debug)
self.load_extensions(extensions)
async def worker(fname):
client = aioftp.Client(loop=loop)
await client.connect("127.0.0.1", PORT)
await client.login()
await client.upload("tests/foo/foo.txt",
str.format("tests/foo/{}", fname),
write_into=True)
await client.quit()
async def test_multiply_connections_server_limit_error(loop, client, server):
clients = [aioftp.Client(loop=loop) for _ in range(5)]
for client in clients:
await client.connect("127.0.0.1", PORT)
# await client.login("foo")
for client in clients:
await client.quit()
async def test_multiply_connections_limited_error(loop, client, server):
clients = [aioftp.Client(loop=loop, ssl=client.ssl) for _ in range(5)]
for client in clients:
await client.connect("127.0.0.1", PORT)
await client.login()
for client in clients:
await client.quit()
async def test_multiply_connections_no_limits(loop, client, server):
clients = [aioftp.Client(loop=loop) for _ in range(4)]
for client in clients:
await client.connect("127.0.0.1", PORT)
await client.login()
for client in clients:
await client.quit()
async def _update_database(self, server_id: str):
info = self.settings[server_id]
ftp = aioftp.Client()
await ftp.connect(info["ftp_server"])
await ftp.login(info["ftp_username"], info["ftp_password"])
await ftp.download(info["ftp_dbpath"],
"data/kz/{}/kztimer-sqlite.sq3".format(server_id),
write_into=True)
await ftp.quit()
async def test_multiply_connections_server_relogin_balanced(loop, client,
server):
clients = [aioftp.Client(loop=loop) for _ in range(5)]
for client in clients[:-1]:
await client.connect("127.0.0.1", PORT)
await clients[0].quit()
await clients[-1].connect("127.0.0.1", PORT)
for client in clients[1:]:
await client.quit()
def test_client_socket_timeout():
loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
server = SlowServer(loop=loop)
client = aioftp.Client(loop=loop, socket_timeout=1)
async def coro():
try:
await server.start(None, 8888)
await client.connect("127.0.0.1", 8888)
await asyncio.sleep(10, loop=loop)
finally:
await server.close()
loop.run_until_complete(coro())
async def main():
import aioftp
import aiohttp
session = aiohttp.ClientSession()
client = aioftp.Client()
# await client.connect("ftp.bom.gov.au", 21)
# await client.login()
# await download_ftp(client, "/anon/gen/fwo/IDA00009.gif")
# await get_forecast(client, "anon/gen/fwo/" + FORECAST_XML[0])
# await get_forecasts(client)
# await get_trivia_categories(session))
# await get_demotivators(session))
await session.close()
client.close()
def test_client_list_windows():
test_str = textwrap.dedent("""\
11/4/2018 9:09 PM <DIR> .
8/10/2018 1:02 PM <DIR> ..
9/23/2018 2:16 PM <DIR> bin
10/16/2018 10:25 PM <DIR> Desktop
11/4/2018 3:31 PM <DIR> dow
10/16/2018 8:21 PM <DIR> Downloads
10/14/2018 5:34 PM <DIR> msc
9/9/2018 9:32 AM <DIR> opt
10/3/2018 2:58 PM 34,359,738,368 win10.img
6/30/2018 8:36 AM 3,939,237,888 win10.iso
7/26/2018 1:11 PM 189 win10.sh
10/29/2018 11:46 AM 34,359,738,368 win7.img
6/30/2018 8:35 AM 3,319,791,616 win7.iso
10/29/2018 10:55 AM 219 win7.sh
6 files 75,978,506,648 bytes
3 directories 22,198,362,112 bytes free
""")
test_str = test_str.strip().split("\n")
entities = {}
parse = aioftp.Client(encoding="utf-8").parse_list_line_windows
for x in test_str:
with contextlib.suppress(ValueError):
path, stat = parse(x.encode("utf-8"))
entities[path] = stat
dirs = ["bin", "Desktop", "dow", "Downloads", "msc", "opt"]
files = [
"win10.img", "win10.iso", "win10.sh", "win7.img", "win7.iso", "win7.sh"
]
assert len(entities) == len(dirs + files)
for d in dirs:
p = pathlib.PurePosixPath(d)
assert p in entities
assert entities[p]["type"] == "dir"
for f in files:
p = pathlib.PurePosixPath(f)
assert p in entities
assert entities[p]["type"] == "file"
with pytest.raises(ValueError):
parse(b" 10/3/2018 2:58 PM 34,35xxx38,368 win10.img")
async def __aenter__(self):
await self._lock.acquire()
client = self._client
try:
if client is None:
client = aioftp.Client()
await client.connect(self._settings.host, self._settings.port)
await client.login(self._settings.user,
self._settings.password,
aioftp.DEFAULT_ACCOUNT)
self._client = client
if self._expire is not None:
self._expire.cancel()
self._expire = self._loop.call_later(CACHE_TIME,
self._loop.create_task,
self.close())
return client
except Exception:
client.close()
self._lock.release()
raise
async def download_ftp(host: str, filename: Path, output_dir: Path,
semaphore: asyncio.Semaphore):
async with semaphore:
try:
async with timeout(1):
client = aioftp.Client()
await client.connect(host)
session = await client.login()
print(
f"started download {filename} for {host} in folder {output_dir}"
)
stream = await session.download_stream(filename)
async with aiofiles.open(output_dir / filename.name,
mode="wb") as f:
async for chunk in stream.iter_by_block():
await f.write(chunk)
except asyncio.TimeoutError as e:
print(f"timeout error {filename} for {host}")
finally:
client.close()
session.close()
def test_parse_list_line_unix():
lines = {
"file": [
"-rw-rw-r-- 1 poh poh 6595 Feb 27 04:14 history.rst",
"lrwxrwxrwx 1 poh poh 6 Mar 23 05:46 link-tmp.py -> tmp.py",
],
"dir": [
"drw-rw-r-- 1 poh poh 6595 Feb 27 04:14 history.rst",
],
"unknown": [
"Erw-rw-r-- 1 poh poh 6595 Feb 27 04:14 history.rst",
]
}
p = aioftp.Client(encoding="utf-8").parse_list_line_unix
for t, stack in lines.items():
for line in stack:
_, parsed = p(line.encode("utf-8"))
assert parsed["type"] == t
with pytest.raises(ValueError):
p(b"-rw-rw-r-- 1 poh poh 6xx5 Feb 27 04:14 history.rst")
with pytest.raises(ValueError):
p(b"-rw-rw-r-- xx poh poh 6595 Feb 27 04:14 history.rst")
def ftp_login(self):
self.ftp_object = aioftp.Client()
self.ftp_object.connect(self.ftp_uri)
self.ftp_object.login()
print('logged into gutenberg ftp mirror')
def __init__(self,
database_settings: DatabaseSettings,
stripe_settings: StripeSettings,
smtp_settings: SmtpSettings,
friendly_url: str,
frontend_url: str,
root_steam_id: str,
system_email: str,
upload_settings: Tuple[B2UploadSettings,
LocalUploadSettings] = None,
map_images: Dict[str, str] = MAP_IMAGES,
upload_delay: float = 0.00001,
free_upload_size: float = 30.0,
max_upload_size: float = 100.0,
timestamp_format: str = "%m/%d/%Y-%H:%M:%S",
community_types: List[str] = COMMUNITY_TYPES,
webhook_settings: WebhookSettings = WebhookSettings(),
match_max_length: timedelta = timedelta(hours=3),
demo_expires: timedelta = timedelta(weeks=20),
subscription_length: timedelta = timedelta(days=31),
clear_cache: bool = True,
**kwargs) -> None:
"""SQLMatches API.
Parameters
----------
database_settings : DatabaseSettings
stripe_settings : StripeSettings
friendly_url : str
frontend_url : str
root_steam_id : str
system_email : str
upload_settings : [B2UploadSettings, LocalUploadSettings], optional
by default None
map_images : Dict[str, str], optional
by default MAP_IMAGES
upload_delay : float, optional
by default 0.00001
free_upload_size : float, optional
by default 50.0
max_upload_size : float, optional
by default 100.0
timestamp_format : str, optional
by default "%m/%d/%Y-%H:%M:%S"
community_types : List[str], optional
by default COMMUNITY_TYPES
webhook_settings : WebhookSettings, optional
by default WebhookSettings()
match_max_length : timedelta, optional
by default timedelta(hours=3)
clear_cache : bool, optional
by default True
demo_expires : timedelta, optional
by default timedelta(weeks=20)
"""
startup_tasks = [self._startup]
shutdown_tasks = [self._shutdown]
if "on_startup" in kwargs:
startup_tasks = startup_tasks + kwargs["on_startup"]
if "on_shutdown" in kwargs:
shutdown_tasks = shutdown_tasks + kwargs["on_shutdown"]
middlewares = [
Middleware(SessionMiddleware,
secret_key=KeyLoader(name="session").load()),
Middleware(AuthenticationMiddleware,
backend=APIAuthentication(),
on_error=auth_error),
Middleware(CORSMiddleware,
allow_origins=["*"],
allow_methods=["GET", "POST", "DELETE", "OPTIONS"])
]
if "middleware" in kwargs:
middlewares = middlewares + kwargs["middleware"]
if "routes" in kwargs:
routes = kwargs["routes"] + ROUTES
else:
routes = ROUTES
if "exception_handlers" in kwargs:
exception_handlers = kwargs["exception_handlers"] + ERROR_HANDLERS
else:
exception_handlers = ERROR_HANDLERS
if friendly_url[:1] != "/":
friendly_url += "/"
if frontend_url[:1] != "/":
frontend_url += "/"
Config.url = friendly_url
Config.map_images = map_images
Config.upload_delay = upload_delay
Config.free_upload_size = free_upload_size
Config.max_upload_size = max_upload_size
Config.timestamp_format = timestamp_format
Config.root_steam_id_hashed = bcrypt.hashpw(root_steam_id.encode(),
bcrypt.gensalt())
Config.root_webhook_key_hashed = bcrypt.hashpw(
(KeyLoader("webhook").load()).encode(), bcrypt.gensalt())
Config.webhook_timeout = webhook_settings.timeout
Config.webhook_match_end = webhook_settings.match_end
Config.webhook_match_start = webhook_settings.match_start
Config.webhook_round_end = webhook_settings.round_end
Config.webhook_key = webhook_settings.key
Config.match_max_length = match_max_length
Config.system_email = system_email
Config.frontend_url = frontend_url
Config.demo_expires = demo_expires
Config.price_id = stripe_settings.price_id
Config.subscription_length = subscription_length
self.community_types = community_types
self.clear_cache = clear_cache
database_url = "://{}:{}@{}:{}/{}?charset=utf8mb4".format(
database_settings.username, database_settings.password,
database_settings.server, database_settings.port,
database_settings.database)
Sessions.database = Database(database_settings.engine + database_url)
create_tables("{}+{}{}".format(database_settings.engine,
database_settings.alchemy_engine,
database_url))
Sessions.smtp = SMTP(hostname=smtp_settings.hostname,
port=smtp_settings.port,
use_tls=smtp_settings.use_tls,
password=smtp_settings.password,
username=smtp_settings.username)
Sessions.stripe = Stripe(stripe_settings.api_key,
stripe_settings.testing)
Sessions.ftp = aioftp.Client()
if upload_settings:
Config.demo_pathway = upload_settings.pathway
Config.demo_extension = upload_settings.extension
if isinstance(upload_settings, B2UploadSettings):
Config.cdn_url = upload_settings.cdn_url
Config.upload_type = B2UploadSettings
self.b2 = backblaze.Awaiting(upload_settings.key_id,
upload_settings.application_key)
Sessions.bucket = self.b2.bucket(upload_settings.bucket_id)
elif isinstance(upload_settings, LocalUploadSettings):
Config.cdn_url = None
Config.upload_type = LocalUploadSettings
# Dynamically adding mount if local storage.
# Name attribute for Mount isn't working correctly here,
# so please don't change '/demos/'.
for mount in routes:
if type(mount) == Mount and mount.name == "api":
mount.app.routes.append(
Mount("/demos/",
StaticFiles(directory=Config.demo_pathway)))
break
logger.warning(
"Using local storage for demos, use b2 for production.")
else:
Config.upload_type = None
super().__init__(routes=routes,
exception_handlers=exception_handlers,
middleware=middlewares,
on_startup=startup_tasks,
on_shutdown=shutdown_tasks,
**kwargs)
def test_parse_list_line_failed():
with pytest.raises(ValueError):
aioftp.Client(encoding="utf-8").parse_list_line(b"what a hell?!")
def __init__(self, config: Config):
self.client = aioftp.Client()
self.files = config.get('files', [])
self.config = config
