async def handler_image_to_vector(request):
reader = await request.multipart()
field = await reader.next()
image_path = Path(FILES_DIR, field.filename)
with open(image_path, 'wb') as f:
while True:
chunk = await field.read_chunk()
if not chunk:
break
f.write(chunk)
result = image_to_vector_process(image_path).tolist()
wrap(os.remove(image_path))
return web.json_response({'error': False, 'result': result})
async def download_files(self, app: Application):
"""starts the download and waits for all files to finish"""
# run this async, parfive will support aiofiles in the future as stated above
wrapped_function = aiofiles_os.wrap(self.downloader.download)
exporter_settings = get_settings(app)
results = await wrapped_function(
timeouts={
"total": exporter_settings.downloader_max_timeout_seconds,
"sock_read": 90, # default as in parfive code
}
)
log.debug("Download results %s", results)
if len(results) != self.total_files_added:
raise ExporterException(
"Not all files were downloaded. Please check the logs above."
)
import asyncio
import os
import tempfile
from pathlib import Path
from typing import Optional, Union
from aiofiles import os as aiofiles_os
from aiohttp.abc import AbstractStreamWriter
from aiohttp.typedefs import LooseHeaders
from aiohttp.web import FileResponse
makedirs = aiofiles_os.wrap(os.makedirs) # as in aiofiles.os.py module
rename = aiofiles_os.wrap(os.rename) # as in aiofiles.os.py module
path_getsize = aiofiles_os.wrap(os.path.getsize) # as in aiofiles.os.py module
def _candidate_tmp_dir() -> Path:
# pylint: disable=protected-access
# let us all thank codeclimate for this beautiful piece of code
return Path("/") / f"tmp/{next(tempfile._get_candidate_names())}"
async def get_empty_tmp_dir() -> str:
candidate = _candidate_tmp_dir()
while candidate.is_dir() or candidate.is_file() or candidate.is_symlink():
candidate = _candidate_tmp_dir()
await makedirs(candidate, exist_ok=True)
return str(candidate)
class dogstatsd:
create_event = wrap(datadog.api.Event.create)
gauge = wrap(datadog.statsd.gauge)
histogram = wrap(datadog.statsd.histogram)
increment = wrap(datadog.statsd.increment)
import os
import shutil
from typing import Callable, cast
from aiofiles import open as _open
from aiofiles.os import wrap
from aiofiles.threadpool import AsyncTextIOWrapper # type: ignore
class AsyncTextIO(AsyncTextIOWrapper):
async def __aenter__(self):
...
async def __aexit__(self, exc_type, exc, tb):
...
open = cast(Callable[..., AsyncTextIO], _open) # noqa: A001
remove = wrap(os.remove)
makedirs = wrap(os.makedirs)
hardlink = wrap(os.link)
rmtree = wrap(shutil.rmtree)
copy = wrap(shutil.copy2)
symlink = wrap(os.symlink)
isdir = wrap(os.path.isdir)
__all__ = ["open", "remove", "rmtree", "makedirs", "hardlink", "copy", "symlink", "isdir"]
import aiofiles
# Type hinting for wrap must be turned off until (1) is resolved.
# (1) https://github.com/Tinche/aiofiles/issues/8
from aiofiles.os import wrap # type: ignore
from pydantic import (
BaseModel,
StrictBytes,
StrictFloat,
StrictInt,
StrictStr,
root_validator,
)
_copy = wrap(shutil.copy)
strict_number = Union[StrictInt, StrictFloat]
record_data = Union[List[strict_number], Mapping[StrictStr, strict_number],
Mapping[StrictInt, strict_number], List[StrictBytes], ]
def parse_json_key_as_int(obj):
if isinstance(obj, dict):
return {int(k): v for k, v in obj.items()}
return obj
class _DataElement(BaseModel):
class Config:
validate_all = True
def better_file_not_found_error(*files: Union[str, Path], purpose: str = None):
if purpose:
additional = f'required for {purpose } '
else:
additional = ''
for i, path in enumerate(files):
path = Path(path)
if not path.exists():
return f'Path {path} {additional}was not found'
return f'Unknown file {additional}was not found'
###Coroutines###
makedirs = wrap(os.makedirs)
async def time_coro(coro):
start_time = time.time()
await coro
return time.time() - start_time
"""
async def benchmark(async_func: Callable, *args, num_times: int = 5, quiet: bool = False, cleanup: Callable = None,
cleanup_args=(), num_items: int = None, num_bytes: int = None, ignore_first: bool = True, **kwargs):
times = []
if not quiet:
print("Running", async_func.__name__)
total = 0