async def create(self):
kwargs = await self.request.json()
kwargs = unpack_dict(kwargs, (
'parent_id',
'name',
))
ok = all((k in kwargs for k in (
'parent_id',
'name',
)))
if not ok:
raise HTTPBadRequest()
drive: Drive = self.request.app['drive']
parent_id = kwargs['parent_id']
name = kwargs['name']
parent = await drive.get_node_by_id(parent_id)
try:
node = await drive.create_folder(
parent_node=parent,
folder_name=name,
exist_ok=False,
)
return node.to_dict()
except Exception as e:
EXCEPTION('engine', e) << name << parent_id
raise HTTPConflict() from e
def _scan_local(self, node_id: str) -> List[ImageDict]:
assert self._storage is not None
rv: List[ImageDict] = []
top = self._storage.get_path(node_id)
for dirpath, dirnames, filenames in os.walk(top):
dirnames.sort(key=FuzzyName)
filenames.sort(key=FuzzyName)
for filename in filenames:
path = join_path(dirpath, filename)
type_, encoding = guess_type(path)
if type_ is None:
continue
if not type_.startswith('image/'):
continue
try:
image = Image.open(path)
except OSError as e:
EXCEPTION('engine', e) << 'unknown image'
continue
width, height = image.size
rv.append({
'path': path,
'type': type_,
'size': getsize(path),
'width': width,
'height': height,
})
return rv
async def _guard(self):
try:
return await self._main()
except Exception as e:
EXCEPTION('ddld', e)
finally:
self._loop.stop()
return 1
async def __call__(self, args):
self._kwargs = parse_args(args[1:])
self._finished = asyncio.Event()
loop = asyncio.get_running_loop()
loop.add_signal_handler(signal.SIGINT, self._close)
loop.add_signal_handler(signal.SIGTERM, self._close)
try:
return await self._main()
except Exception as e:
EXCEPTION('engine', e)
return 1
async def search(self, pattern):
real_pattern = normalize_search_pattern(pattern)
try:
re.compile(real_pattern)
except Exception as e:
EXCEPTION('ddld', e) << real_pattern
raise u.InvalidPatternError(real_pattern)
se = self._context.search_engine
nodes = await se.get_nodes_by_regex(real_pattern)
return nodes
async def trash_node(drive: Drive, id_or_path: str) -> Optional[str]:
'''
:returns: None if succeed, id_or_path if failed
'''
node = await get_node_by_id_or_path(drive, id_or_path)
if not node:
return id_or_path
try:
await drive.trash_node(node)
except Exception as e:
EXCEPTION('wcpan.drive', e) << 'trash failed'
return id_or_path
return None
async def _run_for_file(
self,
src: pathlib.Path,
dst: pathlib.Path,
) -> Optional[pathlib.Path]:
try:
rv = await self.do_file(src, dst)
except Exception as e:
EXCEPTION('wcpan.drive', e)
display = await self.get_source_display(src)
self._add_failed(display)
rv = None
return rv
async def _close_request(self) -> None:
if not self._bg:
return None
if not self._bg.done():
self._bg.cancel()
try:
rv = await self._bg
return rv
except (Exception, asyncio.CancelledError) as e:
EXCEPTION('wcpan.drive.google', e) << 'close'
finally:
self._queue = asyncio.Queue(maxsize=1)
self._bg = None
async def search_by_name(
search_engine: SearchEngine,
pattern: str,
):
real_pattern = normalize_search_pattern(pattern)
try:
re.compile(real_pattern)
except Exception as e:
EXCEPTION('engine', e) << real_pattern
raise InvalidPatternError(real_pattern)
se = search_engine
nodes = await se.get_nodes_by_regex(real_pattern)
return nodes
async def json(self) -> Any:
if self._parsed_json:
return self._json
# Google Drive API does not use application/json
rv = await self._response.text()
try:
rv = json.loads(rv)
except ValueError as e:
EXCEPTION('wcpan.drive.google') << rv
rv = None
self._json = rv
self._parsed_json = True
return self._json
async def _search(self, pattern: str) -> List[SearchNodeDict]:
lock = self._searching[pattern]
try:
nodes = await self._drive.find_nodes_by_regex(pattern)
nodes = (_ for _ in nodes if not _.trashed)
nodes = (self._make_item(_) for _ in nodes)
nodes = await asyncio.gather(*nodes)
nodes = sorted(nodes, key=lambda _: (_['path'], _['name']))
self._cache[pattern] = nodes
return nodes
except Exception as e:
EXCEPTION('engine', e) << 'search failed, abort'
raise SearchFailedError(str(e))
finally:
del self._searching[pattern]
async with lock:
lock.notify_all()
async def _search(self, pattern):
lock = self._searching[pattern]
try:
nodes = await self._drive.find_nodes_by_regex(pattern)
nodes = {
_.id_: self._drive.get_path(_)
for _ in nodes if not _.trashed
}
nodes = await async_dict(nodes)
self._cache[pattern] = nodes
except Exception as e:
EXCEPTION('ddld', e) << 'search failed, abort'
raise u.SearchFailedError(str(e))
finally:
del self._searching[pattern]
async with lock:
lock.notify_all()
async def _unpack(self, node: Node) -> List[ImageDict]:
assert self._storage is not None
lock = self._unpacking[node.id_]
try:
if node.is_folder:
manifest = await self._unpack_remote(node)
else:
manifest = await self._unpack_local(node.id_)
self._storage.set_cache(node.id_, manifest)
return manifest
except UnpackFailedError:
raise
except Exception as e:
EXCEPTION('engine', e) << 'unpack failed, abort'
raise UnpackFailedError(str(e))
finally:
del self._unpacking[node.id_]
async with lock:
lock.notify_all()
async def _run_for_folder(
self,
src: pathlib.Path,
dst: pathlib.Path,
) -> Optional[pathlib.Path]:
try:
rv = await self.do_folder(src, dst)
except Exception as e:
EXCEPTION('wcpan.drive', e)
display = await self.get_source_display(src)
self._add_failed(display)
rv = None
if not rv:
return None
children = await self.get_children(src)
for child in children:
fn = functools.partial(self._run_one_task, child, rv)
self._queue.post(fn)
return rv
async def renew_token(self, session: aiohttp.ClientSession):
if self._refreshing:
async with self._lock:
await self._lock.wait()
return
if not self.access_token:
raise UnauthorizedError()
if not self.refresh_token:
raise UnauthorizedError()
async with self._guard():
try:
await self._refresh(session)
except Exception as e:
EXCEPTION('wcpan.drive.google', e) << 'error on refresh token'
self._error = True
raise
self._error = False
DEBUG('wcpan.drive.google') << 'refresh access token'
async def _request_loop(
self,
kwargs: dict[str, Any],
) -> aiohttp.ClientResponse:
while True:
await self._wait_backoff()
try:
assert self._session is not None
response = await self._session.request(**kwargs)
except aiohttp.ClientConnectionError:
self._adjust_backoff_level(True)
raise
status = str(response.status)
rv = await self._check_status(status, response)
if rv == Status.OK:
return response
if rv == Status.REFRESH:
assert self._oauth is not None
try:
await self._oauth.renew_token(self._session)
except UnauthorizedError:
raise
except Exception as e:
raise UnauthorizedError() from e
await self._update_token_header(kwargs['headers'])
continue
if rv == Status.BACKOFF:
continue
try:
json_ = await response.json()
except aiohttp.ContentTypeError as e:
text = await response.text()
EXCEPTION('wcpan.drive.google', e) << text
raise
self._raiseError(status, response, json_)
