async def create(self): kwargs = await self.request.json() kwargs = unpack_dict(kwargs, ( 'parent_id', 'name', )) ok = all((k in kwargs for k in ( 'parent_id', 'name', ))) if not ok: raise HTTPBadRequest() drive: Drive = self.request.app['drive'] parent_id = kwargs['parent_id'] name = kwargs['name'] parent = await drive.get_node_by_id(parent_id) try: node = await drive.create_folder( parent_node=parent, folder_name=name, exist_ok=False, ) return node.to_dict() except Exception as e: EXCEPTION('engine', e) << name << parent_id raise HTTPConflict() from e
def _scan_local(self, node_id: str) -> List[ImageDict]: assert self._storage is not None rv: List[ImageDict] = [] top = self._storage.get_path(node_id) for dirpath, dirnames, filenames in os.walk(top): dirnames.sort(key=FuzzyName) filenames.sort(key=FuzzyName) for filename in filenames: path = join_path(dirpath, filename) type_, encoding = guess_type(path) if type_ is None: continue if not type_.startswith('image/'): continue try: image = Image.open(path) except OSError as e: EXCEPTION('engine', e) << 'unknown image' continue width, height = image.size rv.append({ 'path': path, 'type': type_, 'size': getsize(path), 'width': width, 'height': height, }) return rv
async def _guard(self): try: return await self._main() except Exception as e: EXCEPTION('ddld', e) finally: self._loop.stop() return 1
async def __call__(self, args): self._kwargs = parse_args(args[1:]) self._finished = asyncio.Event() loop = asyncio.get_running_loop() loop.add_signal_handler(signal.SIGINT, self._close) loop.add_signal_handler(signal.SIGTERM, self._close) try: return await self._main() except Exception as e: EXCEPTION('engine', e) return 1
async def search(self, pattern): real_pattern = normalize_search_pattern(pattern) try: re.compile(real_pattern) except Exception as e: EXCEPTION('ddld', e) << real_pattern raise u.InvalidPatternError(real_pattern) se = self._context.search_engine nodes = await se.get_nodes_by_regex(real_pattern) return nodes
async def trash_node(drive: Drive, id_or_path: str) -> Optional[str]: ''' :returns: None if succeed, id_or_path if failed ''' node = await get_node_by_id_or_path(drive, id_or_path) if not node: return id_or_path try: await drive.trash_node(node) except Exception as e: EXCEPTION('wcpan.drive', e) << 'trash failed' return id_or_path return None
async def _run_for_file( self, src: pathlib.Path, dst: pathlib.Path, ) -> Optional[pathlib.Path]: try: rv = await self.do_file(src, dst) except Exception as e: EXCEPTION('wcpan.drive', e) display = await self.get_source_display(src) self._add_failed(display) rv = None return rv
async def _close_request(self) -> None: if not self._bg: return None if not self._bg.done(): self._bg.cancel() try: rv = await self._bg return rv except (Exception, asyncio.CancelledError) as e: EXCEPTION('wcpan.drive.google', e) << 'close' finally: self._queue = asyncio.Queue(maxsize=1) self._bg = None
async def search_by_name( search_engine: SearchEngine, pattern: str, ): real_pattern = normalize_search_pattern(pattern) try: re.compile(real_pattern) except Exception as e: EXCEPTION('engine', e) << real_pattern raise InvalidPatternError(real_pattern) se = search_engine nodes = await se.get_nodes_by_regex(real_pattern) return nodes
async def json(self) -> Any: if self._parsed_json: return self._json # Google Drive API does not use application/json rv = await self._response.text() try: rv = json.loads(rv) except ValueError as e: EXCEPTION('wcpan.drive.google') << rv rv = None self._json = rv self._parsed_json = True return self._json
async def _search(self, pattern: str) -> List[SearchNodeDict]: lock = self._searching[pattern] try: nodes = await self._drive.find_nodes_by_regex(pattern) nodes = (_ for _ in nodes if not _.trashed) nodes = (self._make_item(_) for _ in nodes) nodes = await asyncio.gather(*nodes) nodes = sorted(nodes, key=lambda _: (_['path'], _['name'])) self._cache[pattern] = nodes return nodes except Exception as e: EXCEPTION('engine', e) << 'search failed, abort' raise SearchFailedError(str(e)) finally: del self._searching[pattern] async with lock: lock.notify_all()
async def _search(self, pattern): lock = self._searching[pattern] try: nodes = await self._drive.find_nodes_by_regex(pattern) nodes = { _.id_: self._drive.get_path(_) for _ in nodes if not _.trashed } nodes = await async_dict(nodes) self._cache[pattern] = nodes except Exception as e: EXCEPTION('ddld', e) << 'search failed, abort' raise u.SearchFailedError(str(e)) finally: del self._searching[pattern] async with lock: lock.notify_all()
async def _unpack(self, node: Node) -> List[ImageDict]: assert self._storage is not None lock = self._unpacking[node.id_] try: if node.is_folder: manifest = await self._unpack_remote(node) else: manifest = await self._unpack_local(node.id_) self._storage.set_cache(node.id_, manifest) return manifest except UnpackFailedError: raise except Exception as e: EXCEPTION('engine', e) << 'unpack failed, abort' raise UnpackFailedError(str(e)) finally: del self._unpacking[node.id_] async with lock: lock.notify_all()
async def _run_for_folder( self, src: pathlib.Path, dst: pathlib.Path, ) -> Optional[pathlib.Path]: try: rv = await self.do_folder(src, dst) except Exception as e: EXCEPTION('wcpan.drive', e) display = await self.get_source_display(src) self._add_failed(display) rv = None if not rv: return None children = await self.get_children(src) for child in children: fn = functools.partial(self._run_one_task, child, rv) self._queue.post(fn) return rv
async def renew_token(self, session: aiohttp.ClientSession): if self._refreshing: async with self._lock: await self._lock.wait() return if not self.access_token: raise UnauthorizedError() if not self.refresh_token: raise UnauthorizedError() async with self._guard(): try: await self._refresh(session) except Exception as e: EXCEPTION('wcpan.drive.google', e) << 'error on refresh token' self._error = True raise self._error = False DEBUG('wcpan.drive.google') << 'refresh access token'
async def _request_loop( self, kwargs: dict[str, Any], ) -> aiohttp.ClientResponse: while True: await self._wait_backoff() try: assert self._session is not None response = await self._session.request(**kwargs) except aiohttp.ClientConnectionError: self._adjust_backoff_level(True) raise status = str(response.status) rv = await self._check_status(status, response) if rv == Status.OK: return response if rv == Status.REFRESH: assert self._oauth is not None try: await self._oauth.renew_token(self._session) except UnauthorizedError: raise except Exception as e: raise UnauthorizedError() from e await self._update_token_header(kwargs['headers']) continue if rv == Status.BACKOFF: continue try: json_ = await response.json() except aiohttp.ContentTypeError as e: text = await response.text() EXCEPTION('wcpan.drive.google', e) << text raise self._raiseError(status, response, json_)