async def updateCollection(self, urls: List[str], message: Optional[Message] = None): result = self.sites.match(urls) if not result: raise NazurinError('No source matched') logger.info('Collection update: site=%s, match=%s', result['site'], result['match'].groups()) illust = await self.sites.handle_update(result) # Send / Forward to gallery & Save to album # If there're multiple images, then send a new message instead of # forwarding an existing one, since we currently can't forward albums correctly. if message and message.is_forward( ) and not illust.has_multiple_images(): save = asyncio.create_task(message.forward(config.GALLERY_ID)) elif not illust.has_image(): save = asyncio.create_task( self.send_message(config.GALLERY_ID, '\n'.join(urls))) else: save = asyncio.create_task( self.sendIllust(illust, message, config.GALLERY_ID)) download = asyncio.create_task(illust.download()) await asyncio.gather(save, download) await self.storage.store(illust) return True
async def chosen_url(self) -> str: # Conform with limitations of sending photos: https://core.telegram.org/bots/api#sendphoto if self._chosen_url: return self._chosen_url if self.height != 0 and self.width / self.height > 20: raise NazurinError( 'Width and height ratio of image exceeds 20, try download option.' ) self._chosen_url = self.url if self.thumbnail: # For safety reasons, use thumbnail when image size is unkown if (not self.width) or ( not self.height) or self.width + self.height > 10000: self._chosen_url = self.thumbnail logger.info( 'Use thumbnail (Unkown image size or width + height > 10000 [%s, %s]): %s', self.width, self.height, self._chosen_url) else: size = await self.size() if (not size) or size > 5 * 1024 * 1024: self._chosen_url = self.thumbnail logger.info( 'Use thumbnail (Unknown size or size > 5MB [%s]): %s', size, self._chosen_url) return self._chosen_url
def load(self): """Dynamically load all storage drivers.""" for driver_name in STORAGE: driver = importlib.import_module('nazurin.storage.' + driver_name.lower()) self.disks.append(getattr(driver, driver_name)()) logger.info("Storage loaded")
async def requireAuth(self): if Pixiv.api.access_token and time.time() - Pixiv.updated_time < 3600: # Logged in, access_token not expired return if Pixiv.api.refresh_token: # Logged in, access_token expired await self.refreshToken() return # Haven't logged in tokens = await Pixiv.document.get() if tokens: Pixiv.api.access_token = tokens['access_token'] Pixiv.api.refresh_token = tokens['refresh_token'] Pixiv.updated_time = tokens['updated_time'] if time.time() - Pixiv.updated_time >= 3600: # Token expired await self.refreshToken() else: logger.info('Pixiv logged in through cached tokens') else: # Initialize database if not REFRESH_TOKEN: raise NazurinError('Pixiv refresh token is required') Pixiv.api.refresh_token = REFRESH_TOKEN await Pixiv.api_auth() Pixiv.updated_time = time.time() await Pixiv.collection.insert( DOCUMENT, { 'access_token': Pixiv.api.access_token, 'refresh_token': Pixiv.api.refresh_token, 'updated_time': Pixiv.updated_time }) logger.info('Pixiv tokens cached')
async def auth(self, initialize=False): # https://docs.microsoft.com/zh-cn/azure/active-directory/develop/v2-oauth2-auth-code-flow url = 'https://login.microsoftonline.com/common/oauth2/v2.0/token' data = { 'client_id': OD_CLIENT, 'client_secret': OD_SECRET, 'refresh_token': self.refresh_token, 'grant_type': 'refresh_token' } async with Request() as request: async with request.post(url, data=data) as response: response = await response.json() self.access_token = response['access_token'] self.refresh_token = response['refresh_token'] self.expires_at = time.time() + response['expires_in'] credentials = { 'access_token': response['access_token'], 'refresh_token': response['refresh_token'], 'expires_at': self.expires_at } if initialize: await self.collection.insert(OD_DOCUMENT, credentials) logger.info('OneDrive logged in') else: await self.document.update(credentials) logger.info('OneDrive access token updated')
async def bookmark(self, artwork_id: int): response = await self.call(Pixiv.illust_bookmark_add, artwork_id) if 'error' in response.keys(): logger.error(response) raise NazurinError(response['error']['user_message']) else: logger.info('Bookmarked artwork %s', artwork_id) return True
async def getDestination(self): # Try to find the folder and its id self.folder_id = await self.findFolder(OD_FOLDER) # Not found, create a new folder if not self.folder_id: self.folder_id = await self.createFolder(OD_FOLDER) await self.document.update({'folder_id': self.folder_id}) logger.info('OneDrive folder ID cached')
async def getDestination(self): result = await Mega.find_folder(STORAGE_DIR, exclude_deleted=True) if result: Mega.destination = result[0] else: result = await Mega.create_folder(STORAGE_DIR) Mega.destination = result[STORAGE_DIR] await Mega.document.update({'destination': Mega.destination}) logger.info('MEGA destination cached')
async def refreshToken(self): """Refresh tokens and cache in database.""" await Pixiv.api_auth() Pixiv.updated_time = time.time() await Pixiv.document.update({ 'access_token': Pixiv.api.access_token, 'refresh_token': Pixiv.api.refresh_token, 'updated_time': Pixiv.updated_time }) logger.info('Pixiv tokens updated')
def start(self): self.init() if config.ENV == 'production': logger.info('Set webhook') self.executor.set_webhook(webhook_path='/' + config.TOKEN, web_app=self.server) self.executor.run_app(host="0.0.0.0", port=config.PORT) else: # self.server.start() executor.start_polling(self, skip_updates=True)
async def upload(self, file: File): while True: try: await Mega.api_upload(file.path, Mega.destination) break except RequestError as error: # mega.errors.RequestError: ESID, Invalid or expired user session, please relogin if 'relogin' in error.message: logger.info(error) Mega.api.sid = None await self.login()
async def bookmark(self, artwork_id: int, privacy: PixivPrivacy = PixivPrivacy.PUBLIC): response = await self.call(Pixiv.illust_bookmark_add, artwork_id, privacy.value) if 'error' in response.keys(): logger.error(response) raise NazurinError(response.error.user_message or response.error.message) logger.info('Bookmarked artwork %s, privacy = %s', artwork_id, privacy.value) return True
def convert(config: File, output: File): cmd = f'ffmpeg -i {config.path} -vcodec libx264 -vf "pad=ceil(iw/2)*2:ceil(ih/2)*2" -y {output.path}' logger.info('Calling FFmpeg with command: %s', cmd) args = shlex.split(cmd) try: output = subprocess.check_output(args, stderr=subprocess.STDOUT, shell=False) except subprocess.CalledProcessError as error: logger.error('FFmpeg failed with code %s, output:\n %s', error.returncode, error.output) raise NazurinError( 'Failed to convert ugoira to mp4.') from None
async def size(self) -> int: self._size = self._size or await super().size() if self._size: return self._size async with Request( headers={'Referer': 'https://www.pixiv.net/'}) as request: async with request.head(self.url) as response: headers = response.headers if 'Content-Length' in headers.keys(): self._size = int(headers['Content-Length']) logger.info('Got image size: %s', self._size) else: logger.info('Failed to get image size') return self._size
async def requireAuth(self): if not Mega.api.sid: tokens = await Mega.document.get() if tokens and 'sid' in tokens.keys(): Mega.api.sid = tokens['sid'] Mega.api.master_key = tuple(tokens['master_key']) Mega.api.root_id = tokens['root_id'] logger.info('MEGA logged in through cached tokens') if 'destination' in tokens.keys(): Mega.destination = tokens['destination'] logger.info('MEGA retrieved destination from cache') else: # Initialize database await self.login(initialize=True) if not Mega.destination: await self.getDestination()
def start(self): self.init() if config.ENV == 'production': logger.info('Set webhook') self.executor.set_webhook(webhook_path='/' + config.TOKEN, web_app=self.server) # Tell aiohttp to use main thread event loop instead of creating a new one # otherwise bot commands will run in a different loop # from main thread functions and classes like Mongo and Mega.api_upload, # resulting in RuntimeError: Task attached to different loop self.executor.run_app(host="0.0.0.0", port=config.PORT, loop=asyncio.get_event_loop()) else: # self.server.start() executor.start_polling(self, skip_updates=True)
def convert(config: File, output: File): # For some illustrations like https://www.pixiv.net/artworks/44298467, # the output video is in YUV444P colorspace, which can't be played on some devices, # thus we convert to YUV420P colorspace for better compatibility. cmd = f'ffmpeg -i "{config.path}" -vcodec libx264 -pix_fmt yuv420p -vf "pad=ceil(iw/2)*2:ceil(ih/2)*2" -y "{output.path}"' logger.info('Calling FFmpeg with command: %s', cmd) args = shlex.split(cmd) try: output = subprocess.check_output(args, stderr=subprocess.STDOUT, shell=False) except subprocess.CalledProcessError as error: logger.error('FFmpeg failed with code %s, output:\n %s', error.returncode, error.output.decode()) raise NazurinError( 'Failed to convert ugoira to mp4.') from None
async def login(self, initialize=False): await Mega.api_login(MEGA_USER, MEGA_PASS) if initialize: await Mega.collection.insert( MEGA_DOCUMENT, { 'sid': Mega.api.sid, 'master_key': list(Mega.api.master_key), 'root_id': Mega.api.root_id }) else: await Mega.document.update({ 'sid': Mega.api.sid, 'master_key': list(Mega.api.master_key), 'root_id': Mega.api.root_id }) logger.info('MEGA tokens cached')
def load(self): """Dynamically load all site plugins.""" module_paths = glob('nazurin/sites/*/') for module_path in module_paths: module_name = path.basename(path.normpath(module_path)) if module_name.startswith('__'): continue module = import_module('nazurin.sites.' + module_name) self.sites[module_name.lower()] = getattr(module, module_name)() if hasattr(module, 'patterns') and hasattr(module, 'handle'): PRIORITY = getattr(module, 'PRIORITY') patterns = getattr(module, 'patterns') handle = getattr(module, 'handle') self.sources.append((PRIORITY, patterns, handle, module_name)) self.sources.sort(key=lambda s: s[0], reverse=True) logger.info("Sites loaded")
async def requireAuth(self): # https://docs.microsoft.com/zh-cn/azure/active-directory/develop/v2-oauth2-auth-code-flow if self.access_token and self.expires_at > time.time(): # Logged in, access_token not expired return credentials = await self.document.get() if credentials: self.refresh_token = credentials['refresh_token'] if 'folder_id' in credentials.keys(): self.folder_id = credentials['folder_id'] if credentials['expires_at'] > time.time(): self.access_token = credentials['access_token'] self.expires_at = credentials['expires_at'] logger.info('OneDrive logged in through cached tokens') else: await self.auth() # Refresh access_token else: # Database should be initialized self.refresh_token = OD_RF_TOKEN await self.auth(initialize=True)
async def updateCollection(self, urls: List[str], message: Optional[Message] = None): result = self.sites.match(urls) if not result: raise NazurinError('No source matched') logger.info('Collection update: site=%s, match=%s', result['site'], result['match'].groups()) illust = await self.sites.handle_update(result) # Forward to gallery & Save to album if message and message.is_forward(): save = asyncio.create_task(message.forward(config.GALLERY_ID)) elif not illust.has_image(): save = asyncio.create_task( self.send_message(config.GALLERY_ID, '\n'.join(urls))) else: save = asyncio.create_task( self.sendIllust(illust, chat_id=config.GALLERY_ID)) download = asyncio.create_task(illust.download()) await asyncio.gather(save, download) await self.storage.store(illust) return True
async def store(self, illust: Illust): tasks = [] for disk in self.disks: tasks.append(disk.store(illust.all_files)) await asyncio.gather(*tasks) logger.info('Storage completed')