async def updateCollection(self,
urls: List[str],
message: Optional[Message] = None):
result = self.sites.match(urls)
if not result:
raise NazurinError('No source matched')
logger.info('Collection update: site=%s, match=%s', result['site'],
result['match'].groups())
illust = await self.sites.handle_update(result)
# Send / Forward to gallery & Save to album
# If there're multiple images, then send a new message instead of
# forwarding an existing one, since we currently can't forward albums correctly.
if message and message.is_forward(
) and not illust.has_multiple_images():
save = asyncio.create_task(message.forward(config.GALLERY_ID))
elif not illust.has_image():
save = asyncio.create_task(
self.send_message(config.GALLERY_ID, '\n'.join(urls)))
else:
save = asyncio.create_task(
self.sendIllust(illust, message, config.GALLERY_ID))
download = asyncio.create_task(illust.download())
await asyncio.gather(save, download)
await self.storage.store(illust)
return True
async def chosen_url(self) -> str:
# Conform with limitations of sending photos: https://core.telegram.org/bots/api#sendphoto
if self._chosen_url:
return self._chosen_url
if self.height != 0 and self.width / self.height > 20:
raise NazurinError(
'Width and height ratio of image exceeds 20, try download option.'
)
self._chosen_url = self.url
if self.thumbnail:
# For safety reasons, use thumbnail when image size is unkown
if (not self.width) or (
not self.height) or self.width + self.height > 10000:
self._chosen_url = self.thumbnail
logger.info(
'Use thumbnail (Unkown image size or width + height > 10000 [%s, %s]): %s',
self.width, self.height, self._chosen_url)
else:
size = await self.size()
if (not size) or size > 5 * 1024 * 1024:
self._chosen_url = self.thumbnail
logger.info(
'Use thumbnail (Unknown size or size > 5MB [%s]): %s',
size, self._chosen_url)
return self._chosen_url
def load(self):
"""Dynamically load all storage drivers."""
for driver_name in STORAGE:
driver = importlib.import_module('nazurin.storage.' +
driver_name.lower())
self.disks.append(getattr(driver, driver_name)())
logger.info("Storage loaded")
async def requireAuth(self):
if Pixiv.api.access_token and time.time() - Pixiv.updated_time < 3600:
# Logged in, access_token not expired
return
if Pixiv.api.refresh_token:
# Logged in, access_token expired
await self.refreshToken()
return
# Haven't logged in
tokens = await Pixiv.document.get()
if tokens:
Pixiv.api.access_token = tokens['access_token']
Pixiv.api.refresh_token = tokens['refresh_token']
Pixiv.updated_time = tokens['updated_time']
if time.time() - Pixiv.updated_time >= 3600: # Token expired
await self.refreshToken()
else:
logger.info('Pixiv logged in through cached tokens')
else: # Initialize database
if not REFRESH_TOKEN:
raise NazurinError('Pixiv refresh token is required')
Pixiv.api.refresh_token = REFRESH_TOKEN
await Pixiv.api_auth()
Pixiv.updated_time = time.time()
await Pixiv.collection.insert(
DOCUMENT, {
'access_token': Pixiv.api.access_token,
'refresh_token': Pixiv.api.refresh_token,
'updated_time': Pixiv.updated_time
})
logger.info('Pixiv tokens cached')
async def auth(self, initialize=False):
# https://docs.microsoft.com/zh-cn/azure/active-directory/develop/v2-oauth2-auth-code-flow
url = 'https://login.microsoftonline.com/common/oauth2/v2.0/token'
data = {
'client_id': OD_CLIENT,
'client_secret': OD_SECRET,
'refresh_token': self.refresh_token,
'grant_type': 'refresh_token'
}
async with Request() as request:
async with request.post(url, data=data) as response:
response = await response.json()
self.access_token = response['access_token']
self.refresh_token = response['refresh_token']
self.expires_at = time.time() + response['expires_in']
credentials = {
'access_token': response['access_token'],
'refresh_token': response['refresh_token'],
'expires_at': self.expires_at
}
if initialize:
await self.collection.insert(OD_DOCUMENT, credentials)
logger.info('OneDrive logged in')
else:
await self.document.update(credentials)
logger.info('OneDrive access token updated')
async def bookmark(self, artwork_id: int):
response = await self.call(Pixiv.illust_bookmark_add, artwork_id)
if 'error' in response.keys():
logger.error(response)
raise NazurinError(response['error']['user_message'])
else:
logger.info('Bookmarked artwork %s', artwork_id)
return True
async def getDestination(self):
# Try to find the folder and its id
self.folder_id = await self.findFolder(OD_FOLDER)
# Not found, create a new folder
if not self.folder_id:
self.folder_id = await self.createFolder(OD_FOLDER)
await self.document.update({'folder_id': self.folder_id})
logger.info('OneDrive folder ID cached')
async def getDestination(self):
result = await Mega.find_folder(STORAGE_DIR, exclude_deleted=True)
if result:
Mega.destination = result[0]
else:
result = await Mega.create_folder(STORAGE_DIR)
Mega.destination = result[STORAGE_DIR]
await Mega.document.update({'destination': Mega.destination})
logger.info('MEGA destination cached')
async def refreshToken(self):
"""Refresh tokens and cache in database."""
await Pixiv.api_auth()
Pixiv.updated_time = time.time()
await Pixiv.document.update({
'access_token': Pixiv.api.access_token,
'refresh_token': Pixiv.api.refresh_token,
'updated_time': Pixiv.updated_time
})
logger.info('Pixiv tokens updated')
def start(self):
self.init()
if config.ENV == 'production':
logger.info('Set webhook')
self.executor.set_webhook(webhook_path='/' + config.TOKEN,
web_app=self.server)
self.executor.run_app(host="0.0.0.0", port=config.PORT)
else:
# self.server.start()
executor.start_polling(self, skip_updates=True)
async def upload(self, file: File):
while True:
try:
await Mega.api_upload(file.path, Mega.destination)
break
except RequestError as error:
# mega.errors.RequestError: ESID, Invalid or expired user session, please relogin
if 'relogin' in error.message:
logger.info(error)
Mega.api.sid = None
await self.login()
async def bookmark(self,
artwork_id: int,
privacy: PixivPrivacy = PixivPrivacy.PUBLIC):
response = await self.call(Pixiv.illust_bookmark_add, artwork_id,
privacy.value)
if 'error' in response.keys():
logger.error(response)
raise NazurinError(response.error.user_message
or response.error.message)
logger.info('Bookmarked artwork %s, privacy = %s', artwork_id,
privacy.value)
return True
def convert(config: File, output: File):
cmd = f'ffmpeg -i {config.path} -vcodec libx264 -vf "pad=ceil(iw/2)*2:ceil(ih/2)*2" -y {output.path}'
logger.info('Calling FFmpeg with command: %s', cmd)
args = shlex.split(cmd)
try:
output = subprocess.check_output(args,
stderr=subprocess.STDOUT,
shell=False)
except subprocess.CalledProcessError as error:
logger.error('FFmpeg failed with code %s, output:\n %s',
error.returncode, error.output)
raise NazurinError(
'Failed to convert ugoira to mp4.') from None
async def size(self) -> int:
self._size = self._size or await super().size()
if self._size:
return self._size
async with Request(
headers={'Referer': 'https://www.pixiv.net/'}) as request:
async with request.head(self.url) as response:
headers = response.headers
if 'Content-Length' in headers.keys():
self._size = int(headers['Content-Length'])
logger.info('Got image size: %s', self._size)
else:
logger.info('Failed to get image size')
return self._size
async def requireAuth(self):
if not Mega.api.sid:
tokens = await Mega.document.get()
if tokens and 'sid' in tokens.keys():
Mega.api.sid = tokens['sid']
Mega.api.master_key = tuple(tokens['master_key'])
Mega.api.root_id = tokens['root_id']
logger.info('MEGA logged in through cached tokens')
if 'destination' in tokens.keys():
Mega.destination = tokens['destination']
logger.info('MEGA retrieved destination from cache')
else: # Initialize database
await self.login(initialize=True)
if not Mega.destination:
await self.getDestination()
def start(self):
self.init()
if config.ENV == 'production':
logger.info('Set webhook')
self.executor.set_webhook(webhook_path='/' + config.TOKEN,
web_app=self.server)
# Tell aiohttp to use main thread event loop instead of creating a new one
# otherwise bot commands will run in a different loop
# from main thread functions and classes like Mongo and Mega.api_upload,
# resulting in RuntimeError: Task attached to different loop
self.executor.run_app(host="0.0.0.0",
port=config.PORT,
loop=asyncio.get_event_loop())
else:
# self.server.start()
executor.start_polling(self, skip_updates=True)
def convert(config: File, output: File):
# For some illustrations like https://www.pixiv.net/artworks/44298467,
# the output video is in YUV444P colorspace, which can't be played on some devices,
# thus we convert to YUV420P colorspace for better compatibility.
cmd = f'ffmpeg -i "{config.path}" -vcodec libx264 -pix_fmt yuv420p -vf "pad=ceil(iw/2)*2:ceil(ih/2)*2" -y "{output.path}"'
logger.info('Calling FFmpeg with command: %s', cmd)
args = shlex.split(cmd)
try:
output = subprocess.check_output(args,
stderr=subprocess.STDOUT,
shell=False)
except subprocess.CalledProcessError as error:
logger.error('FFmpeg failed with code %s, output:\n %s',
error.returncode, error.output.decode())
raise NazurinError(
'Failed to convert ugoira to mp4.') from None
async def login(self, initialize=False):
await Mega.api_login(MEGA_USER, MEGA_PASS)
if initialize:
await Mega.collection.insert(
MEGA_DOCUMENT, {
'sid': Mega.api.sid,
'master_key': list(Mega.api.master_key),
'root_id': Mega.api.root_id
})
else:
await Mega.document.update({
'sid': Mega.api.sid,
'master_key': list(Mega.api.master_key),
'root_id': Mega.api.root_id
})
logger.info('MEGA tokens cached')
def load(self):
"""Dynamically load all site plugins."""
module_paths = glob('nazurin/sites/*/')
for module_path in module_paths:
module_name = path.basename(path.normpath(module_path))
if module_name.startswith('__'):
continue
module = import_module('nazurin.sites.' + module_name)
self.sites[module_name.lower()] = getattr(module, module_name)()
if hasattr(module, 'patterns') and hasattr(module, 'handle'):
PRIORITY = getattr(module, 'PRIORITY')
patterns = getattr(module, 'patterns')
handle = getattr(module, 'handle')
self.sources.append((PRIORITY, patterns, handle, module_name))
self.sources.sort(key=lambda s: s[0], reverse=True)
logger.info("Sites loaded")
async def requireAuth(self):
# https://docs.microsoft.com/zh-cn/azure/active-directory/develop/v2-oauth2-auth-code-flow
if self.access_token and self.expires_at > time.time():
# Logged in, access_token not expired
return
credentials = await self.document.get()
if credentials:
self.refresh_token = credentials['refresh_token']
if 'folder_id' in credentials.keys():
self.folder_id = credentials['folder_id']
if credentials['expires_at'] > time.time():
self.access_token = credentials['access_token']
self.expires_at = credentials['expires_at']
logger.info('OneDrive logged in through cached tokens')
else:
await self.auth() # Refresh access_token
else:
# Database should be initialized
self.refresh_token = OD_RF_TOKEN
await self.auth(initialize=True)
async def updateCollection(self,
urls: List[str],
message: Optional[Message] = None):
result = self.sites.match(urls)
if not result:
raise NazurinError('No source matched')
logger.info('Collection update: site=%s, match=%s', result['site'],
result['match'].groups())
illust = await self.sites.handle_update(result)
# Forward to gallery & Save to album
if message and message.is_forward():
save = asyncio.create_task(message.forward(config.GALLERY_ID))
elif not illust.has_image():
save = asyncio.create_task(
self.send_message(config.GALLERY_ID, '\n'.join(urls)))
else:
save = asyncio.create_task(
self.sendIllust(illust, chat_id=config.GALLERY_ID))
download = asyncio.create_task(illust.download())
await asyncio.gather(save, download)
await self.storage.store(illust)
return True
async def store(self, illust: Illust):
tasks = []
for disk in self.disks:
tasks.append(disk.store(illust.all_files))
await asyncio.gather(*tasks)
logger.info('Storage completed')
