async def get_sauce(self,
tweet_cache: TweetCache,
index_no: int = 0,
log_index: typing.Optional[str] = None,
trigger: str = TRIGGER_MENTION) -> TweetSauceCache:
"""
Get the sauce of a media tweet
"""
log_index = log_index or 'SYSTEM'
# Have we cached the sauce already?
try:
sauce_manager = SauceManager(tweet_cache, trigger)
return await sauce_manager.get(index_no)
except ShortLimitReachedException:
self.log.warning(
f"[{log_index}] Short API limit reached, throttling for 30 seconds"
)
await asyncio.sleep(30.0)
return await self.get_sauce(tweet_cache, index_no, log_index)
except DailyLimitReachedException:
self.log.error(
f"[{log_index}] Daily API limit reached, throttling for 15 minutes. Please consider upgrading your API key."
)
await asyncio.sleep(900.0)
return await self.get_sauce(tweet_cache, index_no, log_index)
except SauceNaoException as e:
self.log.error(f"[{log_index}] SauceNao exception raised: {e}")
sauce_cache = TweetSauceCache.set(tweet_cache,
index_no=index_no,
trigger=trigger)
return sauce_cache
async def _get_sauce(self, index: int) -> typing.Optional[TweetSauceCache]:
cache = TweetSauceCache.fetch(self.tweet_cache.tweet_id, index)
if cache:
return cache
media = TweetManager.extract_media(self.tweet_cache.tweet)[index]
file = media
if self._downloads_enabled:
file = await self._download_media(media)
if self._downloads_enabled:
sauce_results = await self.sauce.from_file(io.BytesIO(file))
self._log.info(f"Performing saucenao lookup via file upload")
else:
self._log.info(f"Performing saucenao lookup via URL {file}")
sauce_results = await self.sauce.from_url(file)
# No results?
if not sauce_results:
sauce_cache = TweetSauceCache.set(self.tweet_cache, sauce_results,
index, self._trigger)
return sauce_cache
best_result = sauce_results[0]
# Attempt to download a video preview, if it's an anime result
video_preview = None
if self._previews_enabled and isinstance(best_result, AnimeSource):
file = io.BytesIO(file) if self._downloads_enabled else file
is_url = not self._downloads_enabled
video_preview = await self._video_preview(best_result, file,
is_url)
# If we have a video preview, upload it now!
media_id = None
if video_preview:
video_preview = io.BytesIO(video_preview)
media_id = await self._upload_video(video_preview)
return TweetSauceCache.set(self.tweet_cache, sauce_results, index,
self._trigger, media_id)
async def cleanup() -> None:
"""
Purge stale cache entries from the database and display some general analytics
Returns:
None
"""
while True:
try:
# Cache purging
stale_count = TweetCache.purge()
print(
f"\nPurging {stale_count:,} stale cache entries from the database"
)
# Sauce analytics
sauce_count = TweetSauceCache.sauce_count(900)
print(f"We've processed {sauce_count:,} new sauce queries!")
await asyncio.sleep(900.0)
except Exception:
log.exception(
"An unknown error occurred while performing cleanup tasks")
await asyncio.sleep(300.0)
async def get_sauce(
self,
tweet_cache: TweetCache,
index_no: int = 0,
log_index: Optional[str] = None,
trigger: str = TRIGGER_MENTION
) -> Tuple[TweetSauceCache, Optional[bytes]]:
"""
Get the sauce of a media tweet
"""
log_index = log_index or 'SYSTEM'
# Have we cached the sauce already?
cache = TweetSauceCache.fetch(tweet_cache.tweet_id, index_no)
if cache:
return cache, None
media = TweetManager.extract_media(tweet_cache.tweet)[index_no]
# Execute a Tracemoe search query for anime results
async def tracemoe_search(_sauce_results, _path: str,
is_url: bool) -> Optional[dict]:
if not self.tracemoe:
return None
if _sauce_results and isinstance(_sauce_results[0], AnimeSource):
# noinspection PyBroadException
try:
_tracemoe_sauce = await self.tracemoe.search(_path,
is_url=is_url)
except Exception:
self.log.warning(
f"[{log_index}] Tracemoe returned an exception, aborting search query"
)
return None
if not _tracemoe_sauce.get('docs'):
return None
# Make sure our search results match
if await _sauce_results[0].load_ids():
if _sauce_results[0].anilist_id != _tracemoe_sauce['docs'][
0]['anilist_id']:
self.log.info(
f"[{log_index}] saucenao and trace.moe provided mismatched anilist entries: `{_sauce_results[0].anilist_id}` vs. `{_tracemoe_sauce['docs'][0]['anilist_id']}`"
)
return None
self.log.info(
f'[{log_index}] Downloading video preview for AniList entry {_sauce_results[0].anilist_id} from trace.moe'
)
_tracemoe_preview = await self.tracemoe.video_preview_natural(
_tracemoe_sauce)
_tracemoe_sauce['docs'][0]['preview'] = _tracemoe_preview
return _tracemoe_sauce['docs'][0]
return None
# Look up the sauce
try:
if config.getboolean('SauceNao', 'download_files', fallback=False):
self.log.debug(f"[{log_index}] Downloading image from Twitter")
fd, path = tempfile.mkstemp()
try:
with os.fdopen(fd, 'wb') as tmp:
async with aiohttp.ClientSession(
raise_for_status=True) as session:
try:
async with await session.get(media
) as response:
image = await response.read()
tmp.write(image)
if not image:
self.log.error(
f"[{log_index}] Empty file received from Twitter"
)
sauce_cache = TweetSauceCache.set(
tweet_cache,
index_no=index_no,
trigger=trigger)
return sauce_cache
except aiohttp.ClientResponseError as error:
self.log.warning(
f"[{log_index}] Twitter returned a {error.status} error when downloading from tweet {tweet_cache.tweet_id}"
)
sauce_cache = TweetSauceCache.set(
tweet_cache,
index_no=index_no,
trigger=trigger)
return sauce_cache
sauce_results = await self.sauce.from_file(path)
tracemoe_sauce = await tracemoe_search(sauce_results,
_path=path,
is_url=False)
finally:
os.remove(path)
else:
self.log.debug(f"[{log_index}] Performing remote URL lookup")
sauce_results = await self.sauce.from_url(media)
tracemoe_sauce = await tracemoe_search(sauce_results,
_path=media,
is_url=True)
if not sauce_results:
sauce_cache = TweetSauceCache.set(tweet_cache,
sauce_results,
index_no,
trigger=trigger)
return sauce_cache, None
except ShortLimitReachedException:
self.log.warning(
f"[{log_index}] Short API limit reached, throttling for 30 seconds"
)
await asyncio.sleep(30.0)
return await self.get_sauce(tweet_cache, index_no, log_index)
except DailyLimitReachedException:
self.log.error(
f"[{log_index}] Daily API limit reached, throttling for 15 minutes. Please consider upgrading your API key."
)
await asyncio.sleep(900.0)
return await self.get_sauce(tweet_cache, index_no, log_index)
except SauceNaoException as e:
self.log.error(f"[{log_index}] SauceNao exception raised: {e}")
sauce_cache = TweetSauceCache.set(tweet_cache,
index_no=index_no,
trigger=trigger)
return sauce_cache, None
sauce_cache = TweetSauceCache.set(tweet_cache,
sauce_results,
index_no,
trigger=trigger)
return sauce_cache, tracemoe_sauce
async def get_sauce(
self,
tweet_cache: TweetCache,
index_no: int = 0,
log_index: Optional[str] = None,
trigger: str = TRIGGER_MENTION
) -> Tuple[TweetSauceCache, Optional[bytes]]:
"""
Get the sauce of a media tweet
"""
log_index = log_index or 'SYSTEM'
# Have we cached the sauce already?
cache = TweetSauceCache.fetch(tweet_cache.tweet_id, index_no)
if cache:
return cache, None
media = TweetManager.extract_media(tweet_cache.tweet)[index_no]
# Execute a Tracemoe search query for anime results
async def tracemoe_search(_sauce, is_url: bool) -> Optional[dict]:
if not self.tracemoe:
return None
if _sauce.results and _sauce.results[0].index_id in [21, 22]:
# noinspection PyBroadException
try:
_tracemoe_sauce = await self.tracemoe.search(path,
is_url=is_url)
except Exception:
self.log.warning(
f"[{log_index}] Tracemoe returned an exception, aborting search query"
)
return None
if not _tracemoe_sauce.get('docs'):
return None
# Check for an exactly title match first, then fallback to a similarity check.
# Obviously, this is not perfect. Titles don't always match, but sometimes tracemoe returns an accurate
# result with a lower similarity, so we just.. try and guess the best we can for now.
if _tracemoe_sauce['docs'][0]['similarity'] < 0.85:
if _tracemoe_sauce['docs'][0]['title_english'].lower(
) != sauce.results[0].title.lower():
if _tracemoe_sauce['docs'][0]['title_romaji'].lower(
) != sauce.results[0].title.lower():
return None
_tracemoe_preview = await self.tracemoe.video_preview_natural(
_tracemoe_sauce)
_tracemoe_sauce['docs'][0]['preview'] = _tracemoe_preview
return _tracemoe_sauce['docs'][0]
return None
# Look up the sauce
try:
if config.getboolean('SauceNao', 'download_files', fallback=False):
self.log.debug(f"[{log_index}] Downloading image from Twitter")
fd, path = tempfile.mkstemp()
try:
with os.fdopen(fd, 'wb') as tmp:
async with aiohttp.ClientSession(
raise_for_status=True) as session:
try:
async with await session.get(media
) as response:
image = await response.read()
tmp.write(image)
if not image:
self.log.error(
f"[{log_index}] Empty file received from Twitter"
)
sauce_cache = TweetSauceCache.set(
tweet_cache,
index_no=index_no,
trigger=trigger)
return sauce_cache
except aiohttp.ClientResponseError as error:
self.log.warning(
f"[{log_index}] Twitter returned a {error.status} error when downloading from tweet {tweet_cache.tweet_id}"
)
sauce_cache = TweetSauceCache.set(
tweet_cache,
index_no=index_no,
trigger=trigger)
return sauce_cache
sauce = await self.sauce.from_file(path)
tracemoe_sauce = await tracemoe_search(sauce,
is_url=False)
finally:
os.remove(path)
else:
self.log.debug(f"[{log_index}] Performing remote URL lookup")
sauce = await self.sauce.from_url(media)
tracemoe_sauce = await tracemoe_search(sauce, is_url=True)
if not sauce.results:
sauce_cache = TweetSauceCache.set(tweet_cache,
sauce,
index_no,
trigger=trigger)
return sauce_cache, None
except ShortLimitReachedException:
self.log.warning(
f"[{log_index}] Short API limit reached, throttling for 30 seconds"
)
await asyncio.sleep(30.0)
return await self.get_sauce(tweet_cache, index_no, log_index)
except DailyLimitReachedException:
self.log.error(
f"[{log_index}] Daily API limit reached, throttling for 15 minutes. Please consider upgrading your API key."
)
await asyncio.sleep(900.0)
return await self.get_sauce(tweet_cache, index_no, log_index)
except SauceNaoException as e:
self.log.error(f"[{log_index}] SauceNao exception raised: {e}")
sauce_cache = TweetSauceCache.set(tweet_cache,
index_no=index_no,
trigger=trigger)
return sauce_cache, None
sauce_cache = TweetSauceCache.set(tweet_cache,
sauce,
index_no,
trigger=trigger)
return sauce_cache, tracemoe_sauce