def __init__(self, media_tweet: TweetCache, trigger: str = TRIGGER_SELF):
self._log = logging.getLogger(__name__)
self._trigger = trigger
self.tweet_cache = media_tweet
self.media = TweetManager.extract_media(media_tweet.tweet) or []
self._downloads_enabled = config.getboolean('SauceNao', 'download_files', fallback=False)
self._previews_enabled = config.getboolean('TraceMoe', 'enabled', fallback=False)
# SauceNao
self.minsim_mentioned = float(config.get('SauceNao', 'min_similarity_mentioned', fallback=50.0))
self.minsim_monitored = float(config.get('SauceNao', 'min_similarity_monitored', fallback=65.0))
self.minsim_searching = float(config.get('SauceNao', 'min_similarity_searching', fallback=70.0))
self.persistent = config.getboolean('Twitter', 'enable_persistence', fallback=False)
self.anime_link = config.get('SauceNao', 'source_link', fallback='anidb').lower()
self.sauce = SauceNao(
api_key=config.get('SauceNao', 'api_key', fallback=None),
min_similarity=min(self.minsim_mentioned, self.minsim_monitored, self.minsim_searching),
priority=[21, 22, 5, 37, 25]
)
# Twython
self.twython = Twython(config.get('Twitter', 'consumer_key'), config.get('Twitter', 'consumer_secret'),
config.get('Twitter', 'access_token'), config.get('Twitter', 'access_secret'))
self._sauce_cache = {}
def __init__(self):
self.enabled = config.getboolean('Pixiv', 'enabled', fallback=False)
self._refresh_token = config.get('Pixiv', 'refresh_token', fallback=None)
self._log = logging.getLogger(__name__)
self._pixiv = AppPixivAPI()
self._pixiv.set_accept_language(config.get('Pixiv', 'language', fallback='en-US'))
self._re_twitter = re.compile(r'^https?://(www.)?twitter.com/(?P<username>.+)$')
if self.enabled:
self._login()
def __init__(self):
self.log = logging.getLogger(__name__)
# Tweet Cache Manager
self.twitter = TweetManager()
self.twython = Twython(config.get('Twitter', 'consumer_key'),
config.get('Twitter', 'consumer_secret'),
config.get('Twitter', 'access_token'),
config.get('Twitter', 'access_secret'))
self.anime_link = config.get('SauceNao',
'source_link',
fallback='anidb').lower()
self.nsfw_previews = config.getboolean('TraceMoe',
'nsfw_previews',
fallback=False)
self.failed_responses = config.getboolean('SauceNao',
'respond_to_failed',
fallback=True)
self.ignored_indexes = [
int(i) for i in config.get(
'SauceNao', 'ignored_indexes', fallback='').split(',')
]
# Pixiv
self.pixiv = Pixiv()
# Cache some information about ourselves
self.my = api.me()
self.log.info(f"Connected as: {self.my.screen_name}")
# Image URL's are md5 hashed and cached here to prevent duplicate API queries. This is cleared every 24-hours.
# I'll update this in the future to use a real caching mechanism (database or redis)
self._cached_results = {}
# A cached list of ID's for parent posts we've already processed
# Used in the check_monitored() method to prevent re-posting sauces when posts are re-tweeted
self._posts_processed = []
# The ID cutoff, we populate this once via an initial query at startup
try:
self.mention_id = tweepy.Cursor(api.mentions_timeline,
tweet_mode='extended',
count=1).items(1).next().id
except StopIteration:
self.mention_id = 0
try:
self.self_id = tweepy.Cursor(api.user_timeline,
tweet_mode='extended',
count=1).items(1).next().id
except StopIteration:
self.self_id = 0
self.monitored_since = {}
async def check_monitored(self) -> None:
"""
Checks monitored accounts for any new tweets
Returns:
None
"""
monitored_accounts = str(config.get('Twitter', 'monitored_accounts'))
if not monitored_accounts:
return
monitored_accounts = [a.strip() for a in monitored_accounts.split(',')]
for account in monitored_accounts:
# Have we fetched a tweet for this account yet?
if account not in self.monitored_since:
# If not, get the last tweet ID from this account and wait for the next post
tweet = next(
tweepy.Cursor(api.user_timeline,
account,
page=1,
tweet_mode='extended').items())
self.monitored_since[account] = tweet.id
self.log.info(
f"[{account}] Monitoring tweets after {tweet.id}")
continue
# Get all tweets since our last check
self.log.info(
f"[{account}] Retrieving tweets since {self.monitored_since[account]}"
)
tweets = [
*tweepy.Cursor(api.user_timeline,
account,
since_id=self.monitored_since[account],
tweet_mode='extended').items()
]
self.log.info(f"[{account}] {len(tweets)} tweets found")
for tweet in tweets:
try:
# Update the ID cutoff before attempting to parse the tweet
self.monitored_since[account] = max(
[self.monitored_since[account], tweet.id])
# Make sure this isn't a comment / reply
if tweet.in_reply_to_status_id:
self.log.info(
f"[{account}] Tweet is a reply/comment; ignoring")
continue
# Make sure we haven't already processed this post
if tweet.id in self._posts_processed:
self.log.info(
f"[{account}] Post has already been processed; ignoring"
)
continue
self._posts_processed.append(tweet.id)
# Make sure this isn't a re-tweet
if 'RT @' in tweet.full_text or hasattr(
tweet, 'retweeted_status'):
self.log.info(f"[{account}] Retweeted post; ignoring")
continue
original_cache, media_cache, media = self.get_closest_media(
tweet, account)
self.log.info(
f"[{account}] Found new media post in tweet {tweet.id}: {media[0]}"
)
# Get the sauce
sauce_cache = await self.get_sauce(
media_cache,
log_index=account,
trigger=TRIGGER_MONITORED)
sauce = sauce_cache.sauce
self.log.info(
f"[{account}] Found {sauce.index} sauce for tweet {tweet.id}"
if sauce else
f"[{account}] Failed to find sauce for tweet {tweet.id}"
)
await self.send_reply(tweet_cache=original_cache,
media_cache=media_cache,
sauce_cache=sauce_cache,
requested=False)
except TwSauceNoMediaException:
self.log.info(
f"[{account}] No sauce found for tweet {tweet.id}")
continue
except Exception as e:
self.log.exception(
f"[{account}] An unknown error occurred while processing tweet {tweet.id}: {e}"
)
continue
tweepy.api.API
"""
auth = tweepy.OAuthHandler(key, secret)
auth.set_access_token(token, token_secret)
_api = tweepy.API(auth,
wait_on_rate_limit=True,
wait_on_rate_limit_notify=True)
try:
_api.verify_credentials()
except Exception as e:
logger.critical("Error loading API", exc_info=True)
raise e
logger.info("Twitter API ready")
return _api
api = _twitter_api(config.get('Twitter', 'consumer_key'),
config.get('Twitter', 'consumer_secret'),
config.get('Twitter', 'access_token'),
config.get('Twitter', 'access_secret'))
if config.has_section('TwitterReadOnly'):
readonly_api = _twitter_api(
config.get('TwitterReadOnly', 'consumer_key'),
config.get('TwitterReadOnly', 'consumer_secret'),
config.get('TwitterReadOnly', 'access_token'),
config.get('TwitterReadOnly', 'access_secret'))
else:
readonly_api = None
# Set up logging
import logging
from twsaucenao.config import config
logLevel = getattr(logging, str(config.get('Logging', 'log_level', fallback='ERROR')).upper())
logFormat = logging.Formatter("[%(asctime)s] %(levelname)s: %(message)s", "%Y-%m-%d %H:%M:%S")
log = logging.getLogger('twsaucenao')
log.setLevel(logLevel)
ch = logging.StreamHandler()
ch.setLevel(logLevel)
ch.setFormatter(logFormat)
log.addHandler(ch)
import asyncio
from twsaucenao.config import config
from twsaucenao.log import log
from twsaucenao.models.database import TweetCache, TweetSauceCache
from twsaucenao.server import TwitterSauce
# Get our polling intervals
mentioned_interval = float(
config.get('Twitter', 'mentioned_interval', fallback=15.0))
monitored_interval = float(
config.get('Twitter', 'monitored_interval', fallback=60.0))
search_interval = float(config.get('Twitter', 'search_interval',
fallback=60.0))
twitter = TwitterSauce()
# noinspection PyBroadException
async def mentions() -> None:
"""
Respond to any mentions requesting sauce lookups
Returns:
None
"""
while True:
try:
# Mentions
await twitter.check_mentions()
await asyncio.sleep(mentioned_interval)
except Exception:
def __init__(self):
self.log = logging.getLogger(__name__)
# Tweet Cache Manager
self.twitter = TweetManager()
self.twython = Twython(config.get('Twitter', 'consumer_key'),
config.get('Twitter', 'consumer_secret'),
config.get('Twitter', 'access_token'),
config.get('Twitter', 'access_secret'))
# SauceNao
self.minsim_mentioned = float(
config.get('SauceNao', 'min_similarity_mentioned', fallback=50.0))
self.minsim_monitored = float(
config.get('SauceNao', 'min_similarity_monitored', fallback=65.0))
self.minsim_searching = float(
config.get('SauceNao', 'min_similarity_searching', fallback=70.0))
self.persistent = config.getboolean('Twitter',
'enable_persistence',
fallback=False)
self.anime_link = config.get('SauceNao',
'source_link',
fallback='anidb').lower()
self.sauce = SauceNao(api_key=config.get('SauceNao',
'api_key',
fallback=None),
min_similarity=min(self.minsim_mentioned,
self.minsim_monitored,
self.minsim_searching),
priority=[21, 22, 5])
# Trace.moe
self.tracemoe = None # type: Optional[ATraceMoe]
if config.getboolean('TraceMoe', 'enabled', fallback=False):
self.tracemoe = ATraceMoe(
config.get('TraceMoe', 'token', fallback=None))
self.nsfw_previews = config.getboolean('TraceMoe',
'nsfw_previews',
fallback=False)
# Pixiv
self.pixiv = Pixiv()
# Cache some information about ourselves
self.my = api.me()
self.log.info(f"Connected as: {self.my.screen_name}")
# Image URL's are md5 hashed and cached here to prevent duplicate API queries. This is cleared every 24-hours.
# I'll update this in the future to use a real caching mechanism (database or redis)
self._cached_results = {}
# A cached list of ID's for parent posts we've already processed
# Used in the check_monitored() method to prevent re-posting sauces when posts are re-tweeted
self._posts_processed = []
# The ID cutoff, we populate this once via an initial query at startup
try:
self.since_id = tweepy.Cursor(api.mentions_timeline,
tweet_mode='extended',
count=1).items(1).next().id
except StopIteration:
self.since_id = 0
self.monitored_since = {}
import pysaucenao
import tweepy
from pony.orm import *
from pysaucenao import GenericSource
from pysaucenao.containers import PixivSource, SauceNaoResults, VideoSource
from twsaucenao.api import api
from twsaucenao.config import config
from twsaucenao.log import log
db = Database()
if config.has_section('MySQL'):
db.bind(provider='mysql',
host=config.get('MySQL', 'hostname'),
user=config.get('MySQL', 'username'),
passwd=config.get('MySQL', 'password'),
db=config.get('MySQL', 'database'),
charset='utf8mb4')
else:
db.bind(provider='sqlite', filename='database.sqlite', create_db=True)
TRIGGER_MENTION = 'mentioned'
TRIGGER_MONITORED = 'monitored'
TRIGGER_SEARCH = 'searching'
# noinspection PyMethodParameters
class TweetCache(db.Entity):
tweet_id = PrimaryKey(int, size=64)
import logging
import os
import random
import typing
from configparser import ConfigParser
import tweepy
from twsaucenao.config import config
# Set up localization for use elsewhere in the application
_language = config.get('System', 'Language', fallback='english')
_language_config = ConfigParser()
_language_config.read(os.path.join('lang', f'{_language}.ini'), 'utf-8')
def lang(category: str,
key: str,
replacements: typing.Optional[dict] = None,
default=None,
user: typing.Optional[tweepy.models.User] = None):
"""
Provides easy to use application localization in the form of ini configuration files
Language strings can be added or altered in the data/lang folder
"""
string = _language_config.get(category, key, fallback=default) # type: str
if string:
if replacements:
for rkey, rvalue in replacements.items():
string = string.replace(f"{{{rkey}}}", str(rvalue))
# Set up logging
import logging
import sentry_sdk
from twsaucenao.config import config
logLevel = getattr(logging, str(config.get('System', 'log_level', fallback='ERROR')).upper())
logFormat = logging.Formatter("[%(asctime)s] %(levelname)s: %(message)s", "%Y-%m-%d %H:%M:%S")
log = logging.getLogger('twsaucenao')
log.setLevel(logLevel)
ch = logging.StreamHandler()
ch.setLevel(logLevel)
ch.setFormatter(logFormat)
log.addHandler(ch)
# Unless you're running your own custom fork of saucebot, you probably don't need this.
if config.has_option('System', 'sentry_logging') and config.getboolean('System', 'sentry_logging'):
sentry_sdk.init(config.get('System', 'sentry_dsn'), traces_sample_rate=0.25)
# Set up logging
import logging
from twsaucenao.config import config
logLevel = getattr(
logging,
str(config.get('System', 'log_level', fallback='ERROR')).upper())
logFormat = logging.Formatter("[%(asctime)s] %(levelname)s: %(message)s",
"%Y-%m-%d %H:%M:%S")
log = logging.getLogger('twsaucenao')
log.setLevel(logLevel)
ch = logging.StreamHandler()
ch.setLevel(logLevel)
ch.setFormatter(logFormat)
log.addHandler(ch)
del image
encoded = b64encode(data.getvalue()).decode("utf-8")
response = await self.session.post(url,
json={
"image": encoded,
"filter": search_filter
})
else:
response = await self.session.get(url, params={"url": path})
return loads(await response.text())
elif isinstance(path, io.BufferedIOBase):
encoded = b64encode(path.read()).decode("utf-8")
response = await self.session.post(url,
json={
"image": encoded,
"filter": search_filter
})
return loads(await response.text())
else:
with open(path, "rb") as f:
encoded = b64encode(f.read()).decode("utf-8")
response = await self.session.post(url,
json={
"image": encoded,
"filter": search_filter
})
return loads(await response.text())
tracemoe = ATraceMoe(config.get('TraceMoe', 'token', fallback=None))
