def observe():
try:
with rabbitpy.Connection('amqp://{username}:{password}@{host}:{port}/{vhost}'.format(
username=settings.get('RABBITMQ_USERNAME'),
password=settings.get('RABBITMQ_PASSWORD'),
host=settings.get('RABBITMQ_HOST'),
port=settings.get('RABBITMQ_PORT'),
vhost=settings.get('RABBITMQ_VHOST')
)) as conn:
with conn.channel() as channel:
channel.enable_publisher_confirms()
Ayumi.set_rabbitpy_channel(channel)
Ayumi.info("Now connected AMQP provider.", color=Ayumi.GREEN)
event_handler = IzumiHandler(channel)
observer = Observer()
observer.schedule(event_handler, settings.get('KOTEN_WATCH_PATH', DEFAULT_WATCH_PATH), recursive=True)
observer.start()
Ayumi.info("Now observing: {}".format(settings.get('KOTEN_WATCH_PATH', DEFAULT_WATCH_PATH)), color=Ayumi.BLUE)
try:
while True:
time.sleep(settings.get('KOTEN_SLEEP_INTERVAL', DEFAULT_SLEEP_INTERVAL))
except:
Ayumi.warning("Detected SIGKILL or error, returning...", color=Ayumi.YELLOW)
observer.stop()
observer.join()
except rabbitpy.exceptions.AMQPConnectionForced:
Ayumi.rabbitpy_channel = None
Ayumi.critical("Operator manually closed RabbitMQ connection, shutting down.", color=Ayumi.RED)
# Use return for now because in some cases, calling exit() may invoke the retry() header.
return
def _kishi_list(user):
"""
Helper method to get all of a user's anime list.
Params:
user: String, username of Anilist user
Returns: A tuple of three lists.
The first list is all the Watching
The second list is all the PTW
The third list is all the Paused
Throws an exception if anything goes wrong. This should be caught by any method using this.
"""
watching = list()
paused = list()
ptw = list()
# Anilist API is much nicer to play with.
try:
# Make the request to Anilist, and pass in the userName as the user query
anilist_res = requests.post(_ANILIST_API_URL,
json={
'query': _USER_ANIME_QUERY,
'variables': {
'userName': user
}
})
if anilist_res.status_code != 200:
Ayumi.error(
"Anilist returned a bad status code when attempting to get {}'s lists"
.format(user))
raise Exception()
try:
anilist_res_json = anilist_res.json()
except:
Ayumi.error(
"Anilist returned a response that was not parseable into JSON")
raise Exception()
watching = _add_list_entries("Watching", anilist_res_json)
paused = _add_list_entries("Paused", anilist_res_json)
ptw = _add_list_entries("Planning", anilist_res_json)
except:
Ayumi.critical("Kishi was unable to properly contact Anilist")
raise Exception()
return (watching, paused, ptw)
def is_user_watching_id(user, show_id):
"""
Determines whether or not an Anilist user is watching a show
Checks by show ID
Params:
user: username to look up
id: id of the show to look up
Returns: a boolean - True if watching, False if not
"""
try:
show_id = int(show_id) # Get the int equivalent value of the ID
except:
# Why would you not pass an integer in?
Ayumi.critical(
"Kishi ID search requires an input that can be converted to an int. Returning FALSE"
)
return False
try:
watching, paused, ptw = _kishi_list(user)
for show in watching:
if show_id == show['id']:
Ayumi.debug("Found show ID {} in {}".format(
show_id, "watching"))
return True
for show in paused:
if show_id == show['id']:
Ayumi.debug("Found show ID {} in {}".format(show_id, "paused"))
return True
for show in ptw:
if show_id == show['id']:
Ayumi.debug("Found show ID {} in {}".format(
show_id, "planning"))
return True
Ayumi.debug("Didn't find a match for {}".format(show_id))
return False
except:
# If any errors are encountered, return True (default assumption)
Ayumi.warning(
"An error was encountered while contacting Anilist. Defaulting to TRUE"
)
return True
def consume():
try:
with rabbitpy.Connection(
'amqp://{username}:{password}@{host}:{port}/{vhost}'.format(
username=settings.get('RABBITMQ_USERNAME'),
password=settings.get('RABBITMQ_PASSWORD'),
host=settings.get('RABBITMQ_HOST'),
port=settings.get('RABBITMQ_PORT'),
vhost=settings.get('RABBITMQ_VHOST'))) as conn:
with conn.channel() as channel:
Ayumi.set_rabbitpy_channel(channel)
queue = rabbitpy.Queue(
channel,
settings.get('NOTIFICATIONS_DISCORD_WEBHOOK_QUEUE',
'nonexistent'))
queue.declare(passive=True)
Ayumi.info("Now listening for messages from AMQP provider.",
color=Ayumi.YELLOW)
for message in queue.consume(prefetch=1):
try:
job = json.loads(message.body.decode('utf-8'))
except json.JSONDecodeError:
Ayumi.warning(
"Received a job that is invalid json, not processing.",
color=Ayumi.LRED)
message.reject()
continue
Ayumi.info("Received a new job: {}".format(
json.dumps(job)),
color=Ayumi.CYAN)
if metsuke.validate(job):
Ayumi.debug("Loaded show: {}".format(job['show']))
Ayumi.debug("Loaded episode: {}".format(
job['episode']))
Ayumi.debug("Loaded filesize: {}".format(
job['filesize']))
Ayumi.debug("Loaded sub type: {}".format(job['sub']))
embed = _generate_embed(metsuke.generate(job),
hisha.search(job['show']))
Ayumi.info(
"Beginning sending embeds to webhook endpoints.",
color=Ayumi.CYAN)
for endpoint in settings.get(
'NOTIFICATIONS_DISCORD_WEBHOOK_ENDPOINTS'
).to_list():
try:
requests.post(endpoint, json=embed, timeout=5)
Ayumi.debug(
"Sent embed to {}".format(endpoint))
except:
Ayumi.warning(
"Failed to send embed to {}".format(
endpoint),
color=Ayumi.RED)
else:
Ayumi.warning(
"Received a job that Metsuke was not able to validate.",
color=Ayumi.LRED)
Ayumi.warning(json.dumps(job), color=Ayumi.LRED)
Ayumi.info(
"Completed processing this message for {}".format(
job['episode']),
color=Ayumi.LGREEN)
message.ack()
except rabbitpy.exceptions.AMQPConnectionForced:
Ayumi.rabbitpy_channel = None
Ayumi.critical(
"Operator manually closed RabbitMQ connection, shutting down.",
color=Ayumi.RED)
# Use return for now because in some cases, calling exit() may invoke the retry() header.
return
def rss(last_guid=None):
try:
with rabbitpy.Connection(
'amqp://{username}:{password}@{host}:{port}/{vhost}'.format(
username=settings.get_fresh('RABBITMQ_USERNAME'),
password=settings.get_fresh('RABBITMQ_PASSWORD'),
host=settings.get_fresh('RABBITMQ_HOST'),
port=settings.get_fresh('RABBITMQ_PORT'),
vhost=settings.get_fresh('RABBITMQ_VHOST'))) as conn:
with conn.channel() as channel:
Ayumi.set_rabbitpy_channel(channel)
channel.enable_publisher_confirms()
while True:
Ayumi.info("Now starting feed fetch.", color=Ayumi.LCYAN)
feed = feedparser.parse(
settings.get('ACQUISITION_RSS_FEED_URL', None))
accepted_shows = _load_accepted_shows()
Ayumi.debug(
"Loaded accepted shows map: {}".format(accepted_shows))
history = _load_history()
new_history = list()
for entry in feed.entries:
# Fetch data first
title, link, guid = entry.title, entry.link, entry.guid
Ayumi.debug(
'Encountered RSS item with title "{}", and guid "{}"'
.format(title, guid))
# If feed item with last GUID encountered, do not process any further
if guid == last_guid:
Ayumi.debug(
"Encountered RSS item with last_guid {} matching argument, breaking and writing history."
.format(last_guid),
color=Ayumi.YELLOW)
break
# Check the title data
# Use the parsed title to match user provided titles.
parsed_title = anitopy.parse(title)['anime_title']
if _strip_title(parsed_title) not in accepted_shows:
Ayumi.info(
'Feed item with title "{}" (show title: "{}") is not in accepted shows, skipping.'
.format(title, parsed_title))
else:
if guid in history:
# This item has been previously processed, skip it.
Ayumi.info(
'Feed item with title "{}" (show title: "{}") has already been processed, skipping.'
.format(title, parsed_title),
color=Ayumi.GREEN)
else:
# A new feeditem! Let us process it.
Ayumi.info(
'Feed item with title "{}" (show title: "{}") is in accepted shows, processing.'
.format(title, parsed_title),
color=Ayumi.YELLOW)
message = rabbitpy.Message(
channel,
json.dumps({
"title":
title,
"link":
link,
"guid":
guid,
"show_title":
accepted_shows[_strip_title(
parsed_title)]
}))
acquisition_rss_exchange_name = settings.get(
'ACQUISITION_RSS_EXCHANGE')
while not message.publish(
acquisition_rss_exchange_name,
mandatory=True):
Ayumi.warning(
'Failed to publish feed item with title "{}" to exchange "{}", retrying in 60s...'
.format(title,
acquisition_rss_exchange_name),
color=Ayumi.RED)
sleep(60)
Ayumi.info(
'Published feed item with title "{}" to exchange "{}".'
.format(
title,
acquisition_rss_exchange_name,
),
color=Ayumi.LGREEN)
# Keep all items processed in the new history - it will be auto deleted by the expiry of the RSS
Ayumi.debug(
'Appending item "{}" with title "{}" (show title: "{}") to new_history for write.'
.format(guid, title, parsed_title),
color=Ayumi.YELLOW)
new_history.append(guid)
_write_history(new_history)
# Sleep till the next iteration
sleep_duration = settings.get(
'ACQUISITION_RSS_SLEEP_INTERVAL',
_DEFAULT_SLEEP_INTERVAL)
Ayumi.info(
"Now sleeping {} seconds.".format(sleep_duration),
color=Ayumi.LCYAN)
sleep(sleep_duration)
except rabbitpy.exceptions.AMQPConnectionForced:
Ayumi.rabbitpy_channel = None
Ayumi.critical(
"Operator manually closed RabbitMQ connection, shutting down.",
color=Ayumi.RED)
# Use return for now because in some cases, calling exit() may invoke the retry() header.
return
def sig_handler(sig, frame):
Ayumi.critical("SIG command {} detected, exiting...".format(sig),
color=Ayumi.LRED)
sys.exit()
def consume():
try:
with rabbitpy.Connection('amqp://{username}:{password}@{host}:{port}/{vhost}'.format(
username=settings.get('RABBITMQ_USERNAME'),
password=settings.get('RABBITMQ_PASSWORD'),
host=settings.get('RABBITMQ_HOST'),
port=settings.get('RABBITMQ_PORT'),
vhost=settings.get('RABBITMQ_VHOST')
)) as conn:
with conn.channel() as channel:
Ayumi.set_rabbitpy_channel(channel)
queue = rabbitpy.Queue(channel, settings.get('DISTRIBUTORS_RCLONE_QUEUE'))
queue.declare(passive=True)
Ayumi.info("Now listening for messages from AMQP provider.", color=Ayumi.YELLOW)
for message in queue.consume(prefetch=1):
try:
job = json.loads(message.body.decode('utf-8'))
except json.JSONDecodeError:
Ayumi.warning("Received a job that is invalid json, not processing.", color=Ayumi.LRED)
message.reject()
continue
Ayumi.info("Received a new job: {}".format(json.dumps(job)), color=Ayumi.CYAN)
if metsuke.validate(job):
Ayumi.debug("Loaded show: {}".format(job['show']))
Ayumi.debug("Loaded episode: {}".format(job['episode']))
Ayumi.debug("Loaded filesize: {}".format(job['filesize']))
Ayumi.debug("Loaded sub type: {}".format(job['sub']))
metsuke_job = metsuke.Job(
job['show'], job['episode'], job['filesize'], job['sub'])
with tempfile.NamedTemporaryFile(suffix=".conf", mode="w+b") as rconf, tempfile.TemporaryDirectory() as tempdir:
Ayumi.debug("Opening context managed rclone config file under path: {}.".format(rconf.name))
Ayumi.debug("Opening context managed rclone temporary directory under path: {}".format(tempdir))
rconf.write(str.encode(settings.get("RCLONE_CONFIG_FILE")))
rconf.flush() # YOU MUST FLUSH THE FILE SO RCLONE CAN READ IT!
Ayumi.debug("Configurations written to temporary file. Size is {} bytes.".format(rconf.tell()))
dl_sources = None
up_dests = None
if job['sub'].lower() == "softsub":
dl_sources = settings.get("DISTRIBUTORS_RCLONE_SOFTSUB_DOWNLOAD")
up_dests = settings.get("DISTRIBUTORS_RCLONE_SOFTSUB_UPLOAD")
elif job['sub'].lower() == "hardsub":
dl_sources = settings.get("DISTRIBUTORS_RCLONE_HARDSUB_DOWNLOAD")
up_dests = settings.get("DISTRIBUTORS_RCLONE_HARDSUB_UPLOAD")
Ayumi.debug("Fresh fetched download sources as: {}".format(" ".join(dl_sources)))
Ayumi.debug("Fresh fetched upload sources as: {}".format(" ".join(up_dests)))
try:
temp_ep = shikyou.download(metsuke_job, dl_sources, tempdir, rconf.name, settings.get("RCLONE_FLAGS", ""))
if temp_ep:
shikyou.upload(metsuke_job, up_dests, temp_ep, rconf.name, settings.get("RCLONE_FLAGS", ""))
else:
Ayumi.warning("Unable to find requested job in any sources, nacking...", color=Ayumi.RED)
message.nack()
continue
except shikyou.ShikyouResponseException:
Ayumi.critical("Rclone threw an unexpected response code, rejecting.", color=Ayumi.RED)
message.reject()
continue
except shikyou.ShikyouTimeoutException:
Ayumi.warning("Rclone timed out whilhe executing, nacking.", color=Ayumi.RED)
message.nack()
continue
Ayumi.debug("Closed context managed rclone config file.")
Ayumi.debug("Closed context managed temporary directory.")
else:
Ayumi.warning("Received a job that Metsuke was not able to validate.", color=Ayumi.LRED)
Ayumi.warning(json.dumps(job), color=Ayumi.LRED)
Ayumi.info("Completed processing this message for {}".format(job['episode']), color=Ayumi.LGREEN)
message.ack()
except rabbitpy.exceptions.AMQPConnectionForced:
Ayumi.rabbitpy_channel = None
Ayumi.critical("Operator manually closed RabbitMQ connection, shutting down.", color=Ayumi.RED)
# Use return for now because in some cases, calling exit() may invoke the retry() header.
return
