def observe():
try:
with rabbitpy.Connection('amqp://{username}:{password}@{host}:{port}/{vhost}'.format(
username=settings.get('RABBITMQ_USERNAME'),
password=settings.get('RABBITMQ_PASSWORD'),
host=settings.get('RABBITMQ_HOST'),
port=settings.get('RABBITMQ_PORT'),
vhost=settings.get('RABBITMQ_VHOST')
)) as conn:
with conn.channel() as channel:
channel.enable_publisher_confirms()
Ayumi.set_rabbitpy_channel(channel)
Ayumi.info("Now connected AMQP provider.", color=Ayumi.GREEN)
event_handler = IzumiHandler(channel)
observer = Observer()
observer.schedule(event_handler, settings.get('KOTEN_WATCH_PATH', DEFAULT_WATCH_PATH), recursive=True)
observer.start()
Ayumi.info("Now observing: {}".format(settings.get('KOTEN_WATCH_PATH', DEFAULT_WATCH_PATH)), color=Ayumi.BLUE)
try:
while True:
time.sleep(settings.get('KOTEN_SLEEP_INTERVAL', DEFAULT_SLEEP_INTERVAL))
except:
Ayumi.warning("Detected SIGKILL or error, returning...", color=Ayumi.YELLOW)
observer.stop()
observer.join()
except rabbitpy.exceptions.AMQPConnectionForced:
Ayumi.rabbitpy_channel = None
Ayumi.critical("Operator manually closed RabbitMQ connection, shutting down.", color=Ayumi.RED)
# Use return for now because in some cases, calling exit() may invoke the retry() header.
return
def consume():
try:
with rabbitpy.Connection(
'amqp://{username}:{password}@{host}:{port}/{vhost}'.format(
username=settings.get('RABBITMQ_USERNAME'),
password=settings.get('RABBITMQ_PASSWORD'),
host=settings.get('RABBITMQ_HOST'),
port=settings.get('RABBITMQ_PORT'),
vhost=settings.get('RABBITMQ_VHOST'))) as conn:
with conn.channel() as channel:
Ayumi.set_rabbitpy_channel(channel)
queue = rabbitpy.Queue(
channel,
settings.get('NOTIFICATIONS_DISCORD_WEBHOOK_QUEUE',
'nonexistent'))
queue.declare(passive=True)
Ayumi.info("Now listening for messages from AMQP provider.",
color=Ayumi.YELLOW)
for message in queue.consume(prefetch=1):
try:
job = json.loads(message.body.decode('utf-8'))
except json.JSONDecodeError:
Ayumi.warning(
"Received a job that is invalid json, not processing.",
color=Ayumi.LRED)
message.reject()
continue
Ayumi.info("Received a new job: {}".format(
json.dumps(job)),
color=Ayumi.CYAN)
if metsuke.validate(job):
Ayumi.debug("Loaded show: {}".format(job['show']))
Ayumi.debug("Loaded episode: {}".format(
job['episode']))
Ayumi.debug("Loaded filesize: {}".format(
job['filesize']))
Ayumi.debug("Loaded sub type: {}".format(job['sub']))
embed = _generate_embed(metsuke.generate(job),
hisha.search(job['show']))
Ayumi.info(
"Beginning sending embeds to webhook endpoints.",
color=Ayumi.CYAN)
for endpoint in settings.get(
'NOTIFICATIONS_DISCORD_WEBHOOK_ENDPOINTS'
).to_list():
try:
requests.post(endpoint, json=embed, timeout=5)
Ayumi.debug(
"Sent embed to {}".format(endpoint))
except:
Ayumi.warning(
"Failed to send embed to {}".format(
endpoint),
color=Ayumi.RED)
else:
Ayumi.warning(
"Received a job that Metsuke was not able to validate.",
color=Ayumi.LRED)
Ayumi.warning(json.dumps(job), color=Ayumi.LRED)
Ayumi.info(
"Completed processing this message for {}".format(
job['episode']),
color=Ayumi.LGREEN)
message.ack()
except rabbitpy.exceptions.AMQPConnectionForced:
Ayumi.rabbitpy_channel = None
Ayumi.critical(
"Operator manually closed RabbitMQ connection, shutting down.",
color=Ayumi.RED)
# Use return for now because in some cases, calling exit() may invoke the retry() header.
return
def rss(last_guid=None):
try:
with rabbitpy.Connection(
'amqp://{username}:{password}@{host}:{port}/{vhost}'.format(
username=settings.get_fresh('RABBITMQ_USERNAME'),
password=settings.get_fresh('RABBITMQ_PASSWORD'),
host=settings.get_fresh('RABBITMQ_HOST'),
port=settings.get_fresh('RABBITMQ_PORT'),
vhost=settings.get_fresh('RABBITMQ_VHOST'))) as conn:
with conn.channel() as channel:
Ayumi.set_rabbitpy_channel(channel)
channel.enable_publisher_confirms()
while True:
Ayumi.info("Now starting feed fetch.", color=Ayumi.LCYAN)
feed = feedparser.parse(
settings.get('ACQUISITION_RSS_FEED_URL', None))
accepted_shows = _load_accepted_shows()
Ayumi.debug(
"Loaded accepted shows map: {}".format(accepted_shows))
history = _load_history()
new_history = list()
for entry in feed.entries:
# Fetch data first
title, link, guid = entry.title, entry.link, entry.guid
Ayumi.debug(
'Encountered RSS item with title "{}", and guid "{}"'
.format(title, guid))
# If feed item with last GUID encountered, do not process any further
if guid == last_guid:
Ayumi.debug(
"Encountered RSS item with last_guid {} matching argument, breaking and writing history."
.format(last_guid),
color=Ayumi.YELLOW)
break
# Check the title data
# Use the parsed title to match user provided titles.
parsed_title = anitopy.parse(title)['anime_title']
if _strip_title(parsed_title) not in accepted_shows:
Ayumi.info(
'Feed item with title "{}" (show title: "{}") is not in accepted shows, skipping.'
.format(title, parsed_title))
else:
if guid in history:
# This item has been previously processed, skip it.
Ayumi.info(
'Feed item with title "{}" (show title: "{}") has already been processed, skipping.'
.format(title, parsed_title),
color=Ayumi.GREEN)
else:
# A new feeditem! Let us process it.
Ayumi.info(
'Feed item with title "{}" (show title: "{}") is in accepted shows, processing.'
.format(title, parsed_title),
color=Ayumi.YELLOW)
message = rabbitpy.Message(
channel,
json.dumps({
"title":
title,
"link":
link,
"guid":
guid,
"show_title":
accepted_shows[_strip_title(
parsed_title)]
}))
acquisition_rss_exchange_name = settings.get(
'ACQUISITION_RSS_EXCHANGE')
while not message.publish(
acquisition_rss_exchange_name,
mandatory=True):
Ayumi.warning(
'Failed to publish feed item with title "{}" to exchange "{}", retrying in 60s...'
.format(title,
acquisition_rss_exchange_name),
color=Ayumi.RED)
sleep(60)
Ayumi.info(
'Published feed item with title "{}" to exchange "{}".'
.format(
title,
acquisition_rss_exchange_name,
),
color=Ayumi.LGREEN)
# Keep all items processed in the new history - it will be auto deleted by the expiry of the RSS
Ayumi.debug(
'Appending item "{}" with title "{}" (show title: "{}") to new_history for write.'
.format(guid, title, parsed_title),
color=Ayumi.YELLOW)
new_history.append(guid)
_write_history(new_history)
# Sleep till the next iteration
sleep_duration = settings.get(
'ACQUISITION_RSS_SLEEP_INTERVAL',
_DEFAULT_SLEEP_INTERVAL)
Ayumi.info(
"Now sleeping {} seconds.".format(sleep_duration),
color=Ayumi.LCYAN)
sleep(sleep_duration)
except rabbitpy.exceptions.AMQPConnectionForced:
Ayumi.rabbitpy_channel = None
Ayumi.critical(
"Operator manually closed RabbitMQ connection, shutting down.",
color=Ayumi.RED)
# Use return for now because in some cases, calling exit() may invoke the retry() header.
return
def bittorrent():
try:
with rabbitpy.Connection('amqp://{username}:{password}@{host}:{port}/{vhost}'.format(
username=settings.get('RABBITMQ_USERNAME'),
password=settings.get('RABBITMQ_PASSWORD'),
host=settings.get('RABBITMQ_HOST'),
port=settings.get('RABBITMQ_PORT'),
vhost=settings.get('RABBITMQ_VHOST')
)) as conn:
with conn.channel() as channel:
Ayumi.set_rabbitpy_channel(channel)
channel.enable_publisher_confirms()
queue_name = settings.get('ACQUISITION_BITTORRENT_QUEUE')
Ayumi.debug("Connecting to queue: {}".format(queue_name))
queue = rabbitpy.Queue(channel, queue_name)
queue.declare(passive=True)
Ayumi.info('Now listening for messages on queue: {}...'.format(
queue_name), color=Ayumi.LYELLOW)
for message in queue.consume(prefetch=1):
Ayumi.info(
"Received new message, starting...", color=Ayumi.CYAN)
feeditem_preprocess = _load_amqp_message_body(message)
Ayumi.debug('Loaded message raw: "{}"'.format(
feeditem_preprocess))
if not feeditem_preprocess or not metsuke.validate_feeditem(feeditem_preprocess):
Ayumi.error('Invalid message received, rejecting. Output: "{}"'.format(
feeditem_preprocess), color=Ayumi.RED)
message.reject()
continue
# Load initial data
feeditem: metsuke.FeedItem = metsuke.generate_feeditem(
feeditem_preprocess)
shows_map = _load_shows_map()
overload_title = feeditem.show_title
Ayumi.info(
'Setting overload title: "{}"'.format(overload_title))
# If there is a central override, use it instead.
if _strip_title(anitopy.parse(feeditem.title)['anime_title']) in shows_map:
central_overload_title = shows_map[_strip_title(
feeditem.title)]
Ayumi.info('Overwriting overload title with central overload title: "{}"'.format(
central_overload_title))
overload_title = central_overload_title
with tempfile.TemporaryDirectory() as temp_dir:
Ayumi.debug(
'Created temporary directory under path: "{}"'.format(temp_dir))
# Download the episode
try:
res = subprocess.run(
[
"aria2c",
"--seed-time=0",
"--rpc-save-upload-metadata=false",
"--bt-save-metadata=false",
"--dir={}".format(temp_dir),
feeditem.link
]
)
if res.returncode != 0:
Ayumi.warning(
"Aria2 did not return a 0 exit code, assuming download errored and nacking.", color=Ayumi.RED)
message.nack()
continue
except subprocess.TimeoutExpired:
Ayumi.warning(
"Download via webtorrent timed out - nacking.", color=Ayumi.RED)
message.nack()
continue
if res.returncode != 0:
Ayumi.warning(
"Webtorrent did not have a return code of 0, nacking.", color=Ayumi.RED)
message.nack()
continue
# Rename it
potential_files = [f for f in os.listdir(
temp_dir) if f.endswith(".mkv")]
Ayumi.debug(
"Loaded potential files: {}".format(potential_files))
if len(potential_files) != 1:
Ayumi.warning(
"Found more than one .mkv file, rejecting this job.", color=Ayumi.RED)
message.reject()
continue
dl_file = potential_files[0]
Ayumi.info('Found file: "{}"'.format(dl_file))
dl_file_path = os.path.abspath(
'{}/{}'.format(_clean_title(temp_dir), potential_files[0]))
Ayumi.debug(
'dl_file_path: "{}"'.format(dl_file_path))
# Remove unneeded files
# TODO: THIS IS A HOTFIX, CHANGE LOGIC IN B2
bad_files = [f for f in os.listdir(
temp_dir) if not f.endswith(".mkv")]
Ayumi.debug("Found bad files: {}".format(bad_files))
for bf in bad_files:
try:
Ayumi.debug("Removing bad file: {}".format(bf))
os.remove(
'{}/{}'.format(_clean_title(temp_dir), bf))
except:
Ayumi.debug("Removing bad tree: {}".format(bf))
shutil.rmtree(
'{}/{}'.format(_clean_title(temp_dir), bf))
# Move the file to proper layout with updated name
dl_file_new_name = _generate_new_filename(dl_file)
Ayumi.info('Generated new episode name: "{}"'.format(
dl_file_new_name))
dl_file_new_dir = "{}/{}".format(
temp_dir, overload_title)
Ayumi.debug(
'dl_file_new_dir: "{}"'.format(dl_file_new_dir))
dl_file_new_path = "{}/{}".format(
dl_file_new_dir, dl_file_new_name)
Ayumi.debug(
'dl_file_new_path: "{}"'.format(
dl_file_new_path))
Ayumi.debug('Moving "{}" to "{}"'.format(
dl_file_path, dl_file_new_path))
os.mkdir(dl_file_new_dir)
shutil.move(dl_file_path, dl_file_new_path)
# Upload the file to rclone destination
with tempfile.NamedTemporaryFile(suffix=".conf", mode="w+b") as rconf:
rconf.write(str.encode(
settings.get("RCLONE_CONFIG_FILE")))
rconf.flush()
Ayumi.debug(
'Created temporary rclone file under path: "{}"'.format(rconf.name))
rclone_dest = _clean_title(settings.get(
"ACQUISITION_BITTORRENT_RCLONE_DEST"))
rclone_flags = settings.get("RCLONE_FLAGS", "")
command = [
"rclone", "--config={}".format(rconf.name), "copy", temp_dir, rclone_dest]
command.extend(rclone_flags.split())
Ayumi.debug(
'Rclone command to be run: "{}"'.format(command))
try:
Ayumi.info(
'Now uploading new blob to: "{}"'.format(rclone_dest))
rclone_res = subprocess.run(
command, timeout=3600)
if rclone_res.returncode != 0:
Ayumi.warning('Rclone returned non-zero code of {}, nacking.'.format(
rclone_res.returncode), color=Ayumi.LRED)
message.nack()
except subprocess.TimeoutExpired:
Ayumi.warning(
'Rclone upload timed out, nacking.', color=Ayumi.LRED)
message.nack()
continue
# Fetch information on the file to create a job
new_message = rabbitpy.Message(channel, dumps(
{
"show": overload_title,
"episode": dl_file_new_name,
"filesize": int(os.path.getsize(dl_file_new_path)),
"sub": "SOFTSUB"
}
))
acquisition_bittorrent_exchange_name = settings.get(
'ACQUISITION_BITTORRENT_EXCHANGE')
Ayumi.info('Sending to exchange: "{}"'.format(
acquisition_bittorrent_exchange_name), color=Ayumi.CYAN)
while not new_message.publish(acquisition_bittorrent_exchange_name, mandatory=True):
Ayumi.warning(
"Failed to publish feed item, trying again in 60 seconds")
sleep(60)
Ayumi.info("Published feed item with title: " +
overload_title, color=Ayumi.LGREEN)
message.ack()
except rabbitpy.exceptions.AMQPConnectionForced:
Ayumi.warning(
"Operator manually closed RabbitMQ connection, shutting down.", color=Ayumi.LYELLOW)
return
def consume():
try:
with rabbitpy.Connection('amqp://{username}:{password}@{host}:{port}/{vhost}'.format(
username=settings.get('RABBITMQ_USERNAME'),
password=settings.get('RABBITMQ_PASSWORD'),
host=settings.get('RABBITMQ_HOST'),
port=settings.get('RABBITMQ_PORT'),
vhost=settings.get('RABBITMQ_VHOST')
)) as conn:
with conn.channel() as channel:
Ayumi.set_rabbitpy_channel(channel)
queue = rabbitpy.Queue(channel, settings.get('DISTRIBUTORS_RCLONE_QUEUE'))
queue.declare(passive=True)
Ayumi.info("Now listening for messages from AMQP provider.", color=Ayumi.YELLOW)
for message in queue.consume(prefetch=1):
try:
job = json.loads(message.body.decode('utf-8'))
except json.JSONDecodeError:
Ayumi.warning("Received a job that is invalid json, not processing.", color=Ayumi.LRED)
message.reject()
continue
Ayumi.info("Received a new job: {}".format(json.dumps(job)), color=Ayumi.CYAN)
if metsuke.validate(job):
Ayumi.debug("Loaded show: {}".format(job['show']))
Ayumi.debug("Loaded episode: {}".format(job['episode']))
Ayumi.debug("Loaded filesize: {}".format(job['filesize']))
Ayumi.debug("Loaded sub type: {}".format(job['sub']))
metsuke_job = metsuke.Job(
job['show'], job['episode'], job['filesize'], job['sub'])
with tempfile.NamedTemporaryFile(suffix=".conf", mode="w+b") as rconf, tempfile.TemporaryDirectory() as tempdir:
Ayumi.debug("Opening context managed rclone config file under path: {}.".format(rconf.name))
Ayumi.debug("Opening context managed rclone temporary directory under path: {}".format(tempdir))
rconf.write(str.encode(settings.get("RCLONE_CONFIG_FILE")))
rconf.flush() # YOU MUST FLUSH THE FILE SO RCLONE CAN READ IT!
Ayumi.debug("Configurations written to temporary file. Size is {} bytes.".format(rconf.tell()))
dl_sources = None
up_dests = None
if job['sub'].lower() == "softsub":
dl_sources = settings.get("DISTRIBUTORS_RCLONE_SOFTSUB_DOWNLOAD")
up_dests = settings.get("DISTRIBUTORS_RCLONE_SOFTSUB_UPLOAD")
elif job['sub'].lower() == "hardsub":
dl_sources = settings.get("DISTRIBUTORS_RCLONE_HARDSUB_DOWNLOAD")
up_dests = settings.get("DISTRIBUTORS_RCLONE_HARDSUB_UPLOAD")
Ayumi.debug("Fresh fetched download sources as: {}".format(" ".join(dl_sources)))
Ayumi.debug("Fresh fetched upload sources as: {}".format(" ".join(up_dests)))
try:
temp_ep = shikyou.download(metsuke_job, dl_sources, tempdir, rconf.name, settings.get("RCLONE_FLAGS", ""))
if temp_ep:
shikyou.upload(metsuke_job, up_dests, temp_ep, rconf.name, settings.get("RCLONE_FLAGS", ""))
else:
Ayumi.warning("Unable to find requested job in any sources, nacking...", color=Ayumi.RED)
message.nack()
continue
except shikyou.ShikyouResponseException:
Ayumi.critical("Rclone threw an unexpected response code, rejecting.", color=Ayumi.RED)
message.reject()
continue
except shikyou.ShikyouTimeoutException:
Ayumi.warning("Rclone timed out whilhe executing, nacking.", color=Ayumi.RED)
message.nack()
continue
Ayumi.debug("Closed context managed rclone config file.")
Ayumi.debug("Closed context managed temporary directory.")
else:
Ayumi.warning("Received a job that Metsuke was not able to validate.", color=Ayumi.LRED)
Ayumi.warning(json.dumps(job), color=Ayumi.LRED)
Ayumi.info("Completed processing this message for {}".format(job['episode']), color=Ayumi.LGREEN)
message.ack()
except rabbitpy.exceptions.AMQPConnectionForced:
Ayumi.rabbitpy_channel = None
Ayumi.critical("Operator manually closed RabbitMQ connection, shutting down.", color=Ayumi.RED)
# Use return for now because in some cases, calling exit() may invoke the retry() header.
return
