def find_closest_title(title):
"""
Finds the closest title from Anilist for Airing and Not_Yet_Released shows
"""
now = datetime.datetime.now()
date_next_month = int((now + datetime.timedelta(weeks=4)).strftime("%Y%m%d"))
date_last_month = int((now - datetime.timedelta(weeks=4)).strftime("%Y%m%d"))
shows = list()
heap = list()
shows.extend(_get_shows("RELEASING"))
shows.extend(_get_shows("NOT_YET_RELEASED", start_date=date_next_month))
shows.extend(_get_shows("FINISHED", end_date=date_last_month))
for show in shows:
ratio = _similarity(title, show)
Ayumi.debug('Matched "{}" against "{}" for a ratio of {}'.format(title, show, ratio))
heapq.heappush(heap, (ratio, show))
top_5 = heapq.nlargest(5, heap)
Ayumi.info("Displaying (up to) top 5 matches of {} results:".format(len(heap)), color=Ayumi.LBLUE)
for top in top_5:
Ayumi.info("{}: {}".format(top[1], top[0]), color=Ayumi.LBLUE)
if top_5:
Ayumi.info('Returning top match: {}'.format(top_5[0][1]), color=Ayumi.LGREEN)
return top_5[0][1]
else:
Ayumi.warning("No shows were fetched by Naomi, returning None", color=Ayumi.LYELLOW)
return None
def _check_exists(config: str, source: str, job: Job) -> bool:
"""
Checks if the jobs exists under the source.
Note: Does not validate under which path, just validates that jobs exists somewhere in:
source/(...probably job.show)/job.episode
"""
try:
response = _run([
"rclone", "--config={}".format(config), "lsjson", "-R",
"{}/{}/".format(source, job.show)
])
episode_list = loads(response.stdout.decode('utf-8'))
for episode in episode_list:
Ayumi.debug("Checking {} against episode {}".format(
job.episode, episode['Name']))
if episode['Name'] == job.episode:
Ayumi.info("Found episode {} in {}".format(
job.episode, source))
return True
Ayumi.info("Didn't find episode {} in {}".format(job.episode, source))
return False
except:
# Typically hit if the source doesn't exist.
Ayumi.warning(
"Error occured while checking source {} - does it exist?".format(
source))
return False
def _get_main_studio_info(studios):
"""
Goes through the studio edges and returns the main (studio name, siteurl)
Params:
studios - The studios body from the Anilist GraphQL json response
Returns: A tuple (studio name: str, site url: str), or None if not found
"""
try:
edges = studios['edges']
for edge in edges:
Ayumi.debug("Checking edge {}".format(edge['node']['name']))
if edge['isMain']:
Ayumi.debug("Found main studio edge, returning tuple")
node = edge['node']
return (node['name'], node['siteUrl'])
# If a main studio isn't found, return None
Ayumi.debug("Didn't find any main studio edge, returning None")
return (None, None)
except Exception as e:
Ayumi.warning(
"Didn't find any main studio edge due to error, returning None")
Ayumi.warning(e)
return (None, None)
def observe():
try:
with rabbitpy.Connection('amqp://{username}:{password}@{host}:{port}/{vhost}'.format(
username=settings.get('RABBITMQ_USERNAME'),
password=settings.get('RABBITMQ_PASSWORD'),
host=settings.get('RABBITMQ_HOST'),
port=settings.get('RABBITMQ_PORT'),
vhost=settings.get('RABBITMQ_VHOST')
)) as conn:
with conn.channel() as channel:
channel.enable_publisher_confirms()
Ayumi.set_rabbitpy_channel(channel)
Ayumi.info("Now connected AMQP provider.", color=Ayumi.GREEN)
event_handler = IzumiHandler(channel)
observer = Observer()
observer.schedule(event_handler, settings.get('KOTEN_WATCH_PATH', DEFAULT_WATCH_PATH), recursive=True)
observer.start()
Ayumi.info("Now observing: {}".format(settings.get('KOTEN_WATCH_PATH', DEFAULT_WATCH_PATH)), color=Ayumi.BLUE)
try:
while True:
time.sleep(settings.get('KOTEN_SLEEP_INTERVAL', DEFAULT_SLEEP_INTERVAL))
except:
Ayumi.warning("Detected SIGKILL or error, returning...", color=Ayumi.YELLOW)
observer.stop()
observer.join()
except rabbitpy.exceptions.AMQPConnectionForced:
Ayumi.rabbitpy_channel = None
Ayumi.critical("Operator manually closed RabbitMQ connection, shutting down.", color=Ayumi.RED)
# Use return for now because in some cases, calling exit() may invoke the retry() header.
return
def _add_list_entries(list_name, list_json):
"""
Helper method to add all list entries into a medialist (watching/paused/ptw)
Params:
list_name: the name that the list appears to be from Anilist ("Watching")
list_json: anilist's raw api response (json format) {'data':'MediaListCollection'}
Returns: A list with populated Anilist Media entries (a list of dicts)
"""
try:
entries = list()
media_lists = list_json['data']['MediaListCollection']['lists']
for media_list in media_lists:
if list_name.lower() == media_list['name'].lower():
for entry in media_list['entries']:
entries.append(entry['media'])
return entries
except:
Ayumi.warning("Kishi was unable to process list entries for {}".format(
list_name))
raise Exception()
def _load_amqp_message_body(message) -> Dict:
try:
feeditem = loads(message.body.decode('utf-8'))
return feeditem
except JSONDecodeError:
Ayumi.warning(
"Received an AMQP message that is invalid JSON, will not process...", color=Ayumi.RED)
return None
def download(job: Job, sources: List[str], tempfolder: str, config: str,
flags: str) -> str:
"""
Download the provided episode from sources
Returns the path of the downloaded file
job: Job to do!
sources: list of rclone sources (EncoderConf.downloading_sources)
tmppath: Path of the temporary folder
rclone_config: Path to the rclone config file
flags: rclone flags
"""
for source in sources:
Ayumi.debug("Checking for existence from source: {}".format(source))
if _check_exists(config, source, job):
Ayumi.info(
"Now downloading episode from source: {}".format(source))
src_file = "{}/{}/{}".format(_clean(source), job.show, job.episode)
Ayumi.debug("Sourcing from rclone path: {}".format(src_file))
dest_file = "{}/{}".format(_clean(tempfolder), "temp")
Ayumi.debug("Downloading to destination: {}".format(dest_file))
command = [
"rclone", "--config={}".format(config), "copyto", src_file,
dest_file
]
command.extend(flags.split())
Ayumi.debug("Now running command: {}".format(" ".join(command)))
Ayumi.info("Now starting download.", color=Ayumi.LCYAN)
try:
_run(command)
except ShikyouResponseException:
Ayumi.error(
"Rclone command returned a bad return code, contact the developer.",
color=Ayumi.LRED)
raise ShikyouResponseException()
except ShikyouTimeoutException:
Ayumi.error("Rclone command timed out, are your sources okay?",
color=Ayumi.LRED)
raise ShikyouTimeoutException()
Ayumi.info("Completed downloading files.", color=Ayumi.LGREEN)
return dest_file
else:
Ayumi.debug(
"Requested episode doesn't exist under source {}".format(
source))
Ayumi.warning("No download sources contained the file.")
return None
def is_user_watching_id(user, show_id):
"""
Determines whether or not an Anilist user is watching a show
Checks by show ID
Params:
user: username to look up
id: id of the show to look up
Returns: a boolean - True if watching, False if not
"""
try:
show_id = int(show_id) # Get the int equivalent value of the ID
except:
# Why would you not pass an integer in?
Ayumi.critical(
"Kishi ID search requires an input that can be converted to an int. Returning FALSE"
)
return False
try:
watching, paused, ptw = _kishi_list(user)
for show in watching:
if show_id == show['id']:
Ayumi.debug("Found show ID {} in {}".format(
show_id, "watching"))
return True
for show in paused:
if show_id == show['id']:
Ayumi.debug("Found show ID {} in {}".format(show_id, "paused"))
return True
for show in ptw:
if show_id == show['id']:
Ayumi.debug("Found show ID {} in {}".format(
show_id, "planning"))
return True
Ayumi.debug("Didn't find a match for {}".format(show_id))
return False
except:
# If any errors are encountered, return True (default assumption)
Ayumi.warning(
"An error was encountered while contacting Anilist. Defaulting to TRUE"
)
return True
def _run(command: Iterable[str]) -> str:
# Subprocess passes signals properly to the ffmpeg child, so we can just run it as is.
try:
response = subprocess.run(command, capture_output=True, timeout=3600)
except subprocess.TimeoutExpired:
Ayumi.warning("Command expired by timeout, killing...")
raise ShikyouResponseException()
if response.returncode != 0:
Ayumi.warning(
"Command caused return code of {}, returning None.".format(
response.returncode))
raise ShikyouResponseException()
return response
def is_user_watching_names(user, show_name):
"""
Determines whether or not an Anilist user is watching a show
Checks by show name
Params:
user: username to look up
show_name: name of the show to look up. this should already be the anilist name.
Returns: a boolean - True if watching, False if not
"""
try:
watching, paused, ptw = _kishi_list(user)
for show in watching:
for title in show['title'].values():
if _check_equality(title, show_name):
Ayumi.debug("Matched {} to {} in {}".format(
title, show_name, "watching"))
return True
for show in paused:
for title in show['title'].values():
if _check_equality(title, show_name):
Ayumi.debug("Matched {} to {} in {}".format(
title, show_name, "paused"))
return True
for show in ptw:
for title in show['title'].values():
if _check_equality(title, show_name):
Ayumi.debug("Matched {} to {} in {}".format(
title, show_name, "planning"))
return True
Ayumi.debug("Didn't find a match for {}".format(show_name))
return False
except:
# If any errors are encountered, return True (default assumption)
Ayumi.warning(
"An error was encountered while contacting Anilist. Defaulting to TRUE"
)
return True
def _query_request(query, search, status) -> dict:
"""
Makes requests to Anlist, returns response in JSON.
Query: One of the Queries objects.
Search: Name of show to search for.
Status: Status of the show to search for.
"""
try:
Ayumi.debug(
"Making request to {}, searching for {} under status {}".format(
ANILIST_API_URL, search, status))
ani_res = requests.post(ANILIST_API_URL,
json={
'query': query,
'variables': {
'search': search,
'status': status
}
},
timeout=10)
if ani_res.status_code != 200:
Ayumi.warning(
"Anilist returned unaccepted HTTP code {} upon request.".
format(ani_res.status_code),
color=Ayumi.LRED)
raise Exception()
# Get request response as JSON object.
try:
ani_json = ani_res.json()
return ani_json['data']
except:
Ayumi.warning("Anilist returned a non-JSON response.",
color=Ayumi.LRED)
raise Exception()
except requests.exceptions.Timeout:
Ayumi.warning("Request to Anilist timed out.", color=Ayumi.LRED)
raise Exception()
except requests.exceptions.ConnectionError:
Ayumi.warning("Unable to contact Anilist, maybe it's down?",
color=Ayumi.LRED)
raise Exception()
def _kitsu_basic_search(title):
"""Quick Kitsu implementation"""
title_lower = title.lower()
request_url = requests.utils.requote_uri(KITSU_API_URL + title_lower)
Ayumi.debug("Created Kitsu request URL: {}".format(request_url))
try:
kitsu_res = requests.get(request_url, timeout=10)
try:
kitsu_json = kitsu_res.json()
return kitsu_json
except:
Ayumi.warning("Kitsu did not return a valid JSON response.",
color=Ayumi.RED)
raise Exception()
except requests.exceptions.Timeout:
Ayumi.warning("Kitsu request timed out.", color=Ayumi.LRED)
raise Exception()
except requests.exceptions.ConnectionError:
Ayumi.warning("Unable to contact Kitsu, maybe it's down?",
color=Ayumi.LRED)
raise Exception()
def consume():
try:
with rabbitpy.Connection(
'amqp://{username}:{password}@{host}:{port}/{vhost}'.format(
username=settings.get('RABBITMQ_USERNAME'),
password=settings.get('RABBITMQ_PASSWORD'),
host=settings.get('RABBITMQ_HOST'),
port=settings.get('RABBITMQ_PORT'),
vhost=settings.get('RABBITMQ_VHOST'))) as conn:
with conn.channel() as channel:
Ayumi.set_rabbitpy_channel(channel)
queue = rabbitpy.Queue(
channel,
settings.get('NOTIFICATIONS_DISCORD_WEBHOOK_QUEUE',
'nonexistent'))
queue.declare(passive=True)
Ayumi.info("Now listening for messages from AMQP provider.",
color=Ayumi.YELLOW)
for message in queue.consume(prefetch=1):
try:
job = json.loads(message.body.decode('utf-8'))
except json.JSONDecodeError:
Ayumi.warning(
"Received a job that is invalid json, not processing.",
color=Ayumi.LRED)
message.reject()
continue
Ayumi.info("Received a new job: {}".format(
json.dumps(job)),
color=Ayumi.CYAN)
if metsuke.validate(job):
Ayumi.debug("Loaded show: {}".format(job['show']))
Ayumi.debug("Loaded episode: {}".format(
job['episode']))
Ayumi.debug("Loaded filesize: {}".format(
job['filesize']))
Ayumi.debug("Loaded sub type: {}".format(job['sub']))
embed = _generate_embed(metsuke.generate(job),
hisha.search(job['show']))
Ayumi.info(
"Beginning sending embeds to webhook endpoints.",
color=Ayumi.CYAN)
for endpoint in settings.get(
'NOTIFICATIONS_DISCORD_WEBHOOK_ENDPOINTS'
).to_list():
try:
requests.post(endpoint, json=embed, timeout=5)
Ayumi.debug(
"Sent embed to {}".format(endpoint))
except:
Ayumi.warning(
"Failed to send embed to {}".format(
endpoint),
color=Ayumi.RED)
else:
Ayumi.warning(
"Received a job that Metsuke was not able to validate.",
color=Ayumi.LRED)
Ayumi.warning(json.dumps(job), color=Ayumi.LRED)
Ayumi.info(
"Completed processing this message for {}".format(
job['episode']),
color=Ayumi.LGREEN)
message.ack()
except rabbitpy.exceptions.AMQPConnectionForced:
Ayumi.rabbitpy_channel = None
Ayumi.critical(
"Operator manually closed RabbitMQ connection, shutting down.",
color=Ayumi.RED)
# Use return for now because in some cases, calling exit() may invoke the retry() header.
return
def sig_handler(sig, frame):
Ayumi.warning("SIG command {} detected, exiting...".format(sig),
color=Ayumi.LRED)
sys.exit()
def rss(last_guid=None):
try:
with rabbitpy.Connection(
'amqp://{username}:{password}@{host}:{port}/{vhost}'.format(
username=settings.get_fresh('RABBITMQ_USERNAME'),
password=settings.get_fresh('RABBITMQ_PASSWORD'),
host=settings.get_fresh('RABBITMQ_HOST'),
port=settings.get_fresh('RABBITMQ_PORT'),
vhost=settings.get_fresh('RABBITMQ_VHOST'))) as conn:
with conn.channel() as channel:
Ayumi.set_rabbitpy_channel(channel)
channel.enable_publisher_confirms()
while True:
Ayumi.info("Now starting feed fetch.", color=Ayumi.LCYAN)
feed = feedparser.parse(
settings.get('ACQUISITION_RSS_FEED_URL', None))
accepted_shows = _load_accepted_shows()
Ayumi.debug(
"Loaded accepted shows map: {}".format(accepted_shows))
history = _load_history()
new_history = list()
for entry in feed.entries:
# Fetch data first
title, link, guid = entry.title, entry.link, entry.guid
Ayumi.debug(
'Encountered RSS item with title "{}", and guid "{}"'
.format(title, guid))
# If feed item with last GUID encountered, do not process any further
if guid == last_guid:
Ayumi.debug(
"Encountered RSS item with last_guid {} matching argument, breaking and writing history."
.format(last_guid),
color=Ayumi.YELLOW)
break
# Check the title data
# Use the parsed title to match user provided titles.
parsed_title = anitopy.parse(title)['anime_title']
if _strip_title(parsed_title) not in accepted_shows:
Ayumi.info(
'Feed item with title "{}" (show title: "{}") is not in accepted shows, skipping.'
.format(title, parsed_title))
else:
if guid in history:
# This item has been previously processed, skip it.
Ayumi.info(
'Feed item with title "{}" (show title: "{}") has already been processed, skipping.'
.format(title, parsed_title),
color=Ayumi.GREEN)
else:
# A new feeditem! Let us process it.
Ayumi.info(
'Feed item with title "{}" (show title: "{}") is in accepted shows, processing.'
.format(title, parsed_title),
color=Ayumi.YELLOW)
message = rabbitpy.Message(
channel,
json.dumps({
"title":
title,
"link":
link,
"guid":
guid,
"show_title":
accepted_shows[_strip_title(
parsed_title)]
}))
acquisition_rss_exchange_name = settings.get(
'ACQUISITION_RSS_EXCHANGE')
while not message.publish(
acquisition_rss_exchange_name,
mandatory=True):
Ayumi.warning(
'Failed to publish feed item with title "{}" to exchange "{}", retrying in 60s...'
.format(title,
acquisition_rss_exchange_name),
color=Ayumi.RED)
sleep(60)
Ayumi.info(
'Published feed item with title "{}" to exchange "{}".'
.format(
title,
acquisition_rss_exchange_name,
),
color=Ayumi.LGREEN)
# Keep all items processed in the new history - it will be auto deleted by the expiry of the RSS
Ayumi.debug(
'Appending item "{}" with title "{}" (show title: "{}") to new_history for write.'
.format(guid, title, parsed_title),
color=Ayumi.YELLOW)
new_history.append(guid)
_write_history(new_history)
# Sleep till the next iteration
sleep_duration = settings.get(
'ACQUISITION_RSS_SLEEP_INTERVAL',
_DEFAULT_SLEEP_INTERVAL)
Ayumi.info(
"Now sleeping {} seconds.".format(sleep_duration),
color=Ayumi.LCYAN)
sleep(sleep_duration)
except rabbitpy.exceptions.AMQPConnectionForced:
Ayumi.rabbitpy_channel = None
Ayumi.critical(
"Operator manually closed RabbitMQ connection, shutting down.",
color=Ayumi.RED)
# Use return for now because in some cases, calling exit() may invoke the retry() header.
return
def consume():
try:
with rabbitpy.Connection('amqp://{username}:{password}@{host}:{port}/{vhost}'.format(
username=settings.get('RABBITMQ_USERNAME'),
password=settings.get('RABBITMQ_PASSWORD'),
host=settings.get('RABBITMQ_HOST'),
port=settings.get('RABBITMQ_PORT'),
vhost=settings.get('RABBITMQ_VHOST')
)) as conn:
with conn.channel() as channel:
Ayumi.set_rabbitpy_channel(channel)
queue = rabbitpy.Queue(channel, settings.get('DISTRIBUTORS_RCLONE_QUEUE'))
queue.declare(passive=True)
Ayumi.info("Now listening for messages from AMQP provider.", color=Ayumi.YELLOW)
for message in queue.consume(prefetch=1):
try:
job = json.loads(message.body.decode('utf-8'))
except json.JSONDecodeError:
Ayumi.warning("Received a job that is invalid json, not processing.", color=Ayumi.LRED)
message.reject()
continue
Ayumi.info("Received a new job: {}".format(json.dumps(job)), color=Ayumi.CYAN)
if metsuke.validate(job):
Ayumi.debug("Loaded show: {}".format(job['show']))
Ayumi.debug("Loaded episode: {}".format(job['episode']))
Ayumi.debug("Loaded filesize: {}".format(job['filesize']))
Ayumi.debug("Loaded sub type: {}".format(job['sub']))
metsuke_job = metsuke.Job(
job['show'], job['episode'], job['filesize'], job['sub'])
with tempfile.NamedTemporaryFile(suffix=".conf", mode="w+b") as rconf, tempfile.TemporaryDirectory() as tempdir:
Ayumi.debug("Opening context managed rclone config file under path: {}.".format(rconf.name))
Ayumi.debug("Opening context managed rclone temporary directory under path: {}".format(tempdir))
rconf.write(str.encode(settings.get("RCLONE_CONFIG_FILE")))
rconf.flush() # YOU MUST FLUSH THE FILE SO RCLONE CAN READ IT!
Ayumi.debug("Configurations written to temporary file. Size is {} bytes.".format(rconf.tell()))
dl_sources = None
up_dests = None
if job['sub'].lower() == "softsub":
dl_sources = settings.get("DISTRIBUTORS_RCLONE_SOFTSUB_DOWNLOAD")
up_dests = settings.get("DISTRIBUTORS_RCLONE_SOFTSUB_UPLOAD")
elif job['sub'].lower() == "hardsub":
dl_sources = settings.get("DISTRIBUTORS_RCLONE_HARDSUB_DOWNLOAD")
up_dests = settings.get("DISTRIBUTORS_RCLONE_HARDSUB_UPLOAD")
Ayumi.debug("Fresh fetched download sources as: {}".format(" ".join(dl_sources)))
Ayumi.debug("Fresh fetched upload sources as: {}".format(" ".join(up_dests)))
try:
temp_ep = shikyou.download(metsuke_job, dl_sources, tempdir, rconf.name, settings.get("RCLONE_FLAGS", ""))
if temp_ep:
shikyou.upload(metsuke_job, up_dests, temp_ep, rconf.name, settings.get("RCLONE_FLAGS", ""))
else:
Ayumi.warning("Unable to find requested job in any sources, nacking...", color=Ayumi.RED)
message.nack()
continue
except shikyou.ShikyouResponseException:
Ayumi.critical("Rclone threw an unexpected response code, rejecting.", color=Ayumi.RED)
message.reject()
continue
except shikyou.ShikyouTimeoutException:
Ayumi.warning("Rclone timed out whilhe executing, nacking.", color=Ayumi.RED)
message.nack()
continue
Ayumi.debug("Closed context managed rclone config file.")
Ayumi.debug("Closed context managed temporary directory.")
else:
Ayumi.warning("Received a job that Metsuke was not able to validate.", color=Ayumi.LRED)
Ayumi.warning(json.dumps(job), color=Ayumi.LRED)
Ayumi.info("Completed processing this message for {}".format(job['episode']), color=Ayumi.LGREEN)
message.ack()
except rabbitpy.exceptions.AMQPConnectionForced:
Ayumi.rabbitpy_channel = None
Ayumi.critical("Operator manually closed RabbitMQ connection, shutting down.", color=Ayumi.RED)
# Use return for now because in some cases, calling exit() may invoke the retry() header.
return
def bittorrent():
try:
with rabbitpy.Connection('amqp://{username}:{password}@{host}:{port}/{vhost}'.format(
username=settings.get('RABBITMQ_USERNAME'),
password=settings.get('RABBITMQ_PASSWORD'),
host=settings.get('RABBITMQ_HOST'),
port=settings.get('RABBITMQ_PORT'),
vhost=settings.get('RABBITMQ_VHOST')
)) as conn:
with conn.channel() as channel:
Ayumi.set_rabbitpy_channel(channel)
channel.enable_publisher_confirms()
queue_name = settings.get('ACQUISITION_BITTORRENT_QUEUE')
Ayumi.debug("Connecting to queue: {}".format(queue_name))
queue = rabbitpy.Queue(channel, queue_name)
queue.declare(passive=True)
Ayumi.info('Now listening for messages on queue: {}...'.format(
queue_name), color=Ayumi.LYELLOW)
for message in queue.consume(prefetch=1):
Ayumi.info(
"Received new message, starting...", color=Ayumi.CYAN)
feeditem_preprocess = _load_amqp_message_body(message)
Ayumi.debug('Loaded message raw: "{}"'.format(
feeditem_preprocess))
if not feeditem_preprocess or not metsuke.validate_feeditem(feeditem_preprocess):
Ayumi.error('Invalid message received, rejecting. Output: "{}"'.format(
feeditem_preprocess), color=Ayumi.RED)
message.reject()
continue
# Load initial data
feeditem: metsuke.FeedItem = metsuke.generate_feeditem(
feeditem_preprocess)
shows_map = _load_shows_map()
overload_title = feeditem.show_title
Ayumi.info(
'Setting overload title: "{}"'.format(overload_title))
# If there is a central override, use it instead.
if _strip_title(anitopy.parse(feeditem.title)['anime_title']) in shows_map:
central_overload_title = shows_map[_strip_title(
feeditem.title)]
Ayumi.info('Overwriting overload title with central overload title: "{}"'.format(
central_overload_title))
overload_title = central_overload_title
with tempfile.TemporaryDirectory() as temp_dir:
Ayumi.debug(
'Created temporary directory under path: "{}"'.format(temp_dir))
# Download the episode
try:
res = subprocess.run(
[
"aria2c",
"--seed-time=0",
"--rpc-save-upload-metadata=false",
"--bt-save-metadata=false",
"--dir={}".format(temp_dir),
feeditem.link
]
)
if res.returncode != 0:
Ayumi.warning(
"Aria2 did not return a 0 exit code, assuming download errored and nacking.", color=Ayumi.RED)
message.nack()
continue
except subprocess.TimeoutExpired:
Ayumi.warning(
"Download via webtorrent timed out - nacking.", color=Ayumi.RED)
message.nack()
continue
if res.returncode != 0:
Ayumi.warning(
"Webtorrent did not have a return code of 0, nacking.", color=Ayumi.RED)
message.nack()
continue
# Rename it
potential_files = [f for f in os.listdir(
temp_dir) if f.endswith(".mkv")]
Ayumi.debug(
"Loaded potential files: {}".format(potential_files))
if len(potential_files) != 1:
Ayumi.warning(
"Found more than one .mkv file, rejecting this job.", color=Ayumi.RED)
message.reject()
continue
dl_file = potential_files[0]
Ayumi.info('Found file: "{}"'.format(dl_file))
dl_file_path = os.path.abspath(
'{}/{}'.format(_clean_title(temp_dir), potential_files[0]))
Ayumi.debug(
'dl_file_path: "{}"'.format(dl_file_path))
# Remove unneeded files
# TODO: THIS IS A HOTFIX, CHANGE LOGIC IN B2
bad_files = [f for f in os.listdir(
temp_dir) if not f.endswith(".mkv")]
Ayumi.debug("Found bad files: {}".format(bad_files))
for bf in bad_files:
try:
Ayumi.debug("Removing bad file: {}".format(bf))
os.remove(
'{}/{}'.format(_clean_title(temp_dir), bf))
except:
Ayumi.debug("Removing bad tree: {}".format(bf))
shutil.rmtree(
'{}/{}'.format(_clean_title(temp_dir), bf))
# Move the file to proper layout with updated name
dl_file_new_name = _generate_new_filename(dl_file)
Ayumi.info('Generated new episode name: "{}"'.format(
dl_file_new_name))
dl_file_new_dir = "{}/{}".format(
temp_dir, overload_title)
Ayumi.debug(
'dl_file_new_dir: "{}"'.format(dl_file_new_dir))
dl_file_new_path = "{}/{}".format(
dl_file_new_dir, dl_file_new_name)
Ayumi.debug(
'dl_file_new_path: "{}"'.format(
dl_file_new_path))
Ayumi.debug('Moving "{}" to "{}"'.format(
dl_file_path, dl_file_new_path))
os.mkdir(dl_file_new_dir)
shutil.move(dl_file_path, dl_file_new_path)
# Upload the file to rclone destination
with tempfile.NamedTemporaryFile(suffix=".conf", mode="w+b") as rconf:
rconf.write(str.encode(
settings.get("RCLONE_CONFIG_FILE")))
rconf.flush()
Ayumi.debug(
'Created temporary rclone file under path: "{}"'.format(rconf.name))
rclone_dest = _clean_title(settings.get(
"ACQUISITION_BITTORRENT_RCLONE_DEST"))
rclone_flags = settings.get("RCLONE_FLAGS", "")
command = [
"rclone", "--config={}".format(rconf.name), "copy", temp_dir, rclone_dest]
command.extend(rclone_flags.split())
Ayumi.debug(
'Rclone command to be run: "{}"'.format(command))
try:
Ayumi.info(
'Now uploading new blob to: "{}"'.format(rclone_dest))
rclone_res = subprocess.run(
command, timeout=3600)
if rclone_res.returncode != 0:
Ayumi.warning('Rclone returned non-zero code of {}, nacking.'.format(
rclone_res.returncode), color=Ayumi.LRED)
message.nack()
except subprocess.TimeoutExpired:
Ayumi.warning(
'Rclone upload timed out, nacking.', color=Ayumi.LRED)
message.nack()
continue
# Fetch information on the file to create a job
new_message = rabbitpy.Message(channel, dumps(
{
"show": overload_title,
"episode": dl_file_new_name,
"filesize": int(os.path.getsize(dl_file_new_path)),
"sub": "SOFTSUB"
}
))
acquisition_bittorrent_exchange_name = settings.get(
'ACQUISITION_BITTORRENT_EXCHANGE')
Ayumi.info('Sending to exchange: "{}"'.format(
acquisition_bittorrent_exchange_name), color=Ayumi.CYAN)
while not new_message.publish(acquisition_bittorrent_exchange_name, mandatory=True):
Ayumi.warning(
"Failed to publish feed item, trying again in 60 seconds")
sleep(60)
Ayumi.info("Published feed item with title: " +
overload_title, color=Ayumi.LGREEN)
message.ack()
except rabbitpy.exceptions.AMQPConnectionForced:
Ayumi.warning(
"Operator manually closed RabbitMQ connection, shutting down.", color=Ayumi.LYELLOW)
return
def _get_shows(status: str, **kwargs):
"""
Fetch all the airing, recently finished, and soon to be airing shows
Run string siimlarity and return the string most likely to be the same show.
"""
shows = list()
page = 1
has_next_page = True
while has_next_page:
Ayumi.info("Now requesting shows from page {} of status {}...".format(page, status), color=Ayumi.CYAN)
variables = {
'page': page,
'status': status
}
k_year = kwargs.get('year')
if k_year:
Ayumi.debug("Set seasonYear argument to {}".format(k_year), color=Ayumi.CYAN)
variables['seasonYear'] = int(k_year)
k_sd = kwargs.get('start_date')
if k_sd:
Ayumi.debug("Set startDate_lesser argument to {}".format(k_sd), color=Ayumi.CYAN)
variables['startDate'] = int(k_sd)
k_ed = kwargs.get('end_date')
if k_ed:
Ayumi.debug("Set endDate_greater argument to {}".format(k_ed), color=Ayumi.CYAN)
variables['endDate'] = int(k_ed)
try:
ani_res = requests.post(
ANILIST_API_URL,
json={
'query': QUERY,
'variables': variables
})
except requests.exceptions.ConnectionError:
Ayumi.warning("Unable to contact Anilist, the site or your connection may be down.", color=Ayumi.LRED)
return shows
if ani_res.status_code != 200:
Ayumi.warning("Anilist returned unaccepted HTTP code {} upon request.".format(ani_res.status_code), color=Ayumi.LRED)
return shows
try:
ani_json = ani_res.json()['data']['Page']
has_next_page = ani_json['pageInfo']['hasNextPage']
page += 1
for media in ani_json['media']:
for media_title in media['title'].values():
if media_title:
Ayumi.debug("Adding show {} to show list".format(media_title))
shows.append(media_title)
except:
Ayumi.warning("Unable to parse JSON response from Anilist.", color=Ayumi.LRED)
return shows
return shows
# Send the notification to RabbitMQ
# Because of threading reasons the publishing could fail, but we're not concerned with guaranteeing it.
try:
message = rabbitpy.Message(
channel,
job,
properties={
"content_type": "application/json",
"delivery_mode": 2
})
if message.publish(settings.get('KOTEN_EXCHANGE')):
Ayumi.info("Job successfully published to RabbitMQ", color=Ayumi.LGREEN)
else:
Ayumi.warning("Job unsuccessfully published to RabbitMQ", color=Ayumi.LRED)
raise Exception()
except:
# RabbitPy could have timed out, etc. at this point
Ayumi.warning("Some kind of error occured when attempting to publish to RabbitMQ, printing json body.", color=Ayumi.LYELLOW)
Ayumi.warning(str(job))
if settings.get('KOTEN_CLEANUP'):
Ayumi.info("Cleanup mode enabled, removing original file.", color=Ayumi.LYELLOW)
# Two cleanup modes - only the file itself, or possibly also the parent folder.
if util._show_manually_specified(new_file):
Ayumi.info("Mode 1 detected, removing file and parent directory if allowed.")
p = pathlib.Path(src_path)
p.unlink()
