def _check_exists(config: str, source: str, job: Job) -> bool: """ Checks if the jobs exists under the source. Note: Does not validate under which path, just validates that jobs exists somewhere in: source/(...probably job.show)/job.episode """ try: response = _run([ "rclone", "--config={}".format(config), "lsjson", "-R", "{}/{}/".format(source, job.show) ]) episode_list = loads(response.stdout.decode('utf-8')) for episode in episode_list: Ayumi.debug("Checking {} against episode {}".format( job.episode, episode['Name'])) if episode['Name'] == job.episode: Ayumi.info("Found episode {} in {}".format( job.episode, source)) return True Ayumi.info("Didn't find episode {} in {}".format(job.episode, source)) return False except: # Typically hit if the source doesn't exist. Ayumi.warning( "Error occured while checking source {} - does it exist?".format( source)) return False
def observe(): try: with rabbitpy.Connection('amqp://{username}:{password}@{host}:{port}/{vhost}'.format( username=settings.get('RABBITMQ_USERNAME'), password=settings.get('RABBITMQ_PASSWORD'), host=settings.get('RABBITMQ_HOST'), port=settings.get('RABBITMQ_PORT'), vhost=settings.get('RABBITMQ_VHOST') )) as conn: with conn.channel() as channel: channel.enable_publisher_confirms() Ayumi.set_rabbitpy_channel(channel) Ayumi.info("Now connected AMQP provider.", color=Ayumi.GREEN) event_handler = IzumiHandler(channel) observer = Observer() observer.schedule(event_handler, settings.get('KOTEN_WATCH_PATH', DEFAULT_WATCH_PATH), recursive=True) observer.start() Ayumi.info("Now observing: {}".format(settings.get('KOTEN_WATCH_PATH', DEFAULT_WATCH_PATH)), color=Ayumi.BLUE) try: while True: time.sleep(settings.get('KOTEN_SLEEP_INTERVAL', DEFAULT_SLEEP_INTERVAL)) except: Ayumi.warning("Detected SIGKILL or error, returning...", color=Ayumi.YELLOW) observer.stop() observer.join() except rabbitpy.exceptions.AMQPConnectionForced: Ayumi.rabbitpy_channel = None Ayumi.critical("Operator manually closed RabbitMQ connection, shutting down.", color=Ayumi.RED) # Use return for now because in some cases, calling exit() may invoke the retry() header. return
def search(show): airing = _single_search(show, "RELEASING") if airing: Ayumi.info( "Found show {} in RELEASING, returning Hisha object.".format(show), color=Ayumi.LCYAN) return _create_hisha_object(airing, show) finished = _page_search(show, "FINISHED") if finished: Ayumi.info( "Found show {} in FINISHED, returning Hisha object.".format(show), color=Ayumi.LCYAN) return _create_hisha_object(finished, show) not_yet_released = _single_search(show, "NOT_YET_RELEASED") if not_yet_released: Ayumi.info( "Found show {} in NOT_YET_RELEASED, returning Hisha object.". format(show), color=Ayumi.LCYAN) return _create_hisha_object(not_yet_released, show) # None of them found a show, so create a dummy Hisha object. Ayumi.info( "Creating dummy Hisha object for {} with default values.".format(show), color=Ayumi.LCYAN) return _create_hisha_object(None, show)
def is_user_watching_id(user, malID, times=5): """ Is a user watching this show or not? Params: user: username to lookup malID: malID to match against """ Ayumi.debug("Now finding if \"{}\" is in {}'s list".format(malID, user)) anime_list = akari_list(user, times) for show in anime_list: if str(show['mal_id']) == str(malID): Ayumi.info("\"{}\" was found in {}'s list".format(malID, user), color=Ayumi.LGREEN) return True Ayumi.info("\"{}\" was not found in {}'s list".format(malID, user), color=Ayumi.LYELLOW) return False
def find_closest_title(title): """ Finds the closest title from Anilist for Airing and Not_Yet_Released shows """ now = datetime.datetime.now() date_next_month = int((now + datetime.timedelta(weeks=4)).strftime("%Y%m%d")) date_last_month = int((now - datetime.timedelta(weeks=4)).strftime("%Y%m%d")) shows = list() heap = list() shows.extend(_get_shows("RELEASING")) shows.extend(_get_shows("NOT_YET_RELEASED", start_date=date_next_month)) shows.extend(_get_shows("FINISHED", end_date=date_last_month)) for show in shows: ratio = _similarity(title, show) Ayumi.debug('Matched "{}" against "{}" for a ratio of {}'.format(title, show, ratio)) heapq.heappush(heap, (ratio, show)) top_5 = heapq.nlargest(5, heap) Ayumi.info("Displaying (up to) top 5 matches of {} results:".format(len(heap)), color=Ayumi.LBLUE) for top in top_5: Ayumi.info("{}: {}".format(top[1], top[0]), color=Ayumi.LBLUE) if top_5: Ayumi.info('Returning top match: {}'.format(top_5[0][1]), color=Ayumi.LGREEN) return top_5[0][1] else: Ayumi.warning("No shows were fetched by Naomi, returning None", color=Ayumi.LYELLOW) return None
def is_user_watching_names(user, show_name, times=5): """ Is a user watching this show or not? Params: user: username to lookup show_name: show name to match against Returns True if the show was found in the list, false if not """ Ayumi.debug("Now finding if \"{}\" is in {}'s list".format( show_name, user)) anime_list = akari_list(user, times) for show in anime_list: if show['title'] == show_name: Ayumi.info("\"{}\" was found in {}'s list".format(show_name, user), color=Ayumi.LGREEN) return True Ayumi.info("\"{}\" was not found in {}'s list".format(show_name, user), color=Ayumi.LYELLOW) return False
def upload(job: Job, destinations: List[str], upload_file: str, config: str, flags: str) -> None: """ Upload the completed new hardsub file into the rclone destinations Returns a boolean based on success job: Job to do! This is the job of the HARDSUB file destinations: list of rlcone destinations (e.g., EncoderConf.uploading_destinations) upload_file: Path to the file to be uploaded rclone_config: Path to the rclone config file flags: rclone flag This method will upload the file and include its show name: e.g., 'temp.mp4' --> destination/show/episode.mp4 """ for dest in destinations: rclone_dest = "{}/{}/{}".format(_clean(dest), job.show, job.episode) command = [ "rclone", "--config={}".format(config), "copyto", upload_file, rclone_dest ] command.extend(flags.split()) Ayumi.debug("Now running command: {}".format(" ".join(command))) Ayumi.info("Now uploading file to {}".format(dest)) try: _run(command) except ShikyouResponseException: Ayumi.error( "Rclone command returned a bad return code, contact the developer.", color=Ayumi.LRED) raise ShikyouResponseException() except ShikyouTimeoutException: Ayumi.error("Rclone command timed out, are your sources okay?", color=Ayumi.LRED) raise ShikyouTimeoutException() Ayumi.info("Completed uploading files.", color=Ayumi.LGREEN)
def on_created(self, event): Ayumi.info('Detected new creation event under path "{}", running preflight checks.'.format(event.src_path), color=Ayumi.LCYAN) # Do not process new directory events if event.is_directory: Ayumi.info('Detected new event directory under path "{}", ignoring.'.format(event.src_path), color=Ayumi.LYELLOW) return if not os.path.exists(event.src_path): Ayumi.info("Event detected as false positive, will not proceed.", color=Ayumi.LYELLOW) return Ayumi.info("Preflight check good, starting pipeline.", color=Ayumi.LGREEN) on_new_file(event.src_path, self.channel)
def download(job: Job, sources: List[str], tempfolder: str, config: str, flags: str) -> str: """ Download the provided episode from sources Returns the path of the downloaded file job: Job to do! sources: list of rclone sources (EncoderConf.downloading_sources) tmppath: Path of the temporary folder rclone_config: Path to the rclone config file flags: rclone flags """ for source in sources: Ayumi.debug("Checking for existence from source: {}".format(source)) if _check_exists(config, source, job): Ayumi.info( "Now downloading episode from source: {}".format(source)) src_file = "{}/{}/{}".format(_clean(source), job.show, job.episode) Ayumi.debug("Sourcing from rclone path: {}".format(src_file)) dest_file = "{}/{}".format(_clean(tempfolder), "temp") Ayumi.debug("Downloading to destination: {}".format(dest_file)) command = [ "rclone", "--config={}".format(config), "copyto", src_file, dest_file ] command.extend(flags.split()) Ayumi.debug("Now running command: {}".format(" ".join(command))) Ayumi.info("Now starting download.", color=Ayumi.LCYAN) try: _run(command) except ShikyouResponseException: Ayumi.error( "Rclone command returned a bad return code, contact the developer.", color=Ayumi.LRED) raise ShikyouResponseException() except ShikyouTimeoutException: Ayumi.error("Rclone command timed out, are your sources okay?", color=Ayumi.LRED) raise ShikyouTimeoutException() Ayumi.info("Completed downloading files.", color=Ayumi.LGREEN) return dest_file else: Ayumi.debug( "Requested episode doesn't exist under source {}".format( source)) Ayumi.warning("No download sources contained the file.") return None
def on_new_file(src_path, channel): show_name = None episode_name = None new_file = src_path.replace(os.path.commonpath([settings.get('KOTEN_WATCH_PATH', DEFAULT_WATCH_PATH), src_path]) + "/", "") if m := util._show_manually_specified(new_file): Ayumi.info("Detected show name and episode name in event, using Mode 1.") show_name = m.group(1) episode_name = util._clean_episode_name(m.group(2)) Ayumi.info("New show name: {}".format(show_name), color=Ayumi.LYELLOW) Ayumi.info("New episode name: {}".format(episode_name), color=Ayumi.LYELLOW)
def consume(): try: with rabbitpy.Connection( 'amqp://{username}:{password}@{host}:{port}/{vhost}'.format( username=settings.get('RABBITMQ_USERNAME'), password=settings.get('RABBITMQ_PASSWORD'), host=settings.get('RABBITMQ_HOST'), port=settings.get('RABBITMQ_PORT'), vhost=settings.get('RABBITMQ_VHOST'))) as conn: with conn.channel() as channel: Ayumi.set_rabbitpy_channel(channel) queue = rabbitpy.Queue( channel, settings.get('NOTIFICATIONS_DISCORD_WEBHOOK_QUEUE', 'nonexistent')) queue.declare(passive=True) Ayumi.info("Now listening for messages from AMQP provider.", color=Ayumi.YELLOW) for message in queue.consume(prefetch=1): try: job = json.loads(message.body.decode('utf-8')) except json.JSONDecodeError: Ayumi.warning( "Received a job that is invalid json, not processing.", color=Ayumi.LRED) message.reject() continue Ayumi.info("Received a new job: {}".format( json.dumps(job)), color=Ayumi.CYAN) if metsuke.validate(job): Ayumi.debug("Loaded show: {}".format(job['show'])) Ayumi.debug("Loaded episode: {}".format( job['episode'])) Ayumi.debug("Loaded filesize: {}".format( job['filesize'])) Ayumi.debug("Loaded sub type: {}".format(job['sub'])) embed = _generate_embed(metsuke.generate(job), hisha.search(job['show'])) Ayumi.info( "Beginning sending embeds to webhook endpoints.", color=Ayumi.CYAN) for endpoint in settings.get( 'NOTIFICATIONS_DISCORD_WEBHOOK_ENDPOINTS' ).to_list(): try: requests.post(endpoint, json=embed, timeout=5) Ayumi.debug( "Sent embed to {}".format(endpoint)) except: Ayumi.warning( "Failed to send embed to {}".format( endpoint), color=Ayumi.RED) else: Ayumi.warning( "Received a job that Metsuke was not able to validate.", color=Ayumi.LRED) Ayumi.warning(json.dumps(job), color=Ayumi.LRED) Ayumi.info( "Completed processing this message for {}".format( job['episode']), color=Ayumi.LGREEN) message.ack() except rabbitpy.exceptions.AMQPConnectionForced: Ayumi.rabbitpy_channel = None Ayumi.critical( "Operator manually closed RabbitMQ connection, shutting down.", color=Ayumi.RED) # Use return for now because in some cases, calling exit() may invoke the retry() header. return
def rss(last_guid=None): try: with rabbitpy.Connection( 'amqp://{username}:{password}@{host}:{port}/{vhost}'.format( username=settings.get_fresh('RABBITMQ_USERNAME'), password=settings.get_fresh('RABBITMQ_PASSWORD'), host=settings.get_fresh('RABBITMQ_HOST'), port=settings.get_fresh('RABBITMQ_PORT'), vhost=settings.get_fresh('RABBITMQ_VHOST'))) as conn: with conn.channel() as channel: Ayumi.set_rabbitpy_channel(channel) channel.enable_publisher_confirms() while True: Ayumi.info("Now starting feed fetch.", color=Ayumi.LCYAN) feed = feedparser.parse( settings.get('ACQUISITION_RSS_FEED_URL', None)) accepted_shows = _load_accepted_shows() Ayumi.debug( "Loaded accepted shows map: {}".format(accepted_shows)) history = _load_history() new_history = list() for entry in feed.entries: # Fetch data first title, link, guid = entry.title, entry.link, entry.guid Ayumi.debug( 'Encountered RSS item with title "{}", and guid "{}"' .format(title, guid)) # If feed item with last GUID encountered, do not process any further if guid == last_guid: Ayumi.debug( "Encountered RSS item with last_guid {} matching argument, breaking and writing history." .format(last_guid), color=Ayumi.YELLOW) break # Check the title data # Use the parsed title to match user provided titles. parsed_title = anitopy.parse(title)['anime_title'] if _strip_title(parsed_title) not in accepted_shows: Ayumi.info( 'Feed item with title "{}" (show title: "{}") is not in accepted shows, skipping.' .format(title, parsed_title)) else: if guid in history: # This item has been previously processed, skip it. Ayumi.info( 'Feed item with title "{}" (show title: "{}") has already been processed, skipping.' .format(title, parsed_title), color=Ayumi.GREEN) else: # A new feeditem! Let us process it. Ayumi.info( 'Feed item with title "{}" (show title: "{}") is in accepted shows, processing.' .format(title, parsed_title), color=Ayumi.YELLOW) message = rabbitpy.Message( channel, json.dumps({ "title": title, "link": link, "guid": guid, "show_title": accepted_shows[_strip_title( parsed_title)] })) acquisition_rss_exchange_name = settings.get( 'ACQUISITION_RSS_EXCHANGE') while not message.publish( acquisition_rss_exchange_name, mandatory=True): Ayumi.warning( 'Failed to publish feed item with title "{}" to exchange "{}", retrying in 60s...' .format(title, acquisition_rss_exchange_name), color=Ayumi.RED) sleep(60) Ayumi.info( 'Published feed item with title "{}" to exchange "{}".' .format( title, acquisition_rss_exchange_name, ), color=Ayumi.LGREEN) # Keep all items processed in the new history - it will be auto deleted by the expiry of the RSS Ayumi.debug( 'Appending item "{}" with title "{}" (show title: "{}") to new_history for write.' .format(guid, title, parsed_title), color=Ayumi.YELLOW) new_history.append(guid) _write_history(new_history) # Sleep till the next iteration sleep_duration = settings.get( 'ACQUISITION_RSS_SLEEP_INTERVAL', _DEFAULT_SLEEP_INTERVAL) Ayumi.info( "Now sleeping {} seconds.".format(sleep_duration), color=Ayumi.LCYAN) sleep(sleep_duration) except rabbitpy.exceptions.AMQPConnectionForced: Ayumi.rabbitpy_channel = None Ayumi.critical( "Operator manually closed RabbitMQ connection, shutting down.", color=Ayumi.RED) # Use return for now because in some cases, calling exit() may invoke the retry() header. return
def bittorrent(): try: with rabbitpy.Connection('amqp://{username}:{password}@{host}:{port}/{vhost}'.format( username=settings.get('RABBITMQ_USERNAME'), password=settings.get('RABBITMQ_PASSWORD'), host=settings.get('RABBITMQ_HOST'), port=settings.get('RABBITMQ_PORT'), vhost=settings.get('RABBITMQ_VHOST') )) as conn: with conn.channel() as channel: Ayumi.set_rabbitpy_channel(channel) channel.enable_publisher_confirms() queue_name = settings.get('ACQUISITION_BITTORRENT_QUEUE') Ayumi.debug("Connecting to queue: {}".format(queue_name)) queue = rabbitpy.Queue(channel, queue_name) queue.declare(passive=True) Ayumi.info('Now listening for messages on queue: {}...'.format( queue_name), color=Ayumi.LYELLOW) for message in queue.consume(prefetch=1): Ayumi.info( "Received new message, starting...", color=Ayumi.CYAN) feeditem_preprocess = _load_amqp_message_body(message) Ayumi.debug('Loaded message raw: "{}"'.format( feeditem_preprocess)) if not feeditem_preprocess or not metsuke.validate_feeditem(feeditem_preprocess): Ayumi.error('Invalid message received, rejecting. Output: "{}"'.format( feeditem_preprocess), color=Ayumi.RED) message.reject() continue # Load initial data feeditem: metsuke.FeedItem = metsuke.generate_feeditem( feeditem_preprocess) shows_map = _load_shows_map() overload_title = feeditem.show_title Ayumi.info( 'Setting overload title: "{}"'.format(overload_title)) # If there is a central override, use it instead. if _strip_title(anitopy.parse(feeditem.title)['anime_title']) in shows_map: central_overload_title = shows_map[_strip_title( feeditem.title)] Ayumi.info('Overwriting overload title with central overload title: "{}"'.format( central_overload_title)) overload_title = central_overload_title with tempfile.TemporaryDirectory() as temp_dir: Ayumi.debug( 'Created temporary directory under path: "{}"'.format(temp_dir)) # Download the episode try: res = subprocess.run( [ "aria2c", "--seed-time=0", "--rpc-save-upload-metadata=false", "--bt-save-metadata=false", "--dir={}".format(temp_dir), feeditem.link ] ) if res.returncode != 0: Ayumi.warning( "Aria2 did not return a 0 exit code, assuming download errored and nacking.", color=Ayumi.RED) message.nack() continue except subprocess.TimeoutExpired: Ayumi.warning( "Download via webtorrent timed out - nacking.", color=Ayumi.RED) message.nack() continue if res.returncode != 0: Ayumi.warning( "Webtorrent did not have a return code of 0, nacking.", color=Ayumi.RED) message.nack() continue # Rename it potential_files = [f for f in os.listdir( temp_dir) if f.endswith(".mkv")] Ayumi.debug( "Loaded potential files: {}".format(potential_files)) if len(potential_files) != 1: Ayumi.warning( "Found more than one .mkv file, rejecting this job.", color=Ayumi.RED) message.reject() continue dl_file = potential_files[0] Ayumi.info('Found file: "{}"'.format(dl_file)) dl_file_path = os.path.abspath( '{}/{}'.format(_clean_title(temp_dir), potential_files[0])) Ayumi.debug( 'dl_file_path: "{}"'.format(dl_file_path)) # Remove unneeded files # TODO: THIS IS A HOTFIX, CHANGE LOGIC IN B2 bad_files = [f for f in os.listdir( temp_dir) if not f.endswith(".mkv")] Ayumi.debug("Found bad files: {}".format(bad_files)) for bf in bad_files: try: Ayumi.debug("Removing bad file: {}".format(bf)) os.remove( '{}/{}'.format(_clean_title(temp_dir), bf)) except: Ayumi.debug("Removing bad tree: {}".format(bf)) shutil.rmtree( '{}/{}'.format(_clean_title(temp_dir), bf)) # Move the file to proper layout with updated name dl_file_new_name = _generate_new_filename(dl_file) Ayumi.info('Generated new episode name: "{}"'.format( dl_file_new_name)) dl_file_new_dir = "{}/{}".format( temp_dir, overload_title) Ayumi.debug( 'dl_file_new_dir: "{}"'.format(dl_file_new_dir)) dl_file_new_path = "{}/{}".format( dl_file_new_dir, dl_file_new_name) Ayumi.debug( 'dl_file_new_path: "{}"'.format( dl_file_new_path)) Ayumi.debug('Moving "{}" to "{}"'.format( dl_file_path, dl_file_new_path)) os.mkdir(dl_file_new_dir) shutil.move(dl_file_path, dl_file_new_path) # Upload the file to rclone destination with tempfile.NamedTemporaryFile(suffix=".conf", mode="w+b") as rconf: rconf.write(str.encode( settings.get("RCLONE_CONFIG_FILE"))) rconf.flush() Ayumi.debug( 'Created temporary rclone file under path: "{}"'.format(rconf.name)) rclone_dest = _clean_title(settings.get( "ACQUISITION_BITTORRENT_RCLONE_DEST")) rclone_flags = settings.get("RCLONE_FLAGS", "") command = [ "rclone", "--config={}".format(rconf.name), "copy", temp_dir, rclone_dest] command.extend(rclone_flags.split()) Ayumi.debug( 'Rclone command to be run: "{}"'.format(command)) try: Ayumi.info( 'Now uploading new blob to: "{}"'.format(rclone_dest)) rclone_res = subprocess.run( command, timeout=3600) if rclone_res.returncode != 0: Ayumi.warning('Rclone returned non-zero code of {}, nacking.'.format( rclone_res.returncode), color=Ayumi.LRED) message.nack() except subprocess.TimeoutExpired: Ayumi.warning( 'Rclone upload timed out, nacking.', color=Ayumi.LRED) message.nack() continue # Fetch information on the file to create a job new_message = rabbitpy.Message(channel, dumps( { "show": overload_title, "episode": dl_file_new_name, "filesize": int(os.path.getsize(dl_file_new_path)), "sub": "SOFTSUB" } )) acquisition_bittorrent_exchange_name = settings.get( 'ACQUISITION_BITTORRENT_EXCHANGE') Ayumi.info('Sending to exchange: "{}"'.format( acquisition_bittorrent_exchange_name), color=Ayumi.CYAN) while not new_message.publish(acquisition_bittorrent_exchange_name, mandatory=True): Ayumi.warning( "Failed to publish feed item, trying again in 60 seconds") sleep(60) Ayumi.info("Published feed item with title: " + overload_title, color=Ayumi.LGREEN) message.ack() except rabbitpy.exceptions.AMQPConnectionForced: Ayumi.warning( "Operator manually closed RabbitMQ connection, shutting down.", color=Ayumi.LYELLOW) return
def _get_shows(status: str, **kwargs): """ Fetch all the airing, recently finished, and soon to be airing shows Run string siimlarity and return the string most likely to be the same show. """ shows = list() page = 1 has_next_page = True while has_next_page: Ayumi.info("Now requesting shows from page {} of status {}...".format(page, status), color=Ayumi.CYAN) variables = { 'page': page, 'status': status } k_year = kwargs.get('year') if k_year: Ayumi.debug("Set seasonYear argument to {}".format(k_year), color=Ayumi.CYAN) variables['seasonYear'] = int(k_year) k_sd = kwargs.get('start_date') if k_sd: Ayumi.debug("Set startDate_lesser argument to {}".format(k_sd), color=Ayumi.CYAN) variables['startDate'] = int(k_sd) k_ed = kwargs.get('end_date') if k_ed: Ayumi.debug("Set endDate_greater argument to {}".format(k_ed), color=Ayumi.CYAN) variables['endDate'] = int(k_ed) try: ani_res = requests.post( ANILIST_API_URL, json={ 'query': QUERY, 'variables': variables }) except requests.exceptions.ConnectionError: Ayumi.warning("Unable to contact Anilist, the site or your connection may be down.", color=Ayumi.LRED) return shows if ani_res.status_code != 200: Ayumi.warning("Anilist returned unaccepted HTTP code {} upon request.".format(ani_res.status_code), color=Ayumi.LRED) return shows try: ani_json = ani_res.json()['data']['Page'] has_next_page = ani_json['pageInfo']['hasNextPage'] page += 1 for media in ani_json['media']: for media_title in media['title'].values(): if media_title: Ayumi.debug("Adding show {} to show list".format(media_title)) shows.append(media_title) except: Ayumi.warning("Unable to parse JSON response from Anilist.", color=Ayumi.LRED) return shows return shows
show_name = None episode_name = None new_file = src_path.replace(os.path.commonpath([settings.get('KOTEN_WATCH_PATH', DEFAULT_WATCH_PATH), src_path]) + "/", "") if m := util._show_manually_specified(new_file): Ayumi.info("Detected show name and episode name in event, using Mode 1.") show_name = m.group(1) episode_name = util._clean_episode_name(m.group(2)) Ayumi.info("New show name: {}".format(show_name), color=Ayumi.LYELLOW) Ayumi.info("New episode name: {}".format(episode_name), color=Ayumi.LYELLOW) else: Ayumi.debug("Non-conformant episode provided, using Naomi to find show name.") episode_name = util._clean_episode_name(pathlib.PurePath(src_path).name) show_name = naomi.find_closest_title(anitopy.parse(new_file)['anime_title']) Ayumi.info("New show name: {}".format(show_name), color=Ayumi.LYELLOW) Ayumi.info("New episode name: {}".format(episode_name), color=Ayumi.LYELLOW) # There is an event where Anilist is down, and Naomi could return None. # In this case, use the assumed-parsed show as the title if not show_name: show_name = anitopy.parse(new_file)['anime_title'] job = { "show": show_name, "episode": episode_name, "filesize": os.path.getsize(src_path), "sub": "softsub" } # Create the temporary rclone file
def consume(): try: with rabbitpy.Connection('amqp://{username}:{password}@{host}:{port}/{vhost}'.format( username=settings.get('RABBITMQ_USERNAME'), password=settings.get('RABBITMQ_PASSWORD'), host=settings.get('RABBITMQ_HOST'), port=settings.get('RABBITMQ_PORT'), vhost=settings.get('RABBITMQ_VHOST') )) as conn: with conn.channel() as channel: Ayumi.set_rabbitpy_channel(channel) queue = rabbitpy.Queue(channel, settings.get('DISTRIBUTORS_RCLONE_QUEUE')) queue.declare(passive=True) Ayumi.info("Now listening for messages from AMQP provider.", color=Ayumi.YELLOW) for message in queue.consume(prefetch=1): try: job = json.loads(message.body.decode('utf-8')) except json.JSONDecodeError: Ayumi.warning("Received a job that is invalid json, not processing.", color=Ayumi.LRED) message.reject() continue Ayumi.info("Received a new job: {}".format(json.dumps(job)), color=Ayumi.CYAN) if metsuke.validate(job): Ayumi.debug("Loaded show: {}".format(job['show'])) Ayumi.debug("Loaded episode: {}".format(job['episode'])) Ayumi.debug("Loaded filesize: {}".format(job['filesize'])) Ayumi.debug("Loaded sub type: {}".format(job['sub'])) metsuke_job = metsuke.Job( job['show'], job['episode'], job['filesize'], job['sub']) with tempfile.NamedTemporaryFile(suffix=".conf", mode="w+b") as rconf, tempfile.TemporaryDirectory() as tempdir: Ayumi.debug("Opening context managed rclone config file under path: {}.".format(rconf.name)) Ayumi.debug("Opening context managed rclone temporary directory under path: {}".format(tempdir)) rconf.write(str.encode(settings.get("RCLONE_CONFIG_FILE"))) rconf.flush() # YOU MUST FLUSH THE FILE SO RCLONE CAN READ IT! Ayumi.debug("Configurations written to temporary file. Size is {} bytes.".format(rconf.tell())) dl_sources = None up_dests = None if job['sub'].lower() == "softsub": dl_sources = settings.get("DISTRIBUTORS_RCLONE_SOFTSUB_DOWNLOAD") up_dests = settings.get("DISTRIBUTORS_RCLONE_SOFTSUB_UPLOAD") elif job['sub'].lower() == "hardsub": dl_sources = settings.get("DISTRIBUTORS_RCLONE_HARDSUB_DOWNLOAD") up_dests = settings.get("DISTRIBUTORS_RCLONE_HARDSUB_UPLOAD") Ayumi.debug("Fresh fetched download sources as: {}".format(" ".join(dl_sources))) Ayumi.debug("Fresh fetched upload sources as: {}".format(" ".join(up_dests))) try: temp_ep = shikyou.download(metsuke_job, dl_sources, tempdir, rconf.name, settings.get("RCLONE_FLAGS", "")) if temp_ep: shikyou.upload(metsuke_job, up_dests, temp_ep, rconf.name, settings.get("RCLONE_FLAGS", "")) else: Ayumi.warning("Unable to find requested job in any sources, nacking...", color=Ayumi.RED) message.nack() continue except shikyou.ShikyouResponseException: Ayumi.critical("Rclone threw an unexpected response code, rejecting.", color=Ayumi.RED) message.reject() continue except shikyou.ShikyouTimeoutException: Ayumi.warning("Rclone timed out whilhe executing, nacking.", color=Ayumi.RED) message.nack() continue Ayumi.debug("Closed context managed rclone config file.") Ayumi.debug("Closed context managed temporary directory.") else: Ayumi.warning("Received a job that Metsuke was not able to validate.", color=Ayumi.LRED) Ayumi.warning(json.dumps(job), color=Ayumi.LRED) Ayumi.info("Completed processing this message for {}".format(job['episode']), color=Ayumi.LGREEN) message.ack() except rabbitpy.exceptions.AMQPConnectionForced: Ayumi.rabbitpy_channel = None Ayumi.critical("Operator manually closed RabbitMQ connection, shutting down.", color=Ayumi.RED) # Use return for now because in some cases, calling exit() may invoke the retry() header. return