def download(job: Job, sources: List[str], tempfolder: str, config: str, flags: str) -> str: """ Download the provided episode from sources Returns the path of the downloaded file job: Job to do! sources: list of rclone sources (EncoderConf.downloading_sources) tmppath: Path of the temporary folder rclone_config: Path to the rclone config file flags: rclone flags """ for source in sources: Ayumi.debug("Checking for existence from source: {}".format(source)) if _check_exists(config, source, job): Ayumi.info( "Now downloading episode from source: {}".format(source)) src_file = "{}/{}/{}".format(_clean(source), job.show, job.episode) Ayumi.debug("Sourcing from rclone path: {}".format(src_file)) dest_file = "{}/{}".format(_clean(tempfolder), "temp") Ayumi.debug("Downloading to destination: {}".format(dest_file)) command = [ "rclone", "--config={}".format(config), "copyto", src_file, dest_file ] command.extend(flags.split()) Ayumi.debug("Now running command: {}".format(" ".join(command))) Ayumi.info("Now starting download.", color=Ayumi.LCYAN) try: _run(command) except ShikyouResponseException: Ayumi.error( "Rclone command returned a bad return code, contact the developer.", color=Ayumi.LRED) raise ShikyouResponseException() except ShikyouTimeoutException: Ayumi.error("Rclone command timed out, are your sources okay?", color=Ayumi.LRED) raise ShikyouTimeoutException() Ayumi.info("Completed downloading files.", color=Ayumi.LGREEN) return dest_file else: Ayumi.debug( "Requested episode doesn't exist under source {}".format( source)) Ayumi.warning("No download sources contained the file.") return None
def _kishi_list(user): """ Helper method to get all of a user's anime list. Params: user: String, username of Anilist user Returns: A tuple of three lists. The first list is all the Watching The second list is all the PTW The third list is all the Paused Throws an exception if anything goes wrong. This should be caught by any method using this. """ watching = list() paused = list() ptw = list() # Anilist API is much nicer to play with. try: # Make the request to Anilist, and pass in the userName as the user query anilist_res = requests.post(_ANILIST_API_URL, json={ 'query': _USER_ANIME_QUERY, 'variables': { 'userName': user } }) if anilist_res.status_code != 200: Ayumi.error( "Anilist returned a bad status code when attempting to get {}'s lists" .format(user)) raise Exception() try: anilist_res_json = anilist_res.json() except: Ayumi.error( "Anilist returned a response that was not parseable into JSON") raise Exception() watching = _add_list_entries("Watching", anilist_res_json) paused = _add_list_entries("Paused", anilist_res_json) ptw = _add_list_entries("Planning", anilist_res_json) except: Ayumi.critical("Kishi was unable to properly contact Anilist") raise Exception() return (watching, paused, ptw)
def upload(job: Job, destinations: List[str], upload_file: str, config: str, flags: str) -> None: """ Upload the completed new hardsub file into the rclone destinations Returns a boolean based on success job: Job to do! This is the job of the HARDSUB file destinations: list of rlcone destinations (e.g., EncoderConf.uploading_destinations) upload_file: Path to the file to be uploaded rclone_config: Path to the rclone config file flags: rclone flag This method will upload the file and include its show name: e.g., 'temp.mp4' --> destination/show/episode.mp4 """ for dest in destinations: rclone_dest = "{}/{}/{}".format(_clean(dest), job.show, job.episode) command = [ "rclone", "--config={}".format(config), "copyto", upload_file, rclone_dest ] command.extend(flags.split()) Ayumi.debug("Now running command: {}".format(" ".join(command))) Ayumi.info("Now uploading file to {}".format(dest)) try: _run(command) except ShikyouResponseException: Ayumi.error( "Rclone command returned a bad return code, contact the developer.", color=Ayumi.LRED) raise ShikyouResponseException() except ShikyouTimeoutException: Ayumi.error("Rclone command timed out, are your sources okay?", color=Ayumi.LRED) raise ShikyouTimeoutException() Ayumi.info("Completed uploading files.", color=Ayumi.LGREEN)
def bittorrent(): try: with rabbitpy.Connection('amqp://{username}:{password}@{host}:{port}/{vhost}'.format( username=settings.get('RABBITMQ_USERNAME'), password=settings.get('RABBITMQ_PASSWORD'), host=settings.get('RABBITMQ_HOST'), port=settings.get('RABBITMQ_PORT'), vhost=settings.get('RABBITMQ_VHOST') )) as conn: with conn.channel() as channel: Ayumi.set_rabbitpy_channel(channel) channel.enable_publisher_confirms() queue_name = settings.get('ACQUISITION_BITTORRENT_QUEUE') Ayumi.debug("Connecting to queue: {}".format(queue_name)) queue = rabbitpy.Queue(channel, queue_name) queue.declare(passive=True) Ayumi.info('Now listening for messages on queue: {}...'.format( queue_name), color=Ayumi.LYELLOW) for message in queue.consume(prefetch=1): Ayumi.info( "Received new message, starting...", color=Ayumi.CYAN) feeditem_preprocess = _load_amqp_message_body(message) Ayumi.debug('Loaded message raw: "{}"'.format( feeditem_preprocess)) if not feeditem_preprocess or not metsuke.validate_feeditem(feeditem_preprocess): Ayumi.error('Invalid message received, rejecting. Output: "{}"'.format( feeditem_preprocess), color=Ayumi.RED) message.reject() continue # Load initial data feeditem: metsuke.FeedItem = metsuke.generate_feeditem( feeditem_preprocess) shows_map = _load_shows_map() overload_title = feeditem.show_title Ayumi.info( 'Setting overload title: "{}"'.format(overload_title)) # If there is a central override, use it instead. if _strip_title(anitopy.parse(feeditem.title)['anime_title']) in shows_map: central_overload_title = shows_map[_strip_title( feeditem.title)] Ayumi.info('Overwriting overload title with central overload title: "{}"'.format( central_overload_title)) overload_title = central_overload_title with tempfile.TemporaryDirectory() as temp_dir: Ayumi.debug( 'Created temporary directory under path: "{}"'.format(temp_dir)) # Download the episode try: res = subprocess.run( [ "aria2c", "--seed-time=0", "--rpc-save-upload-metadata=false", "--bt-save-metadata=false", "--dir={}".format(temp_dir), feeditem.link ] ) if res.returncode != 0: Ayumi.warning( "Aria2 did not return a 0 exit code, assuming download errored and nacking.", color=Ayumi.RED) message.nack() continue except subprocess.TimeoutExpired: Ayumi.warning( "Download via webtorrent timed out - nacking.", color=Ayumi.RED) message.nack() continue if res.returncode != 0: Ayumi.warning( "Webtorrent did not have a return code of 0, nacking.", color=Ayumi.RED) message.nack() continue # Rename it potential_files = [f for f in os.listdir( temp_dir) if f.endswith(".mkv")] Ayumi.debug( "Loaded potential files: {}".format(potential_files)) if len(potential_files) != 1: Ayumi.warning( "Found more than one .mkv file, rejecting this job.", color=Ayumi.RED) message.reject() continue dl_file = potential_files[0] Ayumi.info('Found file: "{}"'.format(dl_file)) dl_file_path = os.path.abspath( '{}/{}'.format(_clean_title(temp_dir), potential_files[0])) Ayumi.debug( 'dl_file_path: "{}"'.format(dl_file_path)) # Remove unneeded files # TODO: THIS IS A HOTFIX, CHANGE LOGIC IN B2 bad_files = [f for f in os.listdir( temp_dir) if not f.endswith(".mkv")] Ayumi.debug("Found bad files: {}".format(bad_files)) for bf in bad_files: try: Ayumi.debug("Removing bad file: {}".format(bf)) os.remove( '{}/{}'.format(_clean_title(temp_dir), bf)) except: Ayumi.debug("Removing bad tree: {}".format(bf)) shutil.rmtree( '{}/{}'.format(_clean_title(temp_dir), bf)) # Move the file to proper layout with updated name dl_file_new_name = _generate_new_filename(dl_file) Ayumi.info('Generated new episode name: "{}"'.format( dl_file_new_name)) dl_file_new_dir = "{}/{}".format( temp_dir, overload_title) Ayumi.debug( 'dl_file_new_dir: "{}"'.format(dl_file_new_dir)) dl_file_new_path = "{}/{}".format( dl_file_new_dir, dl_file_new_name) Ayumi.debug( 'dl_file_new_path: "{}"'.format( dl_file_new_path)) Ayumi.debug('Moving "{}" to "{}"'.format( dl_file_path, dl_file_new_path)) os.mkdir(dl_file_new_dir) shutil.move(dl_file_path, dl_file_new_path) # Upload the file to rclone destination with tempfile.NamedTemporaryFile(suffix=".conf", mode="w+b") as rconf: rconf.write(str.encode( settings.get("RCLONE_CONFIG_FILE"))) rconf.flush() Ayumi.debug( 'Created temporary rclone file under path: "{}"'.format(rconf.name)) rclone_dest = _clean_title(settings.get( "ACQUISITION_BITTORRENT_RCLONE_DEST")) rclone_flags = settings.get("RCLONE_FLAGS", "") command = [ "rclone", "--config={}".format(rconf.name), "copy", temp_dir, rclone_dest] command.extend(rclone_flags.split()) Ayumi.debug( 'Rclone command to be run: "{}"'.format(command)) try: Ayumi.info( 'Now uploading new blob to: "{}"'.format(rclone_dest)) rclone_res = subprocess.run( command, timeout=3600) if rclone_res.returncode != 0: Ayumi.warning('Rclone returned non-zero code of {}, nacking.'.format( rclone_res.returncode), color=Ayumi.LRED) message.nack() except subprocess.TimeoutExpired: Ayumi.warning( 'Rclone upload timed out, nacking.', color=Ayumi.LRED) message.nack() continue # Fetch information on the file to create a job new_message = rabbitpy.Message(channel, dumps( { "show": overload_title, "episode": dl_file_new_name, "filesize": int(os.path.getsize(dl_file_new_path)), "sub": "SOFTSUB" } )) acquisition_bittorrent_exchange_name = settings.get( 'ACQUISITION_BITTORRENT_EXCHANGE') Ayumi.info('Sending to exchange: "{}"'.format( acquisition_bittorrent_exchange_name), color=Ayumi.CYAN) while not new_message.publish(acquisition_bittorrent_exchange_name, mandatory=True): Ayumi.warning( "Failed to publish feed item, trying again in 60 seconds") sleep(60) Ayumi.info("Published feed item with title: " + overload_title, color=Ayumi.LGREEN) message.ack() except rabbitpy.exceptions.AMQPConnectionForced: Ayumi.warning( "Operator manually closed RabbitMQ connection, shutting down.", color=Ayumi.LYELLOW) return