def download(job: Job, sources: List[str], tempfolder: str, config: str,
flags: str) -> str:
"""
Download the provided episode from sources
Returns the path of the downloaded file
job: Job to do!
sources: list of rclone sources (EncoderConf.downloading_sources)
tmppath: Path of the temporary folder
rclone_config: Path to the rclone config file
flags: rclone flags
"""
for source in sources:
Ayumi.debug("Checking for existence from source: {}".format(source))
if _check_exists(config, source, job):
Ayumi.info(
"Now downloading episode from source: {}".format(source))
src_file = "{}/{}/{}".format(_clean(source), job.show, job.episode)
Ayumi.debug("Sourcing from rclone path: {}".format(src_file))
dest_file = "{}/{}".format(_clean(tempfolder), "temp")
Ayumi.debug("Downloading to destination: {}".format(dest_file))
command = [
"rclone", "--config={}".format(config), "copyto", src_file,
dest_file
]
command.extend(flags.split())
Ayumi.debug("Now running command: {}".format(" ".join(command)))
Ayumi.info("Now starting download.", color=Ayumi.LCYAN)
try:
_run(command)
except ShikyouResponseException:
Ayumi.error(
"Rclone command returned a bad return code, contact the developer.",
color=Ayumi.LRED)
raise ShikyouResponseException()
except ShikyouTimeoutException:
Ayumi.error("Rclone command timed out, are your sources okay?",
color=Ayumi.LRED)
raise ShikyouTimeoutException()
Ayumi.info("Completed downloading files.", color=Ayumi.LGREEN)
return dest_file
else:
Ayumi.debug(
"Requested episode doesn't exist under source {}".format(
source))
Ayumi.warning("No download sources contained the file.")
return None
def _kishi_list(user):
"""
Helper method to get all of a user's anime list.
Params:
user: String, username of Anilist user
Returns: A tuple of three lists.
The first list is all the Watching
The second list is all the PTW
The third list is all the Paused
Throws an exception if anything goes wrong. This should be caught by any method using this.
"""
watching = list()
paused = list()
ptw = list()
# Anilist API is much nicer to play with.
try:
# Make the request to Anilist, and pass in the userName as the user query
anilist_res = requests.post(_ANILIST_API_URL,
json={
'query': _USER_ANIME_QUERY,
'variables': {
'userName': user
}
})
if anilist_res.status_code != 200:
Ayumi.error(
"Anilist returned a bad status code when attempting to get {}'s lists"
.format(user))
raise Exception()
try:
anilist_res_json = anilist_res.json()
except:
Ayumi.error(
"Anilist returned a response that was not parseable into JSON")
raise Exception()
watching = _add_list_entries("Watching", anilist_res_json)
paused = _add_list_entries("Paused", anilist_res_json)
ptw = _add_list_entries("Planning", anilist_res_json)
except:
Ayumi.critical("Kishi was unable to properly contact Anilist")
raise Exception()
return (watching, paused, ptw)
def upload(job: Job, destinations: List[str], upload_file: str, config: str,
flags: str) -> None:
"""
Upload the completed new hardsub file into the rclone destinations
Returns a boolean based on success
job: Job to do! This is the job of the HARDSUB file
destinations: list of rlcone destinations (e.g., EncoderConf.uploading_destinations)
upload_file: Path to the file to be uploaded
rclone_config: Path to the rclone config file
flags: rclone flag
This method will upload the file and include its show name:
e.g., 'temp.mp4' --> destination/show/episode.mp4
"""
for dest in destinations:
rclone_dest = "{}/{}/{}".format(_clean(dest), job.show, job.episode)
command = [
"rclone", "--config={}".format(config), "copyto", upload_file,
rclone_dest
]
command.extend(flags.split())
Ayumi.debug("Now running command: {}".format(" ".join(command)))
Ayumi.info("Now uploading file to {}".format(dest))
try:
_run(command)
except ShikyouResponseException:
Ayumi.error(
"Rclone command returned a bad return code, contact the developer.",
color=Ayumi.LRED)
raise ShikyouResponseException()
except ShikyouTimeoutException:
Ayumi.error("Rclone command timed out, are your sources okay?",
color=Ayumi.LRED)
raise ShikyouTimeoutException()
Ayumi.info("Completed uploading files.", color=Ayumi.LGREEN)
def bittorrent():
try:
with rabbitpy.Connection('amqp://{username}:{password}@{host}:{port}/{vhost}'.format(
username=settings.get('RABBITMQ_USERNAME'),
password=settings.get('RABBITMQ_PASSWORD'),
host=settings.get('RABBITMQ_HOST'),
port=settings.get('RABBITMQ_PORT'),
vhost=settings.get('RABBITMQ_VHOST')
)) as conn:
with conn.channel() as channel:
Ayumi.set_rabbitpy_channel(channel)
channel.enable_publisher_confirms()
queue_name = settings.get('ACQUISITION_BITTORRENT_QUEUE')
Ayumi.debug("Connecting to queue: {}".format(queue_name))
queue = rabbitpy.Queue(channel, queue_name)
queue.declare(passive=True)
Ayumi.info('Now listening for messages on queue: {}...'.format(
queue_name), color=Ayumi.LYELLOW)
for message in queue.consume(prefetch=1):
Ayumi.info(
"Received new message, starting...", color=Ayumi.CYAN)
feeditem_preprocess = _load_amqp_message_body(message)
Ayumi.debug('Loaded message raw: "{}"'.format(
feeditem_preprocess))
if not feeditem_preprocess or not metsuke.validate_feeditem(feeditem_preprocess):
Ayumi.error('Invalid message received, rejecting. Output: "{}"'.format(
feeditem_preprocess), color=Ayumi.RED)
message.reject()
continue
# Load initial data
feeditem: metsuke.FeedItem = metsuke.generate_feeditem(
feeditem_preprocess)
shows_map = _load_shows_map()
overload_title = feeditem.show_title
Ayumi.info(
'Setting overload title: "{}"'.format(overload_title))
# If there is a central override, use it instead.
if _strip_title(anitopy.parse(feeditem.title)['anime_title']) in shows_map:
central_overload_title = shows_map[_strip_title(
feeditem.title)]
Ayumi.info('Overwriting overload title with central overload title: "{}"'.format(
central_overload_title))
overload_title = central_overload_title
with tempfile.TemporaryDirectory() as temp_dir:
Ayumi.debug(
'Created temporary directory under path: "{}"'.format(temp_dir))
# Download the episode
try:
res = subprocess.run(
[
"aria2c",
"--seed-time=0",
"--rpc-save-upload-metadata=false",
"--bt-save-metadata=false",
"--dir={}".format(temp_dir),
feeditem.link
]
)
if res.returncode != 0:
Ayumi.warning(
"Aria2 did not return a 0 exit code, assuming download errored and nacking.", color=Ayumi.RED)
message.nack()
continue
except subprocess.TimeoutExpired:
Ayumi.warning(
"Download via webtorrent timed out - nacking.", color=Ayumi.RED)
message.nack()
continue
if res.returncode != 0:
Ayumi.warning(
"Webtorrent did not have a return code of 0, nacking.", color=Ayumi.RED)
message.nack()
continue
# Rename it
potential_files = [f for f in os.listdir(
temp_dir) if f.endswith(".mkv")]
Ayumi.debug(
"Loaded potential files: {}".format(potential_files))
if len(potential_files) != 1:
Ayumi.warning(
"Found more than one .mkv file, rejecting this job.", color=Ayumi.RED)
message.reject()
continue
dl_file = potential_files[0]
Ayumi.info('Found file: "{}"'.format(dl_file))
dl_file_path = os.path.abspath(
'{}/{}'.format(_clean_title(temp_dir), potential_files[0]))
Ayumi.debug(
'dl_file_path: "{}"'.format(dl_file_path))
# Remove unneeded files
# TODO: THIS IS A HOTFIX, CHANGE LOGIC IN B2
bad_files = [f for f in os.listdir(
temp_dir) if not f.endswith(".mkv")]
Ayumi.debug("Found bad files: {}".format(bad_files))
for bf in bad_files:
try:
Ayumi.debug("Removing bad file: {}".format(bf))
os.remove(
'{}/{}'.format(_clean_title(temp_dir), bf))
except:
Ayumi.debug("Removing bad tree: {}".format(bf))
shutil.rmtree(
'{}/{}'.format(_clean_title(temp_dir), bf))
# Move the file to proper layout with updated name
dl_file_new_name = _generate_new_filename(dl_file)
Ayumi.info('Generated new episode name: "{}"'.format(
dl_file_new_name))
dl_file_new_dir = "{}/{}".format(
temp_dir, overload_title)
Ayumi.debug(
'dl_file_new_dir: "{}"'.format(dl_file_new_dir))
dl_file_new_path = "{}/{}".format(
dl_file_new_dir, dl_file_new_name)
Ayumi.debug(
'dl_file_new_path: "{}"'.format(
dl_file_new_path))
Ayumi.debug('Moving "{}" to "{}"'.format(
dl_file_path, dl_file_new_path))
os.mkdir(dl_file_new_dir)
shutil.move(dl_file_path, dl_file_new_path)
# Upload the file to rclone destination
with tempfile.NamedTemporaryFile(suffix=".conf", mode="w+b") as rconf:
rconf.write(str.encode(
settings.get("RCLONE_CONFIG_FILE")))
rconf.flush()
Ayumi.debug(
'Created temporary rclone file under path: "{}"'.format(rconf.name))
rclone_dest = _clean_title(settings.get(
"ACQUISITION_BITTORRENT_RCLONE_DEST"))
rclone_flags = settings.get("RCLONE_FLAGS", "")
command = [
"rclone", "--config={}".format(rconf.name), "copy", temp_dir, rclone_dest]
command.extend(rclone_flags.split())
Ayumi.debug(
'Rclone command to be run: "{}"'.format(command))
try:
Ayumi.info(
'Now uploading new blob to: "{}"'.format(rclone_dest))
rclone_res = subprocess.run(
command, timeout=3600)
if rclone_res.returncode != 0:
Ayumi.warning('Rclone returned non-zero code of {}, nacking.'.format(
rclone_res.returncode), color=Ayumi.LRED)
message.nack()
except subprocess.TimeoutExpired:
Ayumi.warning(
'Rclone upload timed out, nacking.', color=Ayumi.LRED)
message.nack()
continue
# Fetch information on the file to create a job
new_message = rabbitpy.Message(channel, dumps(
{
"show": overload_title,
"episode": dl_file_new_name,
"filesize": int(os.path.getsize(dl_file_new_path)),
"sub": "SOFTSUB"
}
))
acquisition_bittorrent_exchange_name = settings.get(
'ACQUISITION_BITTORRENT_EXCHANGE')
Ayumi.info('Sending to exchange: "{}"'.format(
acquisition_bittorrent_exchange_name), color=Ayumi.CYAN)
while not new_message.publish(acquisition_bittorrent_exchange_name, mandatory=True):
Ayumi.warning(
"Failed to publish feed item, trying again in 60 seconds")
sleep(60)
Ayumi.info("Published feed item with title: " +
overload_title, color=Ayumi.LGREEN)
message.ack()
except rabbitpy.exceptions.AMQPConnectionForced:
Ayumi.warning(
"Operator manually closed RabbitMQ connection, shutting down.", color=Ayumi.LYELLOW)
return
