def notify(job: Job, nconf: NotifierConfigStore) -> None:
"""Notify worker"""
try:
# 1. Get the info of the show
LoggingUtils.info("[1/X] Fetching show information via Hisha...",
color=LoggingUtils.CYAN)
info = hisha.search(job.show)
# 2. Check filters
LoggingUtils.info("[2/X] Checking user list filters...",
color=LoggingUtils.CYAN)
filters = UserListFilter.check(job, info, nconf.anilist_tracker,
nconf.mal_tracker, nconf.whitelist)
if not filters:
LoggingUtils.info(
"User isn't watching this show, concluding job immediately.",
color=LoggingUtils.LYELLOW)
return False
# 3. First automata: Start sending Discord webhooks
LoggingUtils.info("[3/X] Sending Discord Webhook Notifications...",
color=LoggingUtils.CYAN)
DiscordWebhook.send(job, info, nconf.discord_webhooks)
# 4. Send POST requests
LoggingUtils.info("[4/X] Sending POST requests to endpoints...",
color=LoggingUtils.CYAN)
RestSender.send(JobUtils.to_dict(job), nconf.endpoints)
except Exception as e:
# In the event of an exception, we want to simply log it
LoggingUtils.critical(e, color=LoggingUtils.LRED)
raise e
def handle_redis_connection_error(error):
LoggingUtils.critical("It appears that Redis is down.")
response = {
"success": False,
"error": {
"type": "Redis Connection",
"message": "Redis connection error has occured."
}
}
return jsonify(response), 500
def _sig_handler(sig, frame):
LoggingUtils.critical(
"SIG command {} detected, killing all running rclone processes...".
format(sig),
color=LoggingUtils.LRED)
current_proc = psutil.Process()
children = current_proc.children(recursive=True)
for child in children:
LoggingUtils.debug(
"Killing child rclone process with PID {}".format(child.pid))
kill(child.pid, signal.SIGTERM)
raise WorkerCancelledError(helper=True)
def encode(job: Job, rconf: RcloneConfigStore,
econf: EncoderConfigStore) -> None:
"""Job worker"""
tempfolder = TempFolderController.get_temp_folder()
rclone_conf_tempfile = RcloneTempFileController.get_temp_file(rconf)
try:
# Step 1: Copy the file from rclone provided source to temp folder
LoggingUtils.info("[1/7] Starting download of episode file...",
color=LoggingUtils.LCYAN)
src_file = Rclone.download(job, econf.downloading_sources, tempfolder,
rclone_conf_tempfile,
econf.downloading_rclone_flags)
# Step 2: Prepare the file (copy over streams, populate metadata, extract subs, etc)
LoggingUtils.info(
"[2/7] Preparing episode file for hardsub and extracting subs...",
color=LoggingUtils.LCYAN)
sub1_file, sub2_file = FFmpeg.prepare(job, src_file, tempfolder)
# Step 3: Add the OpenSans-Semibold.ttf font
LoggingUtils.info("[3/7] Adding OpenSans-Semibold.ttf font...",
color=LoggingUtils.LCYAN)
FFmpeg.add_font(job, src_file, tempfolder)
# Step 4: Encode the video using the built in attachments. This also fixes the audio file
LoggingUtils.info("[4/7] Beginning hardsub encode of episode...",
color=LoggingUtils.LCYAN)
hardsub_file = FFmpeg.hardsub(job, src_file, tempfolder, sub1_file,
sub2_file)
# Step 5: Create a job for our new file
LoggingUtils.info(
"[5/7] Creating new Job instance for the hardsubbed file...",
color=LoggingUtils.LCYAN)
hardsub_job = EncodeJobGenerator.create_job_for_hardsub(
job, hardsub_file)
# Step 6: Upload the new file
LoggingUtils.info(
"[6/7] Uploading hardsubbed file to destination(s)...",
color=LoggingUtils.LCYAN)
Rclone.upload(hardsub_job, econf.uploading_destinations, hardsub_file,
rclone_conf_tempfile, econf.uploading_rclone_flags)
# Step 7: Send POST requests
LoggingUtils.info("[7/7] Sending POST requests to endpoints...",
color=LoggingUtils.LCYAN)
RestSender.send(JobUtils.to_dict(hardsub_job), econf.endpoints)
# Finally, destroy the temp folder and files
TempFolderController.destroy_temp_folder()
RcloneTempFileController.destroy_temp_file()
except RcloneError as re:
LoggingUtils.critical(re.message, color=LoggingUtils.LRED)
LoggingUtils.critical("S/D: {} to {}".format(re.source, re.dest),
color=LoggingUtils.LRED)
LoggingUtils.critical(re.output, color=LoggingUtils.RED)
# In any case, delete the temp folder
TempFolderController.destroy_temp_folder()
RcloneTempFileController.destroy_temp_file()
# Reraise - this will clutter up the logs but make it visible in RQ-dashboard
raise re
except FFmpegError as fe:
LoggingUtils.critical(fe.message, color=LoggingUtils.LRED)
LoggingUtils.critical(fe.output, color=LoggingUtils.RED)
# In any case, delete the temp folder
TempFolderController.destroy_temp_folder()
RcloneTempFileController.destroy_temp_file()
# Reraise - this will clutter up the logs but make it visible in RQ-dashboard
raise fe
except WorkerCancelledError as we:
LoggingUtils.critical(we.message, color=LoggingUtils.LRED)
TempFolderController.destroy_temp_folder()
RcloneTempFileController.destroy_temp_file()
# Reraise for dashboard
raise we
except Exception as e:
# In the event of an exception, we want to simply log it
LoggingUtils.critical(e, color=LoggingUtils.LRED)
TempFolderController.destroy_temp_folder()
RcloneTempFileController.destroy_temp_file()
raise e
def distribute(job: Job, rconf: RcloneConfigStore,
dconf: DistributorConfigStore) -> None:
"""Job worker"""
tempfolder = TempFolderController.get_temp_folder()
rclone_conf_tempfile = RcloneTempFileController.get_temp_file(rconf)
try:
# Step 1: Check if we should even download the show
LoggingUtils.info("[1/X] Fetching show information via Hisha...",
color=LoggingUtils.CYAN)
info = hisha.search(job.show)
# Step 2: Check filter
LoggingUtils.info("[2/X] Checking user list filters...",
color=LoggingUtils.CYAN)
filters = UserListFilter.check(job, info, dconf.anilist_tracker,
dconf.mal_tracker, dconf.whitelist)
if not filters:
LoggingUtils.info(
"User isn't watching this show, concluding job immediately.",
color=LoggingUtils.LYELLOW)
return False
# Step 3: Download the file
LoggingUtils.info("[3/5] Starting download of episode file...",
color=LoggingUtils.LCYAN)
sources = None
flags = None
if job.sub.lower() == "softsub":
LoggingUtils.info(
"Softsub mode detected, loading softsub download configs",
color=LoggingUtils.CYAN)
sources = dconf.softsub_downloading_sources
flags = dconf.softsub_downloading_rclone_flags
elif job.sub.lower() == "hardsub":
LoggingUtils.info(
"Hardsub mode detected, loading hardsub download configs",
color=LoggingUtils.CYAN)
sources = dconf.hardsub_downloading_sources
flags = dconf.hardsub_downloading_rclone_flags
else:
raise JobSubTypeError(job, "Unknown sub type {}".format(job.sub))
src_file = Rclone.download(job, sources, tempfolder,
rclone_conf_tempfile, flags)
# Step 4: Upload it elsewhere
LoggingUtils.info(
"[4/5] Uploading hardsubbed file to destination(s)...",
color=LoggingUtils.LCYAN)
destinations = None
flags = None
if job.sub.lower() == "softsub":
LoggingUtils.info(
"Softsub mode detected, loading softsub upload configs",
color=LoggingUtils.CYAN)
destinations = dconf.softsub_uploading_destinations
flags = dconf.softsub_uploading_rclone_flags
elif job.sub.lower() == "hardsub":
LoggingUtils.info(
"Hardsub mode detected, loading hardsub upload configs",
color=LoggingUtils.CYAN)
destinations = dconf.hardsub_uploading_destinations
flags = dconf.hardsub_uploading_rclone_flags
else:
raise JobSubTypeError(job, "Unknown sub type {}".format(job.sub))
Rclone.upload(job, destinations, src_file, rclone_conf_tempfile, flags)
# Step 5: Send POST requests
LoggingUtils.info("[5/5] Sending POST requests to endpoints...",
color=LoggingUtils.LCYAN)
RestSender.send(JobUtils.to_dict(job), dconf.endpoints)
# Finally, destroy the temp folder
TempFolderController.destroy_temp_folder()
RcloneTempFileController.destroy_temp_file()
except RcloneError as re:
LoggingUtils.critical(re.message, color=LoggingUtils.LRED)
LoggingUtils.critical("S/D: {} to {}".format(re.source, re.dest),
color=LoggingUtils.LRED)
LoggingUtils.critical(re.output, color=LoggingUtils.RED)
# In any case, delete the temp folder
TempFolderController.destroy_temp_folder()
RcloneTempFileController.destroy_temp_file()
# Reraise - this will clutter up the logs but make it visible in RQ-dashboard
raise re
except JobSubTypeError as jste:
LoggingUtils.critical(jste.message, color=LoggingUtils.LRED)
LoggingUtils.critical("Job: {}".format(jste.job),
color=LoggingUtils.LRED)
TempFolderController.destroy_temp_folder()
RcloneTempFileController.destroy_temp_file()
raise jste
except Exception as e:
# In the event of an exception, we want to simply log it
LoggingUtils.critical(e, color=LoggingUtils.LRED)
TempFolderController.destroy_temp_folder()
RcloneTempFileController.destroy_temp_file()
raise e
qs = sys.argv[1:]
qs = [q for q in qs if q] # Remove empty elements
LoggingUtils.info("*** Listening on {}...".format(', '.join(qs)),
color=LoggingUtils.LGREEN)
while True:
with Connection():
try:
redis_conn = Redis(host=WorkerConf.redis_host,
port=WorkerConf.redis_port,
password=WorkerConf.redis_password,
socket_keepalive=True,
socket_timeout=180,
health_check_interval=60)
w = Worker(qs, connection=redis_conn, name=WORKER_NAME)
w.work()
except RedisConnectionError as rce:
LoggingUtils.critical(
"Lost connection to Redis instance, shutting down.",
color=LoggingUtils.LRED)
sys.exit()
except WorkerCancelledError:
LoggingUtils.warning("Worker killed externally, shutting down...")
sys.exit()
except TimeoutError:
# We expect a timeout error to occur as this forces the worker to reregister
# Silently handle and let the loop continue
# LoggingUtils.debug("Timeout error caught, handling silently...")
pass
def sig_handler(sig, frame):
LoggingUtils.critical("SIG command {} detected, exiting...".format(sig),
color=LoggingUtils.LRED)
sys.exit()
