def notify(job: Job, nconf: NotifierConfigStore) -> None: """Notify worker""" try: # 1. Get the info of the show LoggingUtils.info("[1/X] Fetching show information via Hisha...", color=LoggingUtils.CYAN) info = hisha.search(job.show) # 2. Check filters LoggingUtils.info("[2/X] Checking user list filters...", color=LoggingUtils.CYAN) filters = UserListFilter.check(job, info, nconf.anilist_tracker, nconf.mal_tracker, nconf.whitelist) if not filters: LoggingUtils.info( "User isn't watching this show, concluding job immediately.", color=LoggingUtils.LYELLOW) return False # 3. First automata: Start sending Discord webhooks LoggingUtils.info("[3/X] Sending Discord Webhook Notifications...", color=LoggingUtils.CYAN) DiscordWebhook.send(job, info, nconf.discord_webhooks) # 4. Send POST requests LoggingUtils.info("[4/X] Sending POST requests to endpoints...", color=LoggingUtils.CYAN) RestSender.send(JobUtils.to_dict(job), nconf.endpoints) except Exception as e: # In the event of an exception, we want to simply log it LoggingUtils.critical(e, color=LoggingUtils.LRED) raise e
def handle_redis_connection_error(error): LoggingUtils.critical("It appears that Redis is down.") response = { "success": False, "error": { "type": "Redis Connection", "message": "Redis connection error has occured." } } return jsonify(response), 500
def _sig_handler(sig, frame): LoggingUtils.critical( "SIG command {} detected, killing all running rclone processes...". format(sig), color=LoggingUtils.LRED) current_proc = psutil.Process() children = current_proc.children(recursive=True) for child in children: LoggingUtils.debug( "Killing child rclone process with PID {}".format(child.pid)) kill(child.pid, signal.SIGTERM) raise WorkerCancelledError(helper=True)
def encode(job: Job, rconf: RcloneConfigStore, econf: EncoderConfigStore) -> None: """Job worker""" tempfolder = TempFolderController.get_temp_folder() rclone_conf_tempfile = RcloneTempFileController.get_temp_file(rconf) try: # Step 1: Copy the file from rclone provided source to temp folder LoggingUtils.info("[1/7] Starting download of episode file...", color=LoggingUtils.LCYAN) src_file = Rclone.download(job, econf.downloading_sources, tempfolder, rclone_conf_tempfile, econf.downloading_rclone_flags) # Step 2: Prepare the file (copy over streams, populate metadata, extract subs, etc) LoggingUtils.info( "[2/7] Preparing episode file for hardsub and extracting subs...", color=LoggingUtils.LCYAN) sub1_file, sub2_file = FFmpeg.prepare(job, src_file, tempfolder) # Step 3: Add the OpenSans-Semibold.ttf font LoggingUtils.info("[3/7] Adding OpenSans-Semibold.ttf font...", color=LoggingUtils.LCYAN) FFmpeg.add_font(job, src_file, tempfolder) # Step 4: Encode the video using the built in attachments. This also fixes the audio file LoggingUtils.info("[4/7] Beginning hardsub encode of episode...", color=LoggingUtils.LCYAN) hardsub_file = FFmpeg.hardsub(job, src_file, tempfolder, sub1_file, sub2_file) # Step 5: Create a job for our new file LoggingUtils.info( "[5/7] Creating new Job instance for the hardsubbed file...", color=LoggingUtils.LCYAN) hardsub_job = EncodeJobGenerator.create_job_for_hardsub( job, hardsub_file) # Step 6: Upload the new file LoggingUtils.info( "[6/7] Uploading hardsubbed file to destination(s)...", color=LoggingUtils.LCYAN) Rclone.upload(hardsub_job, econf.uploading_destinations, hardsub_file, rclone_conf_tempfile, econf.uploading_rclone_flags) # Step 7: Send POST requests LoggingUtils.info("[7/7] Sending POST requests to endpoints...", color=LoggingUtils.LCYAN) RestSender.send(JobUtils.to_dict(hardsub_job), econf.endpoints) # Finally, destroy the temp folder and files TempFolderController.destroy_temp_folder() RcloneTempFileController.destroy_temp_file() except RcloneError as re: LoggingUtils.critical(re.message, color=LoggingUtils.LRED) LoggingUtils.critical("S/D: {} to {}".format(re.source, re.dest), color=LoggingUtils.LRED) LoggingUtils.critical(re.output, color=LoggingUtils.RED) # In any case, delete the temp folder TempFolderController.destroy_temp_folder() RcloneTempFileController.destroy_temp_file() # Reraise - this will clutter up the logs but make it visible in RQ-dashboard raise re except FFmpegError as fe: LoggingUtils.critical(fe.message, color=LoggingUtils.LRED) LoggingUtils.critical(fe.output, color=LoggingUtils.RED) # In any case, delete the temp folder TempFolderController.destroy_temp_folder() RcloneTempFileController.destroy_temp_file() # Reraise - this will clutter up the logs but make it visible in RQ-dashboard raise fe except WorkerCancelledError as we: LoggingUtils.critical(we.message, color=LoggingUtils.LRED) TempFolderController.destroy_temp_folder() RcloneTempFileController.destroy_temp_file() # Reraise for dashboard raise we except Exception as e: # In the event of an exception, we want to simply log it LoggingUtils.critical(e, color=LoggingUtils.LRED) TempFolderController.destroy_temp_folder() RcloneTempFileController.destroy_temp_file() raise e
def distribute(job: Job, rconf: RcloneConfigStore, dconf: DistributorConfigStore) -> None: """Job worker""" tempfolder = TempFolderController.get_temp_folder() rclone_conf_tempfile = RcloneTempFileController.get_temp_file(rconf) try: # Step 1: Check if we should even download the show LoggingUtils.info("[1/X] Fetching show information via Hisha...", color=LoggingUtils.CYAN) info = hisha.search(job.show) # Step 2: Check filter LoggingUtils.info("[2/X] Checking user list filters...", color=LoggingUtils.CYAN) filters = UserListFilter.check(job, info, dconf.anilist_tracker, dconf.mal_tracker, dconf.whitelist) if not filters: LoggingUtils.info( "User isn't watching this show, concluding job immediately.", color=LoggingUtils.LYELLOW) return False # Step 3: Download the file LoggingUtils.info("[3/5] Starting download of episode file...", color=LoggingUtils.LCYAN) sources = None flags = None if job.sub.lower() == "softsub": LoggingUtils.info( "Softsub mode detected, loading softsub download configs", color=LoggingUtils.CYAN) sources = dconf.softsub_downloading_sources flags = dconf.softsub_downloading_rclone_flags elif job.sub.lower() == "hardsub": LoggingUtils.info( "Hardsub mode detected, loading hardsub download configs", color=LoggingUtils.CYAN) sources = dconf.hardsub_downloading_sources flags = dconf.hardsub_downloading_rclone_flags else: raise JobSubTypeError(job, "Unknown sub type {}".format(job.sub)) src_file = Rclone.download(job, sources, tempfolder, rclone_conf_tempfile, flags) # Step 4: Upload it elsewhere LoggingUtils.info( "[4/5] Uploading hardsubbed file to destination(s)...", color=LoggingUtils.LCYAN) destinations = None flags = None if job.sub.lower() == "softsub": LoggingUtils.info( "Softsub mode detected, loading softsub upload configs", color=LoggingUtils.CYAN) destinations = dconf.softsub_uploading_destinations flags = dconf.softsub_uploading_rclone_flags elif job.sub.lower() == "hardsub": LoggingUtils.info( "Hardsub mode detected, loading hardsub upload configs", color=LoggingUtils.CYAN) destinations = dconf.hardsub_uploading_destinations flags = dconf.hardsub_uploading_rclone_flags else: raise JobSubTypeError(job, "Unknown sub type {}".format(job.sub)) Rclone.upload(job, destinations, src_file, rclone_conf_tempfile, flags) # Step 5: Send POST requests LoggingUtils.info("[5/5] Sending POST requests to endpoints...", color=LoggingUtils.LCYAN) RestSender.send(JobUtils.to_dict(job), dconf.endpoints) # Finally, destroy the temp folder TempFolderController.destroy_temp_folder() RcloneTempFileController.destroy_temp_file() except RcloneError as re: LoggingUtils.critical(re.message, color=LoggingUtils.LRED) LoggingUtils.critical("S/D: {} to {}".format(re.source, re.dest), color=LoggingUtils.LRED) LoggingUtils.critical(re.output, color=LoggingUtils.RED) # In any case, delete the temp folder TempFolderController.destroy_temp_folder() RcloneTempFileController.destroy_temp_file() # Reraise - this will clutter up the logs but make it visible in RQ-dashboard raise re except JobSubTypeError as jste: LoggingUtils.critical(jste.message, color=LoggingUtils.LRED) LoggingUtils.critical("Job: {}".format(jste.job), color=LoggingUtils.LRED) TempFolderController.destroy_temp_folder() RcloneTempFileController.destroy_temp_file() raise jste except Exception as e: # In the event of an exception, we want to simply log it LoggingUtils.critical(e, color=LoggingUtils.LRED) TempFolderController.destroy_temp_folder() RcloneTempFileController.destroy_temp_file() raise e
qs = sys.argv[1:] qs = [q for q in qs if q] # Remove empty elements LoggingUtils.info("*** Listening on {}...".format(', '.join(qs)), color=LoggingUtils.LGREEN) while True: with Connection(): try: redis_conn = Redis(host=WorkerConf.redis_host, port=WorkerConf.redis_port, password=WorkerConf.redis_password, socket_keepalive=True, socket_timeout=180, health_check_interval=60) w = Worker(qs, connection=redis_conn, name=WORKER_NAME) w.work() except RedisConnectionError as rce: LoggingUtils.critical( "Lost connection to Redis instance, shutting down.", color=LoggingUtils.LRED) sys.exit() except WorkerCancelledError: LoggingUtils.warning("Worker killed externally, shutting down...") sys.exit() except TimeoutError: # We expect a timeout error to occur as this forces the worker to reregister # Silently handle and let the loop continue # LoggingUtils.debug("Timeout error caught, handling silently...") pass
def sig_handler(sig, frame): LoggingUtils.critical("SIG command {} detected, exiting...".format(sig), color=LoggingUtils.LRED) sys.exit()