def resume_threads(self): if self.threads: self.logger.debug(whoami() + "Resuming threads") for t, _ in self.threads: t.paused = False else: self.logger.debug(whoami() + "Starting threads") self.start_threads()
def sighandler_unrar(self, a, b): global TERMINATED try: os.chdir(self.wd) except Exception as e: self.logger.warning(whoami() + str(e)) self.logger.info(whoami() + "terminating ...") TERMINATED = True
def close_all_connections(self): for idx, (sn, cn, rt, nobj) in enumerate(self.all_connections): if nobj: try: nobj.quit() self.all_connections[idx] = (sn, cn, rt, None) self.logger.warning(whoami() + "Closed connection #" + str(cn) + " on " + sn) except Exception as e: self.logger.warning(whoami() + "Cannot quit server " + sn + ": " + str(e))
def start_threads(self): if not self.threads: self.logger.debug(whoami() + "starting download threads") self.init_servers() for sn, scon, _, _ in self.all_connections: t = ConnectionWorker((sn, scon), self.articlequeue, self.port, self.servers, self.cfg, self.logger) self.threads.append((t, time.time())) t.start() else: self.logger.debug(whoami() + "threads already started")
def close_connection(self, server_name0, conn_nr): res = False for idx, (sn, cn, rt, nobj) in enumerate(self.all_connections): if sn == server_name0 and cn == conn_nr: if nobj: try: self.logger.warning(whoami() + "Closed connection #" + str(cn) + " on " + sn) self.all_connections[idx] = (sn, cn, rt, None) nobj.quit() res = True except Exception as e: self.logger.error(whoami() + "Server " + server_name0 + " close error: " + str(e)) res = False break return res
def update_fmodtime_nzbfiles(nzbfilelist, dirs, logger): for nzbfile in nzbfilelist: nzbfile_full = dirs["nzb"] + nzbfile try: f = open(nzbfile_full, "a") # trigger inotify "MODIFY" f.write("<!--modified-->\n") f.close() except Exception as e: logger.warning(whoami() + str(e)) return
def remove_nzbdirs(deleted_nzbs, dirs, pwdb, logger, removenzbfile=True): for deleted_nzb in deleted_nzbs: nzbdirname = re.sub(r"[.]nzb$", "", deleted_nzb, flags=re.IGNORECASE) + "/" if removenzbfile: # delete nzb from .ginzibix/nzb try: os.rename(dirs["nzb"] + deleted_nzb, dirs["nzb"] + deleted_nzb + ".bak") logger.debug(whoami() + ": renamed NZB " + deleted_nzb + " to .bak") except Exception as e: logger.warning(whoami() + str(e)) # remove incomplete/$nzb_name try: shutil.rmtree(dirs["incomplete"] + nzbdirname) logger.debug(whoami() + ": deleted incomplete dir for " + deleted_nzb) except Exception as e: logger.warning(whoami() + str(e))
def stop_threads(self): if not self.threads: self.logger.debug(whoami() + "no threads running, exiting ...") return try: self.logger.debug(whoami() + "stopping download threads + servers") for t, _ in self.threads: t.stop() t.last_downloaded_ts = None for t, _ in self.threads: t.join() del self.threads self.threads = [] if self.servers: self.servers.close_all_connections() del self.servers self.servers = None self.logger.debug(whoami() + "all threads / servers stopped") except Exception as e: self.logger.warning(whoami() + str(e))
def scan_for_par2(notrenamedfiles, logger): p2obj0 = None p2basename0 = None for fn, _ in notrenamedfiles: ptype0 = par2lib.check_for_par_filetype(fn) if ptype0 == 1: p2obj0 = par2lib.Par2File(fn) p2basename0 = fn.split(".par2")[0] logger.debug(whoami() + "Found .par2 in _downloaded0: " + fn.split("/")[-1]) break return p2obj0, p2basename0
def retry_connect(self): idx = 0 self.logger.debug(whoami() + "Server " + self.idn + " connecting ...") while idx < 5 and self.running and not self.paused: try: self.servers.close_connection(self.name, self.conn_nr) except Exception as e: self.logger.warning(whoami() + str(e) + ": cannot close " + self.idn) self.nntpobj = self.servers.open_connection(self.name, self.conn_nr) if self.nntpobj: self.logger.debug(whoami() + "Server " + self.idn + " connected!") self.last_timestamp = time.time() self.connectionstate = 1 self.server_name, self.server_url, self.server_user, self.server_password, self.server_port,\ self.server_usessl, self.server_level, self.server_connections, self.server_retention,\ self.useserver = self.servers.get_single_server_config(self.connection[0]) self.wait_running(1) return self.logger.warning(whoami() + "Could not connect to server " + self.idn + ", will retry in 5 sec.") self.wait_running(2) if not self.running or self.paused: break idx += 1 if not self.running: self.logger.warning(whoami() + "No connection retries anymore due to exiting") else: self.logger.error(whoami() + "Connect retries to " + self.idn + " failed!") self.connectionstate = -1
def rename_and_move_rarandremainingfiles_old(p2obj, notrenamedfiles, source_dir, dest_dir, pwdb, renamer_result_queue, filewrite_lock, logger): if p2obj: rarfileslist = [(fn, md5) for fn, md5 in p2obj.md5_16khash() if par2lib.get_file_type(fn) == "rar"] notrenamedfiles0 = notrenamedfiles[:] # rarfiles for a_name, a_md5 in notrenamedfiles0: pp = (a_name, a_md5) try: r_name = [fn for fn, r_md5 in rarfileslist if r_md5 == a_md5][0] if not r_name: continue if r_name != a_name: with filewrite_lock: shutil.copyfile(source_dir + a_name, dest_dir + r_name) else: with filewrite_lock: shutil.copyfile(source_dir + a_name, dest_dir + a_name) r_name = a_name # oldft = pwdb.db_file_get_orig_filetype(a_name) oldft = pwdb.exc("db_file_get_orig_filetype", [a_name], {}) # pwdb.db_file_set_renamed_name(a_name, r_name) pwdb.exc("db_file_set_renamed_name", [a_name, r_name], {}) pwdb.exc("db_file_set_file_type", [a_name, "rar"], {}) renamer_result_queue.put( (r_name, dest_dir + r_name, "rar", a_name, oldft)) # os.rename(source_dir + a_name, source_dir + a_name + ".renamed") with filewrite_lock: os.remove(source_dir + a_name) notrenamedfiles.remove(pp) except IndexError: pass except Exception as e: logger.warning(whoami() + str(e)) for a_name, a_md5 in notrenamedfiles: with filewrite_lock: shutil.copyfile(source_dir + a_name, dest_dir + a_name) ft = par2lib.get_file_type(a_name) # pwdb.db_file_set_renamed_name(a_name, a_name) pwdb.exc("db_file_set_renamed_name", [a_name, a_name], {}) # pwdb.db_file_set_file_type(a_name, ft) pwdb.exc("db_file_set_file_type", [a_name, ft], {}) renamer_result_queue.put((a_name, dest_dir + a_name, ft, a_name, ft)) # os.rename(source_dir + a_name, source_dir + a_name + ".renamed") with filewrite_lock: os.remove(source_dir + a_name)
def make_allfilelist_wait(pwdb, dirs, logger, timeout0): # immediatley get allfileslist try: nzbname = pwdb.exc("make_allfilelist", [dirs["incomplete"], dirs["nzb"]], {}) if nzbname: logger.debug(whoami() + "no timeout, got nzb " + nzbname + " immediately!") return nzbname elif timeout0 and timeout0 <= -1: return None except Exception as e: logger.warning(whoami() + str(e)) return None # setup inotify logger.debug(whoami() + "waiting for new nzb with timeout=" + str(timeout0)) t0 = time.time() if not timeout0: delay0 = 5 else: delay0 = 1 while True: try: nzbname = pwdb.exc("make_allfilelist", [dirs["incomplete"], dirs["nzb"]], {}) except Exception as e: logger.warning(whoami() + str(e)) if nzbname: logger.debug(whoami() + "new nzb found in db, queuing ...") return nzbname if timeout0: if time.time() - t0 > timeout0 / 1000: break time.sleep(delay0) return None
def multipartrar_repair(directory, parvolname, pwdb, nzbname, logger): cwd0 = os.getcwd() os.chdir(directory) logger.info(whoami() + "checking if repair possible for " + parvolname) pwdb.exc("db_msg_insert", [nzbname, "checking if repair is possible", "info"], {}) ssh = subprocess.Popen(['par2verify', parvolname], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) sshres = ssh.stdout.readlines() repair_is_required = False repair_is_possible = False exitstatus = 0 for ss in sshres: ss0 = ss.decode("utf-8") if "Repair is required" in ss0: repair_is_required = True if "Repair is possible" in ss0: repair_is_possible = True if repair_is_possible and repair_is_required: logger.info(whoami() + "repair is required and possible, performing par2repair") pwdb.exc("db_msg_insert", [ nzbname, "repair is required and possible, performing par2repair", "info" ], {}) # repair ssh = subprocess.Popen(['par2repair', parvolname], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) sshres = ssh.stdout.readlines() repair_complete = False for ss in sshres: ss0 = ss.decode("utf-8") if "Repair complete" in ss0: repair_complete = True if not repair_complete: exitstatus = -1 logger.error(whoami() + "could not repair") else: logger.info(whoami() + "repair success!!") exitstatus = 1 elif repair_is_required and not repair_is_possible: logger.error(whoami() + "repair is required but not possible!") pwdb.exc("db_msg_insert", [nzbname, "repair is required but not possible!", "error"], {}) exitstatus = -1 elif not repair_is_required and not repair_is_possible: logger.error(whoami() + "repair is not required - all OK!") exitstatus = 1 os.chdir(cwd0) return exitstatus
def rename_and_move_rarandremainingfiles(p2list, notrenamedfiles, source_dir, dest_dir, pwdb, renamer_result_queue, filewrite_lock, logger): for _, _, _, rarfileslist in p2list: notrenamedfiles0 = notrenamedfiles[:] # rarfiles for fullname, shortname, a_md5 in notrenamedfiles0: pp = (fullname, shortname, a_md5) try: r_name = [fn for fn, r_md5 in rarfileslist if r_md5 == a_md5][0] if not r_name: continue if r_name != shortname: with filewrite_lock: shutil.copyfile(source_dir + shortname, dest_dir + r_name) else: with filewrite_lock: shutil.copyfile(source_dir + shortname, dest_dir + shortname) r_name = shortname oldft = pwdb.exc("db_file_get_orig_filetype", [shortname], {}) pwdb.exc("db_file_set_renamed_name", [shortname, r_name], {}) pwdb.exc("db_file_set_file_type", [shortname, "rar"], {}) renamer_result_queue.put( (r_name, dest_dir + r_name, "rar", shortname, oldft)) with filewrite_lock: os.remove(source_dir + shortname) notrenamedfiles.remove(pp) except IndexError: pass except Exception as e: logger.warning(whoami() + str(e)) for fullname, shortname, a_md5 in notrenamedfiles: with filewrite_lock: shutil.copyfile(source_dir + shortname, dest_dir + shortname) ft = par2lib.get_file_type(fullname, inspect=True) oldft = pwdb.exc("db_file_get_orig_filetype", [shortname], {}) pwdb.exc("db_file_set_renamed_name", [shortname, shortname], {}) pwdb.exc("db_file_set_file_type", [shortname, ft], {}) renamer_result_queue.put( (shortname, dest_dir + shortname, ft, shortname, oldft)) with filewrite_lock: os.remove(source_dir + shortname)
def scan_renamed_dir(renamed_dir, p2obj, filewrite_lock, logger): # get all files in renamed p2list = [] with filewrite_lock: rn = [] p2obj0 = p2obj p2basename0 = None for fn in glob.glob(renamed_dir + "*") + glob.glob(renamed_dir + ".*"): fn0 = fn.split("/")[-1] ptype0 = par2lib.check_for_par_filetype(fn) if ptype0 == 1: p2obj0 = par2lib.Par2File(fn) p2basename0 = fn.split(".par2")[0] logger.debug(whoami() + "Found .par2 in _renamed0: " + fn0) p2list.append((p2obj0, p2basename0)) rn.append(fn0) return rn, p2obj0, p2basename0
def get_not_yet_renamed_files_new(dir0, pwdb, filewrite_lock, logger): nrf = [] with filewrite_lock: for fn in glob.glob(dir0 + "*") + glob.glob(dir0 + ".*"): fn0 = fn.split("/")[-1] rn = pwdb.exc("db_file_get_renamed_name", [fn0], {}) is_renamed = rn and (rn != "N/A") if not is_renamed: nrf.append((fn0, par2lib.calc_file_md5hash_16k(fn0))) p2list = [] with filewrite_lock: for fn in glob.glob(dir0 + "*") + fn in glob.glob(dir0 + ".*"): fn0 = fn.split("/")[-1] ptype0 = par2lib.check_for_par_filetype(fn) if ptype0 == 1: p2obj0 = par2lib.Par2File(fn) p2basename0 = fn.split(".par2")[0] logger.debug(whoami() + "Found .par2 in _renamed0: " + fn0) p2list.append((p2obj0, p2basename0)) return nrf, p2list
def process_next_unrar_child_pass(event_idle, child, logger): str0 = "" timeout = False while True: try: a = child.read_nonblocking(timeout=120).decode("utf-8") str0 += a except pexpect.exceptions.EOF: break except pexpect.exceptions.TIMEOUT: timeout = True break except Exception as e: logger.warning(whoami() + str(e)) if str0[-6:] == "[Q]uit": break if timeout: statmsg = "pexpect.timeout exceeded" status = -3 else: status = 1 statmsg = "" if "WARNING: You need to start extraction from a previous volume" in str0: child.close(force=True) statmsg = "WARNING: You need to start extraction from a previous volume" status = -5 elif "error" in str0: if "packed data checksum" in str0: statmsg = "packed data checksum error (= corrupt rar!)" status = -1 elif "- checksum error" in str0: statmsg = "checksum error (= rar is missing!)" status = -2 else: statmsg = "unknown error" status = -3 else: if "All OK" in str0: status = 0 statmsg = "All OK" return status, statmsg, str0
def stop(self): self.logger.debug(whoami() + "setting event_stopped") self.event_stopped.set()
def decode_articles(mp_work_queue0, mp_loggerqueue, filewrite_lock): setproctitle("gzbx." + os.path.basename(__file__)) logger = mplogging.setup_logger(mp_loggerqueue, __file__) logger.info(whoami() + "starting article decoder process") sh = SigHandler_Decoder(logger) signal.signal(signal.SIGINT, sh.sighandler) signal.signal(signal.SIGTERM, sh.sighandler) pwdb = PWDBSender() bytesfinal = bytearray() while not TERMINATED: res0 = None decoder_set_idle(True) while not TERMINATED: try: res0 = mp_work_queue0.get_nowait() break except (queue.Empty, EOFError): pass except Exception as e: logger.warning(whoami() + str(e)) time.sleep(0.1) if not res0 or TERMINATED: logger.info(whoami() + "exiting decoder process!") break decoder_set_idle(False) infolist, save_dir, filename, filetype = res0 logger.debug(whoami() + "starting decoding for " + filename) # del bytes0 bytesfinal = bytearray() status = 0 # 1: ok, 0: wrong yenc structure, -1: no crc32, -2: crc32 checksum error, -3: decoding error statusmsg = "ok" # ginzyenc full_filename = save_dir + filename i = 0 for info in infolist: try: lastline = info[-1].decode("latin-1") m = re.search('size=(.\d+?) ', lastline) if m: size = int(m.group(1)) except Exception as e: logger.warning(whoami() + str(e) + ", guestimate size ...") size = int(sum(len(i) for i in info.lines) * 1.1) try: decoded_data, output_filename, crc, crc_yenc, crc_correct = ginzyenc.decode_usenet_chunks( info, size) #if filename.endswith("part01.rar") and i == 3: # pass #else: bytesfinal.extend(decoded_data) except Exception as e: logger.warning(whoami() + str(e) + ": cannot perform ginzyenc") status = -3 statusmsg = "ginzyenc decoding error!" # continue decoding, maybe it can be repaired ...? i += 1 logger.debug(whoami() + "decoding for " + filename + ": success!") try: with filewrite_lock: if not os.path.isdir(save_dir): os.makedirs(save_dir) with open(full_filename, "wb") as f0: f0.write(bytesfinal) f0.flush() f0.close() except Exception as e: statusmsg = "file_error" logger.error(whoami() + str(e) + " in file " + filename) status = -4 logger.info(whoami() + filename + " decoded with status " + str(status) + " / " + statusmsg) pwdbstatus = 2 if status in [-3, -4]: pwdbstatus = -1 try: # pwdb.db_file_update_status(filename, pwdbstatus) pwdb.exc("db_file_update_status", [filename, pwdbstatus], {}) logger.debug(whoami() + "updated DB for " + filename + ", db.status=" + str(pwdbstatus)) except Exception as e: logger.error(whoami() + str(e) + ": cannot update DB for " + filename) i += 1 logger.debug(whoami() + "exited!")
def sighandler(self, a, b): global TERMINATED self.logger.info(whoami() + "terminating ...") TERMINATED = True
def get_password(directory, pw_file, nzbname0, logger, get_pw_direct=False): if directory[-1] != "/": directory += "/" rars = get_sorted_rar_list(directory) if not rars: return None rarname0 = rars[0][1] rarname = rarname0.split("/")[-1] nzbname = nzbname0.split(".nzb")[0] logger.debug(whoami() + "trying to get password") cwd0 = os.getcwd() if get_pw_direct: gg = re.search(r"}}.nzb$", nzbname0, flags=re.IGNORECASE) if gg: try: pw0 = nzbname0[:gg.start()].split("{{")[-1] os.chdir(directory) if test_password(pw0, rarname): logger.info(whoami() + "Found PW for NZB " + nzbname + ": " + pw0) os.chdir(cwd0) return pw0 else: logger.warning(whoami() + "provided password was not correct!") except Exception as e: logger.debug(whoami() + str(e) + ": cannot get pw from nzb string") os.chdir(cwd0) # PW file format: # a) pw # pw # pw # b) filename1 <:::> pw1 # filename2 <:::> pw2 if not pw_file: return None logger.debug(whoami() + "reading passwords in " + pw_file) try: with open(pw_file, "r") as f0: pw_list = f0.readlines() except Exception as e: logger.warning(whoami() + str(e) + ": cannot open/read pw file") return None cwd0 = os.getcwd() os.chdir(directory) pwlist = [pw.rstrip("\n") for pw in pw_list] PW = None # first try with <:::> if exists logger.info(whoami() + "trying specified password entries <:::> ...") for pw in pwlist: if "<:::>" not in pw: continue fn0 = pw.split("<:::>")[0].lstrip(" ").rstrip(" ") fn0 = fn0.split(".nzb")[0] pw0 = pw.split("<:::>")[1].lstrip(" ").rstrip(" ") pw0 = pw0.split(".nzb")[0] if fn0 != nzbname: continue logger.debug(whoami() + "Trying with entry: " + fn0 + " / " + pw0 + " for NZB " + nzbname) if test_password(pw0, rarname): PW = pw0 logger.info(whoami() + "Found PW for NZB " + nzbname + ": " + PW) break if PW: os.chdir(cwd0) return PW # try logger.info(whoami() + "trying free password file entries ...") for pw in pwlist: if "<:::>" in pw: continue if test_password(pw, rarname): PW = pw logger.info(whoami() + "Found PW for NZB " + nzbname + ": " + PW) break os.chdir(cwd0) return PW
def postprocess_nzb(nzbname, articlequeue, resultqueue, mp_work_queue, pipes, mpp0, mp_events, cfg, verifiedrar_dir, unpack_dir, nzbdir, rename_dir, main_dir, download_dir, dirs, pw_file, mp_loggerqueue): setproctitle("gzbx." + os.path.basename(__file__)) pwdb = PWDBSender() if pwdb.exc("db_nzb_getstatus", [nzbname], {}) in [4, -4]: sys.exit() logger = setup_logger(mp_loggerqueue, __file__) logger.debug(whoami() + "starting ...") nzbdirname = re.sub(r"[.]nzb$", "", nzbname, flags=re.IGNORECASE) + "/" incompletedir = dirs["incomplete"] + nzbdirname if not os.path.isdir(incompletedir): logger.error(whoami() + "no incomplete_dir, aborting postprocessing") pwdb.exc("db_nzb_update_status", [nzbname, -4], {}) sys.exit() mpp = mpp0.copy() sh = SigHandler_Postprocessing(logger) signal.signal(signal.SIGINT, sh.sighandler_postprocessing) signal.signal(signal.SIGTERM, sh.sighandler_postprocessing) stop_wait(nzbname, dirs, pwdb) event_verifieridle = mp_events["verifier"] event_unrareridle = mp_events["unrarer"] pwdb.exc("db_msg_insert", [nzbname, "starting postprocess", "info"], {}) logger.debug(whoami() + "starting clearing queues & pipes") # clear pipes do_mpconnections(pipes, "clearqueues", None) try: for key, item in pipes.items(): if pipes[key][0].poll(): pipes[key][0].recv() except Exception as e: logger.error(whoami() + str(e)) pwdb.exc("db_nzb_update_status", [nzbname, -4], {}) pwdb.exc("db_msg_insert", [nzbname, "postprocessing/clearing pipes failed!", "error"], {}) sys.exit() logger.debug(whoami() + "clearing queues & pipes done!") # join decoder if mpp_is_alive(mpp, "decoder"): t0 = time.time() timeout_reached = False try: while mp_work_queue.qsize() > 0: if time.time() - t0 > 60 * 3: timeout_reached = True break time.sleep(0.5) except Exception as e: logger.debug(whoami() + str(e)) timeout_reached = True if timeout_reached: logger.warning( whoami() + "Timeout reached/error in joining decoder, terminating decoder ..." ) kill_mpp(mpp, "decoder") logger.info(whoami() + "decoder terminated!") try: while not mp_work_queue.empty(): mp_work_queue.get_no_wait() except Exception: pass stop_wait(nzbname, dirs, pwdb) # --- PAR_VERIFIER --- verifystatus = pwdb.exc("db_nzb_get_verifystatus", [nzbname], {}) all_rars_are_verified, _ = pwdb.exc("db_only_verified_rars", [nzbname], {}) renamed_rar_files = pwdb.exc("get_all_renamed_rar_files", [nzbname], {}) # verifier not running, status = 0 --> start & wait if not mpp_is_alive( mpp, "verifier" ) and verifystatus == 0 and not all_rars_are_verified and renamed_rar_files: logger.info( whoami() + "there are files to check by par_verifier, starting par_verifier ..." ) p2 = pwdb.exc("get_renamed_p2", [rename_dir, nzbname], {}) if p2: pvmode = "verify" else: pvmode = "copy" mpp_verifier = mp.Process(target=par_verifier.par_verifier, args=( pipes["verifier"][1], rename_dir, verifiedrar_dir, main_dir, mp_loggerqueue, nzbname, pvmode, event_verifieridle, cfg, )) mpp_verifier.start() mpp["verifier"] = mpp_verifier time.sleep(1) stop_wait(nzbname, dirs, pwdb) verifystatus = pwdb.exc("db_nzb_get_verifystatus", [nzbname], {}) # verifier is running, status == 1 (running) or -2 -> wait/join if mpp_is_alive(mpp, "verifier") and verifystatus in [1, -2]: logger.info(whoami() + "Waiting for par_verifier to complete") try: # kill par_verifier in deadlock while True: mpp_join(mpp, "verifier", timeout=5) if mpp_is_alive(mpp, "verifier"): # if not finished, check if idle longer than 5 sec -> deadlock!!! t0 = time.time() while event_verifieridle.is_set( ) and time.time() - t0 < 30: time.sleep(0.5) if time.time() - t0 >= 30: logger.info(whoami() + "Verifier deadlock, killing verifier!") kill_mpp(mpp, "verifier") break else: continue else: break except Exception as e: logger.warning(whoami() + str(e)) mpp_join(mpp, "verifier") mpp["verifier"] = None logger.debug(whoami() + "par_verifier completed/terminated!") # if verifier is running but wrong status, or correct status but not running, exit elif mpp_is_alive(mpp, "verifier") or verifystatus == 1: pwdb.exc("db_nzb_update_status", [nzbname, -4], {}) pwdb.exc("db_msg_insert", [nzbname, "par_verifier status inconsistency", "error"], {}) logger.debug( whoami() + "something is wrong with par_verifier, exiting postprocessor") logger.info(whoami() + "postprocess of NZB " + nzbname + " failed!") if mpp_is_alive(mpp, "verifier"): logger.debug(whoami() + "terminating par_verifier") kill_mpp(mpp, "verifier") logger.info(whoami() + "verifier terminated!") sys.exit() stop_wait(nzbname, dirs, pwdb) # --- UNRARER --- # already checked if pw protected? is_pw_checked = pwdb.exc("db_nzb_get_ispw_checked", [nzbname], {}) if not is_pw_checked: logger.debug(whoami() + "testing if pw protected") ispw = passworded_rars.is_rar_password_protected( verifiedrar_dir, logger) pwdb.exc("db_nzb_set_ispw_checked", [nzbname, True], {}) if ispw == 0 or ispw == -2 or ispw == -3: logger.warning( whoami() + "cannot test rar if pw protected, something is wrong: " + str(ispw) + ", exiting ...") pwdb.exc("db_msg_insert", [ nzbname, "postprocessing failed due to pw test not possible", "error" ], {}) pwdb.exc("db_nzb_update_status", [nzbname, -4], {}) sys.exit() elif ispw == 1: pwdb.exc("db_nzb_set_ispw", [nzbname, True], {}) pwdb.exc("db_msg_insert", [nzbname, "rar archive is password protected", "warning"], {}) logger.info(whoami() + "rar archive is pw protected") elif ispw == -1: pwdb.exc("db_nzb_set_ispw", [nzbname, False], {}) pwdb.exc( "db_msg_insert", [nzbname, "rar archive is NOT password protected", "info"], {}) logger.info(whoami() + "rar archive is NOT pw protected") ispw = pwdb.exc("db_nzb_get_ispw", [nzbname], {}) unrarernewstarted = False if ispw: get_pw_direct0 = False try: get_pw_direct0 = ( cfg["OPTIONS"]["GET_PW_DIRECTLY"].lower() == "yes") except Exception as e: logger.warning(whoami() + str(e)) if pwdb.exc("db_nzb_get_password", [nzbname], {}) == "N/A": logger.info(whoami() + "Trying to get password from file for NZB " + nzbname) pwdb.exc("db_msg_insert", [nzbname, "trying to get password", "info"], {}) pw = passworded_rars.get_password(verifiedrar_dir, pw_file, nzbname, logger, get_pw_direct=get_pw_direct0) if pw: logger.info(whoami() + "Found password " + pw + " for NZB " + nzbname) pwdb.exc("db_msg_insert", [nzbname, "found password " + pw, "info"], {}) pwdb.exc("db_nzb_set_password", [nzbname, pw], {}) else: pw = pwdb.exc("db_nzb_get_password", [nzbname], {}) if not pw: pwdb.exc("db_msg_insert", [ nzbname, "Provided password was not correct / no password found in PW file! ", "error" ], {}) logger.error(whoami() + "Cannot find password for NZB " + nzbname + "in postprocess, exiting ...") pwdb.exc("db_nzb_update_status", [nzbname, -4], {}) mpp["unrarer"] = None # sighandler.mpp = mpp sys.exit() event_unrareridle = mp.Event() # passing p2list to unrarer mpp_unrarer = mp.Process(target=partial_unrar.partial_unrar, args=( verifiedrar_dir, unpack_dir, nzbname, mp_loggerqueue, pw, event_unrareridle, cfg, )) unrarernewstarted = True mpp_unrarer.start() mpp["unrarer"] = mpp_unrarer # sighandler.mpp = self.mpp # start unrarer if never started and ok verified/repaired elif not mpp["unrarer"]: logger.debug(whoami() + "checking if unrarer should be started") try: verifystatus = pwdb.exc("db_nzb_get_verifystatus", [nzbname], {}) unrarstatus = pwdb.exc("db_nzb_get_unrarstatus", [nzbname], {}) except Exception as e: logger.warning(whoami() + str(e)) logger.debug(whoami() + "verifystatus: " + str(verifystatus) + " / unrarstatus: " + str(unrarstatus)) if (verifystatus > 0 or verifystatus == -2) and unrarstatus == 0: try: logger.debug(whoami() + "unrarer passiv until now, starting ...") event_unrareridle = mp.Event() # passing p2list to unrarer mpp_unrarer = mp.Process(target=partial_unrar.partial_unrar, args=( verifiedrar_dir, unpack_dir, nzbname, mp_loggerqueue, None, event_unrareridle, cfg, )) unrarernewstarted = True mpp_unrarer.start() mpp["unrarer"] = mpp_unrarer except Exception as e: logger.warning(whoami() + str(e)) finalverifierstate = (pwdb.exc("db_nzb_get_verifystatus", [nzbname], {}) in [0, 2]) stop_wait(nzbname, dirs, pwdb) # join unrarer if mpp_is_alive(mpp, "unrarer"): if finalverifierstate: logger.info(whoami() + "Waiting for unrar to complete") while True: # try to join unrarer mpp_join(mpp, "unrarer", timeout=5) isalive = mpp_is_alive(mpp, "unrarer") if isalive: # if not finished, check if idle longer than 5 sec -> deadlock!!! t0 = time.time() timeout0 = 99999999 if unrarernewstarted else 120 * 2 while event_unrareridle.is_set( ) and time.time() - t0 < timeout0: time.sleep(0.5) if time.time() - t0 >= timeout0: logger.info(whoami() + "Unrarer deadlock, killing unrarer!") kill_mpp(mpp, "unrarer") break else: logger.debug( whoami() + "Unrarer not idle, waiting before terminating") stop_wait(nzbname, dirs, pwdb) time.sleep(0.5) continue else: break else: logger.info(whoami() + "Repair/unrar not possible, killing unrarer!") kill_mpp(mpp, "unrarer") logger.debug(whoami() + "unrarer completed/terminated!") stop_wait(nzbname, dirs, pwdb) # get status finalverifierstate = (pwdb.exc("db_nzb_get_verifystatus", [nzbname], {}) in [0, 2]) finalnonrarstate = pwdb.exc("db_allnonrarfiles_getstate", [nzbname], {}) finalrarstate = (pwdb.exc("db_nzb_get_unrarstatus", [nzbname], {}) in [0, 2]) logger.info(whoami() + "Finalverifierstate: " + str(finalverifierstate) + " / Finalrarstate: " + str(finalrarstate) + " / Finalnonrarstate: " + str(finalnonrarstate)) if finalrarstate and finalnonrarstate and finalverifierstate: pwdb.exc("db_msg_insert", [nzbname, "unrar/par-repair ok!", "success"], {}) logger.info(whoami() + "unrar/par-repair of NZB " + nzbname + " success!") else: pwdb.exc("db_nzb_update_status", [nzbname, -4], {}) pwdb.exc("db_msg_insert", [nzbname, "unrar/par-repair failed!", "error"], {}) logger.info(whoami() + "postprocess of NZB " + nzbname + " failed!") sys.exit() stop_wait(nzbname, dirs, pwdb) # copy to complete logger.info(whoami() + "starting copy-to-complete") pwdb.exc("db_msg_insert", [nzbname, "copying & cleaning directories", "info"], {}) complete_dir = make_complete_dir(dirs, nzbdir, logger) if not complete_dir: pwdb.exc("db_nzb_update_status", [nzbname, -4], {}) pwdb.exc("db_msg_insert", [nzbname, "postprocessing failed!", "error"], {}) logger.info("Cannot create complete_dir for " + nzbname + ", exiting ...") pwdb.exc("db_msg_insert", [nzbname, "postprocessing failed!", "error"], {}) sys.exit() # move all non-rar/par2/par2vol files from renamed to complete for f00 in glob.glob(rename_dir + "*") + glob.glob(rename_dir + ".*"): logger.debug(whoami() + "renamed_dir: checking " + f00 + " / " + str(os.path.isdir(f00))) if os.path.isdir(f00): logger.debug(f00 + "is a directory, skipping") continue f0 = f00.split("/")[-1] file0type = pwdb.exc("db_file_getftype_renamed", [f0], {}) logger.debug(whoami() + "Moving/deleting " + f0) if not file0type: gg = re.search(r"[0-9]+[.]rar[.]+[0-9]", f0, flags=re.IGNORECASE) if gg: try: os.remove(f00) logger.debug(whoami() + "Removed rar.x file " + f0) except Exception as e: pwdb.exc("db_nzb_update_status", [nzbname, -4], {}) logger.warning(whoami() + str(e) + ": cannot remove corrupt rar file!") else: # if unknown file (not in db) move to complete anyway try: shutil.move(f00, complete_dir) logger.debug(whoami() + "moved " + f00 + " to " + complete_dir) except Exception as e: logger.warning(whoami() + str(e) + ": cannot move unknown file to complete!") continue if file0type in ["rar", "par2", "par2vol"]: try: os.remove(f00) logger.debug(whoami() + "removed rar/par2 file " + f0) except Exception as e: pwdb.exc("db_nzb_update_status", [nzbname, -4], {}) logger.warning(whoami() + str(e) + ": cannot remove rar/par2 file!") else: try: shutil.move(f00, complete_dir) logger.debug(whoami() + "moved non-rar/non-par2 file " + f0 + " to complete") except Exception as e: pwdb.exc("db_nzb_update_status", [nzbname, -4], {}) logger.warning(whoami() + str(e) + ": cannot move non-rar/non-par2 file " + f00 + "!") stop_wait(nzbname, dirs, pwdb) # remove download_dir try: shutil.rmtree(download_dir) except Exception as e: pwdb.exc("db_nzb_update_status", [nzbname, -4], {}) logger.warning(whoami() + str(e) + ": cannot remove download_dir!") # move content of unpack dir to complete logger.debug(whoami() + "moving unpack_dir to complete: " + unpack_dir) for f00 in glob.glob(unpack_dir + "*") + glob.glob(unpack_dir + ".*"): logger.debug(whoami() + "unpack_dir: checking " + f00 + " / " + str(os.path.isdir(f00))) d0 = f00.split("/")[-1] logger.debug(whoami() + "Does " + complete_dir + d0 + " already exist?") if os.path.isfile(complete_dir + d0): try: logger.debug(whoami() + complete_dir + d0 + " already exists, deleting!") os.remove(complete_dir + d0) except Exception: logger.debug(whoami() + f00 + " already exists but cannot delete") pwdb.exc("db_nzb_update_status", [nzbname, -4], {}) break else: logger.debug(whoami() + complete_dir + d0 + " does not exist!") if not os.path.isdir(f00): try: shutil.move(f00, complete_dir) logger.debug(whoami() + ": moved " + f00 + " to " + complete_dir) except Exception as e: pwdb.exc("db_nzb_update_status", [nzbname, -4], {}) logger.warning(whoami() + str(e) + ": cannot move unrared file to complete dir!") else: if os.path.isdir(complete_dir + d0): try: shutil.rmtree(complete_dir + d0) logger.debug(whoami() + ": removed tree " + complete_dir + d0) except Exception as e: pwdb.exc("db_nzb_update_status", [nzbname, -4], {}) logger.warning(whoami() + str(e) + ": cannot remove unrared dir in complete!") try: logger.debug(whoami() + "copying tree " + f00 + " to " + complete_dir + d0) shutil.copytree(f00, complete_dir + d0) logger.debug(whoami() + "copied tree " + f00 + " to " + complete_dir + d0) except Exception as e: pwdb.exc("db_nzb_update_status", [nzbname, -4], {}) logger.warning(whoami() + str(e) + ": cannot move non-rar/non-par2 file!") # remove unpack_dir logger.debug(whoami() + ": removing unpacl_dir, verified_rardir and incomplete_dir") if pwdb.exc("db_nzb_getstatus", [nzbname], {}) != -4: try: shutil.rmtree(unpack_dir) shutil.rmtree(verifiedrar_dir) logger.debug(whoami() + ": removed " + unpack_dir + verifiedrar_dir) except Exception as e: pwdb.exc("db_nzb_update_status", [nzbname, -4], {}) logger.warning(whoami() + str(e) + ": cannot remove unpack_dir / verifiedrar_dir") # remove incomplete_dir if pwdb.exc("db_nzb_getstatus", [nzbname], {}) != -4: try: shutil.rmtree(main_dir) logger.debug(whoami() + ": removed " + main_dir) except Exception as e: pwdb.exc("db_nzb_update_status", [nzbname, -4], {}) logger.warning(whoami() + str(e) + ": cannot remove incomplete_dir!") # finalize if pwdb.exc("db_nzb_getstatus", [nzbname], {}) == -4: logger.info(whoami() + "Copy/Move of NZB " + nzbname + " failed!") else: logger.info(whoami() + "Copy/Move of NZB " + nzbname + " success!") pwdb.exc("db_nzb_update_status", [nzbname, 4], {}) logger.info(whoami() + "Postprocess of NZB " + nzbname + " success!") sys.exit()
def par_verifier(child_pipe, renamed_dir, verifiedrar_dir, main_dir, mp_loggerqueue, nzbname, pvmode, event_idle, cfg): setproctitle("gzbx." + os.path.basename(__file__)) logger = mplogging.setup_logger(mp_loggerqueue, __file__) logger.debug(whoami() + "starting ...") sh = SigHandler_Verifier(logger) signal.signal(signal.SIGINT, sh.sighandler_verifier) signal.signal(signal.SIGTERM, sh.sighandler_verifier) pwdb = PWDBSender() event_idle.clear() if pvmode == "verify": # p2 = pwdb.get_renamed_p2(renamed_dir, nzbname) try: p2list = pwdb.exc("db_p2_get_p2list", [nzbname], {}) p2 = P2(p2list) except Exception as e: logger.warning(whoami() + str(e)) # pwdb.db_nzb_update_verify_status(nzbname, 1) pwdb.exc("db_nzb_update_verify_status", [nzbname, 1], {}) # a: verify all unverified files in "renamed" unverified_rarfiles = None try: # unverified_rarfiles = pwdb.get_all_renamed_rar_files(nzbname) unverified_rarfiles = pwdb.exc("get_all_renamed_rar_files", [nzbname], {}) except Exception as e: logger.debug(whoami() + str(e) + ": no unverified rarfiles met in first run, skipping!") doloadpar2vols = False if pvmode == "verify" and not p2: logger.debug(whoami() + "no par2 file found") if pvmode == "verify" and unverified_rarfiles and p2: logger.debug(whoami() + "verifying all unchecked rarfiles") for filename, f_origname in unverified_rarfiles: f_short = filename.split("/")[-1] md5 = calc_file_md5hash(renamed_dir + filename) md5match = [(pmd5 == md5) for pname, pmd5 in p2.filenames() if pname == filename] if False in md5match: logger.warning(whoami() + " error in md5 hash match for file " + f_short) pwdb.exc("db_msg_insert", [ nzbname, "error in md5 hash match for file " + f_short, "warning" ], {}) pwdb.exc("db_nzb_update_verify_status", [nzbname, -2], {}) pwdb.exc("db_file_update_parstatus", [f_origname, -1], {}) child_pipe.send(True) else: logger.info(whoami() + f_short + " md5 hash match ok, copying to verified_rar dir") pwdb.exc("db_msg_insert", [ nzbname, f_short + " md5 hash match ok, copying to verified_rar dir ", "info" ], {}) shutil.copy(renamed_dir + filename, verifiedrar_dir) pwdb.exc("db_file_update_parstatus", [f_origname, 1], {}) elif (pvmode == "verify" and not p2) or (pvmode == "copy"): logger.info(whoami() + "copying all rarfiles") for filename, f_origname in unverified_rarfiles: f_short = filename.split("/")[-1] sfvcheck = pwdb.exc("db_nzb_check_sfvcrc32", [nzbname, renamed_dir, renamed_dir + filename], {}) if sfvcheck == -1: logger.warning(whoami() + " error in crc32 check for file " + f_short) pwdb.exc("db_msg_insert", [ nzbname, "error in crc32 check for file " + f_short, "warning" ], {}) pwdb.exc("db_nzb_update_verify_status", [nzbname, -2], {}) pwdb.exc("db_file_update_parstatus", [f_origname, -1], {}) child_pipe.send(True) continue logger.debug(whoami() + "copying " + f_short + " to verified_rar dir") pwdb.exc("db_msg_insert", [ nzbname, "copying " + f_short + " to verified_rar dir ", "info" ], {}) shutil.copy(renamed_dir + filename, verifiedrar_dir) pwdb.exc("db_file_update_parstatus", [f_origname, 1], {}) # b: inotify renamed_dir inotify = inotify_simple.INotify() watch_flags = inotify_simple.flags.CREATE | inotify_simple.flags.DELETE | inotify_simple.flags.MODIFY | inotify_simple.flags.DELETE_SELF inotify.add_watch(renamed_dir, watch_flags) while not TERMINATED: # allparstatus = pwdb.db_file_getallparstatus(0) allparstatus = pwdb.exc("db_file_getallparstatus", [0], {}) if 0 not in allparstatus: event_idle.clear() logger.info(whoami() + "all renamed rars checked, exiting par_verifier") break events = get_inotify_events(inotify) event_idle.set() if events or 0 in allparstatus: event_idle.clear() if pvmode == "verify" and not p2: try: p2list = pwdb.exc("db_p2_get_p2list", [nzbname], {}) p2 = P2(p2list) except Exception as e: logger.debug(whoami() + str(e)) if pvmode == "verify" and p2: for rar in glob.glob(renamed_dir + "*") + glob.glob(renamed_dir + ".*"): rar0 = rar.split("/")[-1] f0 = pwdb.exc("db_file_get_renamed", [rar0], {}) if not f0: continue f0_origname, f0_renamedname, f0_ftype = f0 if not f0_ftype == "rar": continue if pwdb.exc("db_file_getparstatus", [rar0], {}) == 0 and f0_renamedname != "N/A": f_short = f0_renamedname.split("/")[-1] md5 = calc_file_md5hash(renamed_dir + rar0) md5match = [(pmd5 == md5) for pname, pmd5 in p2.filenames() if pname == f0_renamedname] #print(f0_renamedname, md5, " : ", p2.filenames()) #print(md5match) #print("-" * 80) if True in md5match: logger.info( whoami() + f_short + " md5 hash match ok, copying to verified_rar dir" ) pwdb.exc("db_msg_insert", [ nzbname, f_short + " md5 hash match ok, copying to verified_rar dir ", "info" ], {}) shutil.copy(renamed_dir + f0_renamedname, verifiedrar_dir) pwdb.exc("db_file_update_parstatus", [f0_origname, 1], {}) elif False in md5match: logger.warning( whoami() + "error in md5 hash match for file " + f_short) pwdb.exc("db_msg_insert", [ nzbname, "error in md5 hash match for file " + f_short, "warning" ], {}) pwdb.exc("db_nzb_update_verify_status", [nzbname, -2], {}) pwdb.exc("db_file_update_parstatus", [f0_origname, -1], {}) child_pipe.send(True) else: # if no match at all in p2list -> try sfvcheck / or just copy sfvcheck = pwdb.exc("db_nzb_check_sfvcrc32", [nzbname, renamed_dir, rar], {}) if sfvcheck == -1: logger.warning( whoami() + " error in crc32 check for file " + rar0) pwdb.exc("db_msg_insert", [ nzbname, "error in crc32 check for file " + rar0, "warning" ], {}) pwdb.exc("db_nzb_update_verify_status", [nzbname, -2], {}) pwdb.exc("db_file_update_parstatus", [f0_origname, -1], {}) child_pipe.send(True) continue if pwdb.exc("db_file_getparstatus", [rar0], {}) == 0 and f0_renamedname != "N/A": logger.debug(whoami() + "no md5 check, copying " + f0_renamedname.split("/")[-1] + " to verified_rar dir") pwdb.exc("db_msg_insert", [ nzbname, "no md5 check, copying " + f0_renamedname.split("/")[-1] + " to verified_rar dir", "info" ], {}) shutil.copy(renamed_dir + f0_renamedname, verifiedrar_dir) pwdb.exc("db_file_update_parstatus", [f0_origname, 1], {}) # free rars or copy mode? elif (pvmode == "verify" and not p2) or (pvmode == "copy"): # maybe we can check via sfv file? for file0full in glob.glob(renamed_dir + "*") + glob.glob(renamed_dir + ".*"): file0short = file0full.split("/")[-1] ft = pwdb.exc("db_file_getftype_renamed", [file0short], {}) if ft == "rar": rar0 = file0short f0 = pwdb.exc("db_file_get_renamed", [rar0], {}) if not f0: continue f0_origname, f0_renamedname, f0_ftype = f0 sfvcheck = pwdb.exc("db_nzb_check_sfvcrc32", [nzbname, renamed_dir, file0full], {}) if sfvcheck == -1: logger.warning(whoami() + " error in crc32 check for file " + rar0) pwdb.exc("db_msg_insert", [ nzbname, "error in crc32 check for file " + rar0, "warning" ], {}) pwdb.exc("db_nzb_update_verify_status", [nzbname, -2], {}) pwdb.exc("db_file_update_parstatus", [f0_origname, -1], {}) child_pipe.send(True) continue if pwdb.exc("db_file_getparstatus", [rar0], {}) == 0 and f0_renamedname != "N/A": logger.debug(whoami() + "copying " + f0_renamedname.split("/")[-1] + " to verified_rar dir") pwdb.exc("db_msg_insert", [ nzbname, "copying " + f0_renamedname.split("/")[-1] + " to verified_rar dir", "info" ], {}) shutil.copy(renamed_dir + f0_renamedname, verifiedrar_dir) pwdb.exc("db_file_update_parstatus", [f0_origname, 1], {}) allrarsverified, rvlist = pwdb.exc("db_only_verified_rars", [nzbname], {}) if allrarsverified: break time.sleep(1) if TERMINATED: logger.info(whoami() + "terminated!") sys.exit() logger.debug(whoami() + "all rars are checked!") corruptrars = pwdb.exc("get_all_corrupt_rar_files", [nzbname], {}) if not corruptrars: logger.debug(whoami() + "rar files ok, no repair needed, exiting par_verifier") pwdb.exc("db_nzb_update_verify_status", [nzbname, 2], {}) elif p2list and corruptrars: pwdb.exc("db_msg_insert", [nzbname, "repairing rar files", "info"], {}) logger.info(whoami() + "par2vol files present, repairing ...") allok = True allfound = True corruptrars_1 = [c1 for c1, _ in corruptrars] corruptrars_2 = [c2 for _, c2 in corruptrars] for _, fnshort, fnlong, rarfiles in p2list: rarf_match = [ rarf for rarf, _ in rarfiles if rarf in corruptrars_1 or rarf in corruptrars_2 ] if len(rarf_match) == 0: allfound = False continue lrar = str(len(rarfiles)) pwdb.exc("db_msg_insert", [nzbname, "performing par2verify for " + fnshort, "info"], {}) ssh = subprocess.Popen(['par2verify', fnlong], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) sshres = ssh.stdout.readlines() repair_is_required = False repair_is_possible = False for ss in sshres: ss0 = ss.decode("utf-8") if "Repair is required" in ss0: repair_is_required = True if "Repair is possible" in ss0: repair_is_possible = True if not repair_is_required: pwdb.exc("db_msg_insert", [ nzbname, "par2verify for " + fnshort + ": repair is not required!", "info" ], {}) res0 = 1 elif repair_is_required and not repair_is_possible: pwdb.exc("db_msg_insert", [ nzbname, "par2verify for " + fnshort + ": repair is required but not possible", "error" ], {}) res0 = -1 elif repair_is_required and repair_is_possible: pwdb.exc("db_msg_insert", [ nzbname, "par2verify for " + fnshort + ": repair is required and possible, repairing files", "info" ], {}) logger.info( whoami() + "repair is required and possible, performing par2repair") # repair ssh = subprocess.Popen(['par2repair', fnlong], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) sshres = ssh.stdout.readlines() repair_complete = False for ss in sshres: ss0 = ss.decode("utf-8") if "Repair complete" in ss0: repair_complete = True if not repair_complete: res0 = -1 else: res0 = 1 # res0 = multipartrar_repair(renamed_dir, fnshort, pwdb, nzbname, logger) else: res0 = -1 if res0 != 1: allok = False logger.error(whoami() + "repair failed for " + lrar + "rarfiles in " + fnshort) pwdb.exc("db_msg_insert", [ nzbname, "rar file repair failed for " + lrar + " rarfiles in " + fnshort + "!", "error" ], {}) else: logger.info(whoami() + "repair success for " + lrar + "rarfiles in " + fnshort) pwdb.exc("db_msg_insert", [ nzbname, "rar file repair success for " + lrar + " rarfiles in " + fnshort + "!", "info" ], {}) if not allfound: allok = False logger.error( whoami() + "cannot attempt one or more par2repairs due to missing par2 file(s)!" ) pwdb.exc("db_msg_insert", [ nzbname, "cannot attempt one or more par2repairs due to missing par2 file(s)!", "error" ], {}) if allok: logger.info(whoami() + "repair success") pwdb.exc("db_nzb_update_verify_status", [nzbname, 2], {}) # copy all no yet copied rars to verifiedrar_dir for c_origname, c_renamedname in corruptrars: logger.info(whoami() + "copying " + c_renamedname + " to verifiedrar_dir") pwdb.exc("db_file_update_parstatus", [c_origname, 1], {}) pwdb.exc("db_file_update_status", [c_origname, 2], {}) shutil.copy(renamed_dir + c_renamedname, verifiedrar_dir) else: logger.error(whoami() + "repair failed!") pwdb.exc("db_nzb_update_verify_status", [nzbname, -1], {}) for _, c_origname in corruptrars: pwdb.exc("db_file_update_parstatus", [c_origname, -2], {}) else: pwdb.exc("db_msg_insert", [ "nzbname", "rar file repair failed, no par files available", "error" ], {}) logger.warning( whoami() + "some rars are corrupt but cannot repair (no par2 files)") pwdb.exc("db_nzb_update_verify_status", [nzbname, -1], {}) logger.info(whoami() + "terminated!") sys.exit()
def run(self): self.logger.info(whoami() + self.idn + " thread starting !") timeout = 2 self.tt_pause_started = None # connect to push-socket self.context = zmq.Context() self.socket = self.context.socket(zmq.PUSH) self.socket.connect("tcp://127.0.0.1:%d" % self.port) while self.running: self.download_done = True if self.paused: if not self.tt_pause_started: self.tt_pause_started = time.time() elif time.time() - self.tt_pause_started > self.connection_idle_time and self.nntpobj: if self.servers.close_connection(self.name, self.conn_nr): self.logger.info(whoami() + self.idn + " connection idle, closed!") self.nntpobj = None self.connectionstate = -1 else: self.logger.info(whoami() + self.idn + " connection non existent, closed!") self.nntpobj = None self.connectionstate = -1 time.sleep(0.25) continue else: self.tt_pause_started = None if not self.running: break try: article = self.articlequeue.pop() except (queue.Empty, EOFError, IndexError): time.sleep(0.1) continue except Exception as e: self.logger.warning(whoami() + str(e) + ": problem in clearing article queue") time.sleep(0.1) continue # avoid ctrl-c to interrup downloading itself self.download_done = False if not self.nntpobj: self.retry_connect() filename, age, filetype, nr_articles, art_nr, art_name, remaining_servers1 = article if not remaining_servers1: self.socket.send(pickle.dumps(article + (None,))) continue if self.name not in remaining_servers1[0] or not self.nntpobj: self.articlequeue.append((filename, age, filetype, nr_articles, art_nr, art_name, remaining_servers1)) time.sleep(0.1) continue if not self.nntpobj: self.wait_running(3) continue status, bytesdownloaded, info = self.download_article(art_name, age) # if ctrl-c - exit thread if status == -3 or not self.running: break # if download successfull - put to resultqueue elif status == 1: self.last_downloaded_ts = time.time() timeout = 2 self.bytesdownloaded += bytesdownloaded self.socket.send(pickle.dumps((filename, age, filetype, nr_articles, art_nr, art_name, self.name, info, True))) # if 400 error elif status == -2: # disconnect self.logger.warning(whoami() + self.idn + " server connection error, reconnecting ...") self.connectionstate = -1 try: name, conn_nr = self.connection if self.servers.close_connection(name, conn_nr): self.nntpobj = None except Exception: pass self.nntpobj = None # take next server next_servers = self.remove_from_remaining_servers(self.name, remaining_servers1) next_servers.append([self.name]) # add current server to end of list self.logger.debug(whoami() + "Requeuing " + art_name + " on server " + self.idn) # requeue self.articlequeue.append((filename, age, filetype, nr_articles, art_nr, art_name, next_servers)) self.wait_running(timeout) timeout *= 2 if timeout > 30: timeout = 2 continue # if article could not be found on server / retention not good enough - requeue to other server elif status in [0, -1]: timeout = 2 next_servers = self.remove_from_remaining_servers(self.name, remaining_servers1) if not next_servers: self.logger.error(whoami() + "Download finally failed on server " + self.idn + ": for article " + art_name + " " + str(next_servers)) self.socket.send(pickle.dumps((filename, age, filetype, nr_articles, art_nr, art_name, [], "failed", True))) else: self.logger.debug(whoami() + "Download failed on server " + self.idn + ": for article " + art_name + ", queueing: " + str(next_servers)) self.articlequeue.append((filename, age, filetype, nr_articles, art_nr, art_name, next_servers)) self.logger.info(whoami() + self.idn + " exited!")
def partial_unrar(directory, unpack_dir, nzbname, mp_loggerqueue, password, event_idle, cfg): setproctitle("gzbx." + os.path.basename(__file__)) logger = mplogging.setup_logger(mp_loggerqueue, __file__) logger.debug(whoami() + "starting ...") pwdb = PWDBSender() event_idle.clear() cwd0 = os.getcwd() sh = SigHandler_Unrar(cwd0, logger) signal.signal(signal.SIGINT, sh.sighandler_unrar) signal.signal(signal.SIGTERM, sh.sighandler_unrar) try: os.chdir(directory) except FileNotFoundError: os.mkdir(directory) logger.info(whoami() + "started partial_unrar") # get already present rar files rar_sortedlist0 = None while not TERMINATED: rar_sortedlist0 = passworded_rars.get_sorted_rar_list(directory) # todo: what to do if does not finish here? if rar_sortedlist0: break time.sleep(1) if TERMINATED: logger.info(whoami() + "terminated!") return rar_sortedlist = [] # das muss besser gehen!!! mit rarlist als parameter von main/postproc for r1, r2 in rar_sortedlist0: try: rar_sortedlist.append((r1, r2.split("/")[-1])) except Exception as e: logger.debug(whoami() + whoami() + ": " + str(e)) pwdb.exc("db_nzb_update_unrar_status", [nzbname, 1], {}) nextrarname = rar_sortedlist[0][1] # first valid rar_sortedlist in place, start unrar! if password: cmd = "unrar x -y -o+ -p" + password + " '" + directory + nextrarname + "' '" + unpack_dir + "'" logger.debug(whoami() + "rar archive is passworded, executing " + cmd) pwdb.exc("db_msg_insert", [nzbname, "unraring pw protected rar archive", "info"], {}) status = 1 child = pexpect.spawn(cmd) status, statmsg, str0 = process_next_unrar_child_pass( event_idle, child, logger) if status < 0: logger.info(whoami() + nextrarname + ": " + statmsg) pwdb.exc("db_msg_insert", [nzbname, "unrar " + nextrarname + " failed!", "error"], {}) else: pwdb.exc("db_msg_insert", [nzbname, "checking for double packed rars", "info"], {}) # check if double packed try: child.kill(signal.SIGKILL) except Exception: pass is_double_packed, fn = check_double_packed(unpack_dir) if is_double_packed: pwdb.exc("db_msg_insert", [ nzbname, "rars are double packed, starting unrar 2nd run", "warning" ], {}) logger.debug(whoami() + "rars are double packed, executing " + cmd) # unrar without pausing! cmd = "unrar x -y -o+ -p" + password + " '" + fn + "' '" + unpack_dir + "'" child = pexpect.spawn(cmd) status, statmsg, str0 = process_next_unrar_child_pass( event_idle, child, logger) if status < 0: logger.info(whoami() + "2nd pass: "******"db_msg_insert", [nzbname, "unrar 2nd pass failed!", "error"], {}) elif status == 0: statmsg = "All OK" status = 0 pwdb.exc("db_msg_insert", [ nzbname, "unrar success 2nd pass for all rar files!", "info" ], {}) logger.info(whoami() + "unrar success 2nd pass for all rar files!") logger.debug(whoami() + "deleting all rar files in unpack_dir") delete_all_rar_files(unpack_dir, logger) elif status == 1: status = -3 statmsg = "unknown error" logger.info(whoami() + "2nd pass: "******" / " + str0) pwdb.exc("db_msg_insert", [nzbname, "unrar 2nd pass failed!", "error"], {}) else: statmsg = "All OK" status = 0 pwdb.exc("db_msg_insert", [nzbname, "unrar success for all rar files!", "info"], {}) logger.info(whoami() + "unrar success for all rar files!") else: cmd = "unrar x -y -o+ -vp '" + directory + nextrarname + "' '" + unpack_dir + "'" logger.debug(whoami() + "rar archive is NOT passworded, executing " + cmd) pwdb.exc("db_msg_insert", [nzbname, "unraring rar archive", "info"], {}) child = pexpect.spawn(cmd) status = 1 # 1 ... running, 0 ... exited ok, -1 ... rar corrupt, -2 ..missing rar, -3 ... unknown error while not TERMINATED: oldnextrarname = nextrarname.split("/")[-1] #oldfoldersize = folder_size(path=unpack_dir) #old_t0 = time.time() status, statmsg, str0 = process_next_unrar_child_pass( event_idle, child, logger) #newfoldersize = folder_size(path=unpack_dir) #delta_size = (newfoldersize - oldfoldersize) / (1024 * 1024) #delta_t = time.time() - old_t0 #print(delta_size / delta_t, "M decompressed per sec") if status < 0: logger.info(whoami() + nextrarname + ": " + statmsg) pwdb.exc( "db_msg_insert", [nzbname, "unrar " + oldnextrarname + " failed!", "error"], {}) break logger.info(whoami() + nextrarname + ": unrar success!") if status == 0: pwdb.exc("db_msg_insert", [nzbname, "checking for double packed rars", "info"], {}) # check if double packed try: child.kill(signal.SIGKILL) except Exception: pass is_double_packed, fn = check_double_packed(unpack_dir) if is_double_packed: pwdb.exc("db_msg_insert", [ nzbname, "rars are double packed, starting unrar 2nd run", "warning" ], {}) cmd = "unrar x -y -o+ '" + fn + "' '" + unpack_dir + "'" logger.debug(whoami() + "rars are double packed, executing " + cmd) # unrar without pausing! child = pexpect.spawn(cmd) status, statmsg, str0 = process_next_unrar_child_pass( event_idle, child, logger) if status < 0: logger.info(whoami() + "2nd pass: "******"db_msg_insert", [nzbname, "unrar 2nd pass failed!", "error"], {}) break if status == 0: statmsg = "All OK" status = 0 pwdb.exc("db_msg_insert", [ nzbname, "unrar success 2nd pass for all rar files!", "info" ], {}) logger.info( whoami() + "unrar success 2nd pass for all rar files!") logger.debug(whoami() + "deleting all rar files in unpack_dir") delete_all_rar_files(unpack_dir, logger) break if status == 1: status = -3 statmsg = "unknown error" logger.info(whoami() + "2nd pass: "******" / " + str0) pwdb.exc("db_msg_insert", [nzbname, "unrar 2nd pass failed!", "error"], {}) break else: statmsg = "All OK" status = 0 pwdb.exc( "db_msg_insert", [nzbname, "unrar success for all rar files!", "info"], {}) logger.info(whoami() + "unrar success for all rar files!") break try: gg = re.search(r"Insert disk with ", str0, flags=re.IGNORECASE) gend = gg.span()[1] nextrarname = str0[gend:-19] except Exception as e: logger.warning(whoami() + str(e) + ", unknown error") statmsg = "unknown error in re evalution" status = -4 pwdb.exc( "db_msg_insert", [nzbname, "unrar " + oldnextrarname + " failed!", "error"], {}) break pwdb.exc( "db_msg_insert", [nzbname, "unrar " + oldnextrarname + " success!", "info"], {}) logger.debug(whoami() + "Waiting for next rar: " + nextrarname) # first, if .r00 try: nextrar_number = int( re.search(r"\d{0,9}$", nextrarname).group()) nextrar_wo_number = nextrarname.rstrip("0123456789") except Exception: nextrar_wo_number = nextrarname nextrar_number = -1 # if part01.rar if nextrar_number == -1: try: nextrar_number = int( nextrarname.split(".part")[-1].split(".rar")[0]) nextrar_wo_number = nextrarname.rstrip(".rar").rstrip( "0123456789").rstrip("part").rstrip(".") except Exception: nextrar_wo_number = nextrarname nextrar_number = -1 gotnextrar = False nextrar_short = nextrarname.split("/")[-1] # todo: hier deadlock/unendliches Warten im Postprocess vermeiden, wenn rar nicht auftaucht! event_idle.set() while not gotnextrar and not TERMINATED: time.sleep(1) for f0 in glob.glob(directory + "*") + glob.glob(directory + ".*"): if nextrarname == f0: gotnextrar = True # now check if we waited too long for next rar0 - but how? if not gotnextrar and nextrar_number != -1: for f0 in glob.glob(directory + "*") + glob.glob(directory + ".*"): f0_number = -1 f0_wo_number = f0 try: f0_number = int(re.search(r"\d{0,9}$", f0).group()) f0_wo_number = f0.rstrip("0123456789") except Exception: try: f0_number = int( f0.split(".part")[-1].split(".rar")[0]) f0_wo_number = f0.rstrip(".rar").rstrip( "0123456789").rstrip("part").rstrip(".") except Exception: continue if f0_number == -1: continue if f0_wo_number == nextrar_wo_number and f0_number > nextrar_number: pwdb.exc("db_msg_insert", [ nzbname, "unrar waiting for next rar, but next rar " + nextrar_short + " seems to be skipped, you may want to interrupt ...", "warning" ], {}) break event_idle.clear() if TERMINATED: break time.sleep(1) # achtung hack! child.sendline("C") try: child.kill(signal.SIGKILL) except Exception: pass if TERMINATED: logger.info(whoami() + "exited!") else: logger.info(whoami() + str(status) + " " + statmsg) try: os.chdir(cwd0) if status == 0: pwdb.exc("db_nzb_update_unrar_status", [nzbname, 2], {}) elif status == -5: pwdb.exc("db_nzb_update_unrar_status", [nzbname, -2], {}) else: pwdb.exc("db_nzb_update_unrar_status", [nzbname, -1], {}) except Exception as e: logger.warning(whoami() + str(e)) event_idle.clear() logger.info(whoami() + "exited!") sys.exit()
def run(self): self.socket.setsockopt(zmq.LINGER, 0) socketurl = "tcp://" + self.host + ":" + self.port self.socket.connect(socketurl) dl_running = True while not self.event_stopped.wait(self.delay): # some button pressed, of which main.py should be informed? GLib.idle_add(self.gui.update_logs_and_lists) try: queue_elem = self.guiqueue.get_nowait() self.guiqueue.task_done() except (queue.Empty, EOFError, ValueError): queue_elem = None except Exception as e: self.logger.error(whoami() + str(e)) queue_elem = None if queue_elem: elem_type, elem_val = queue_elem if elem_type == "order_changed": msg0 = "SET_NZB_ORDER" msg0_val = [nzb[0] for nzb in self.appdata.nzbs] elif elem_type == "interrupted": msg0 = "SET_NZB_INTERRUPT" msg0_val = [nzb[0] for nzb in self.appdata.nzbs] elif elem_type == "closeall": msg0 = "SET_CLOSEALL" msg0_val = elem_val # non-empty if apply / restart! self.appdata.closeall = True elif elem_type == "nzb_added": msg0 = "NZB_ADDED" msg0_val, add_button = elem_val elif elem_type == "deleted_from_history": msg0 = "DELETED_FROM_HISTORY" msg0_val = elem_val elif elem_type == "reprocess_from_start": msg0 = "REPROCESS_FROM_START" msg0_val = elem_val elif elem_type == "reprocess_from_last": msg0 = "REPROCESS_FROM_LAST" msg0_val = elem_val elif elem_type == "dl_running": msg0_val = None dl_running_new = elem_val if dl_running != dl_running_new: dl_running = dl_running_new if dl_running: msg0 = "SET_RESUME" else: msg0 = "SET_PAUSE" else: msg0 = None else: msg0 = None if msg0: try: self.socket.send_pyobj((msg0, msg0_val)) datatype, datarec = self.socket.recv_pyobj() except Exception as e: self.logger.error(whoami() + str(e)) if elem_type == "nzb_added": add_button.set_sensitive(True) elif elem_type == "closeall": with self.lock: self.appdata.closeall = False self.logger.debug( whoami() + "received main closeall confirm, shutting down guipoller" ) elif elem_type in ["order_changed", "interrupted"]: GLib.idle_add(self.toggle_buttons) self.logger.debug(whoami() + "order changed/interrupted ok!") elif elem_type in [ "deleted_from_history", "reprocess_from_start", "reprocess_from_last" ]: GLib.idle_add(self.toggle_buttons_history) self.logger.debug(whoami() + "deleted_from_history/reprocess ok!") else: self.logger.error(whoami() + "cannot interpret element in guiqueue") else: try: self.socket.send_pyobj(("REQ", None)) datatype, datarec = self.socket.recv_pyobj() if datatype == "NOOK": continue elif datatype == "DL_DATA": data, server_config, dl_running, nzb_status_string, \ article_health, dlconfig, gb_downloaded, server_ts = datarec try: GLib.idle_add(self.update_mainwindow, data, server_config, dl_running, nzb_status_string, article_health, dlconfig, gb_downloaded, server_ts) continue except Exception as e: self.logger.debug(whoami() + str(e)) except Exception as e: self.logger.error(whoami() + str(e)) # close socket, join queue & exit guipoller self.logger.debug(whoami() + "closing socket") try: self.socket.close() self.context.term() except Exception: self.logger.warning(whoami()) self.logger.debug(whoami() + "joining gui_queue") while True: try: queue_elem = self.guiqueue.get_nowait() self.guiqueue.task_done() except (queue.Empty, EOFError, ValueError): break except Exception as e: self.logger.error(whoami() + str(e)) break self.guiqueue.join() self.logger.info(whoami() + "exiting") sys.exit()
def mpconnector(child_pipe, cfg, server_ts, mp_loggerqueue): setproctitle("gzbx." + os.path.basename(__file__)) logger = mplogging.setup_logger(mp_loggerqueue, __file__) logger.debug(whoami() + "starting mpconnector process") sh = SigHandler_MPconnector(logger) signal.signal(signal.SIGINT, sh.sighandler_mpconnector) signal.signal(signal.SIGTERM, sh.sighandler_mpconnector) # setup ports + zmq streamer device worker_port = 37100 connections_port = worker_port + 1 while is_port_in_use(worker_port) or is_port_in_use(connections_port): worker_port += 1 connections_port = worker_port + 1 # streamerdevice gathers results downloaded from usenet servers and # keeps them ready to be delivered to mpconnector (which is the only worker). # this is not working with ProcessDevice -> stuck on sys.exit!?? streamerdevice = ThreadDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH) streamerdevice.bind_in("tcp://127.0.0.1:%d" % connections_port) streamerdevice.bind_out("tcp://127.0.0.1:%d" % worker_port) streamerdevice.setsockopt_in(zmq.IDENTITY, b"PULL") streamerdevice.setsockopt_out(zmq.IDENTITY, b"PUSH") streamerdevice.start() # setup socket for worker context = zmq.Context() socket = context.socket(zmq.PULL) socket.connect("tcp://127.0.0.1:%d" % worker_port) # poller to check if streamerdevice is empty poller = zmq.Poller() poller.register(socket, zmq.POLLIN) thr_articlequeue = deque() ct = ConnectionThreads(cfg, thr_articlequeue, connections_port, server_ts, logger) cmdlist = ("start", "stop", "pause", "resume", "reset_timestamps", "reset_timestamps_bdl", "get_downloaded_per_server", "exit", "clearqueues", "connection_thread_health", "get_server_config", "set_tmode_sanitycheck", "set_tmode_download", "get_level_servers", "clear_articlequeue", "queues_empty", "clear_resultqueue", "len_articlequeue", "push_articlequeue", "pull_resultqueue", "push_entire_articlequeue", "pull_entire_resultqueue", "get_bytesdownloaded") quit_via_cmdexit = False while not TERMINATED: try: cmd, param = child_pipe.recv() except Exception as e: logger.warning(whoami() + str(e)) continue if cmd in cmdlist: result = True if cmd == "push_articlequeue": try: thr_articlequeue.append(param) except Exception: result = None elif cmd == "push_entire_articlequeue": try: for article0 in param: thr_articlequeue.append(article0) except Exception: result = None elif cmd == "pull_entire_resultqueue": result = get_from_streamingdevice(socket, onlyfirst=False) logger.debug(whoami() + "len_articlequeue: " + str(len(ct.articlequeue))) elif cmd == "pull_resultqueue": result = get_from_streamingdevice(socket, onlyfirst=True) elif cmd == "start": ct.start_threads() elif cmd == "queues_empty": socks = dict(poller.poll(timeout=10)) streamerdevice_empty = True if socket in socks and socks[socket] == zmq.POLLIN: streamerdevice_empty = False result = (len(ct.articlequeue) == 0) and streamerdevice_empty elif cmd == "get_bytesdownloaded": result = ct.get_bytesdownloaded() elif cmd == "len_articlequeue": try: result = int(len(ct.articlequeue)) except Exception: result = 0 elif cmd == "clear_articlequeue": ct.articlequeue.clear() elif cmd == "clear_resultqueue": get_from_streamingdevice(socket, onlyfirst=False) elif cmd == "stop": ct.stop_threads() elif cmd == "resume": ct.resume_threads() elif cmd == "pause": ct.pause_threads() elif cmd == "reset_timestamps": ct.reset_timestamps() elif cmd == "reset_timestamps_bdl": ct.reset_timestamps_bdl() elif cmd == "get_downloaded_per_server": result = ct.get_downloaded_per_server() elif cmd == "connection_thread_health": result = ct.connection_thread_health() elif cmd == "get_server_config": result = ct.get_server_config() elif cmd == "set_tmode_sanitycheck": for t, _ in ct.threads: t.mode = "sanitycheck" elif cmd == "set_tmode_download": for t, _ in ct.threads: t.mode = "download" elif cmd == "get_level_servers": le_serv0 = [] try: retention = param for level, serverlist in ct.level_servers.items(): level_servers = serverlist le_dic = {} for le in level_servers: _, _, _, _, _, _, _, _, age, _ = ct.servers.get_single_server_config(le) le_dic[le] = age les = [le for le in level_servers if le_dic[le] > retention * 0.9] le_serv0.append(les) except Exception: pass result = le_serv0 elif cmd == "exit": quit_via_cmdexit = True break elif cmd == "clearqueues": ct.articlequeue.clear() get_from_streamingdevice(socket, onlyfirst=False) child_pipe.send(result) else: child_pipe.send(None) logger.debug(whoami() + "shutting down ...") ct.stop_threads() get_from_streamingdevice(socket, onlyfirst=False) if quit_via_cmdexit: child_pipe.send(result)
def sighandler_mpconnector(self, a, b): self.logger.info(whoami() + "terminating ...") global TERMINATED TERMINATED = True
def open_connection(self, server_name0, conn_nr): result = None for idx, (sn, cn, rt, nobj) in enumerate(self.all_connections): if sn == server_name0 and cn == conn_nr: if nobj: return nobj else: context = ssl.SSLContext(ssl.PROTOCOL_TLS) sc = self.get_single_server_config(server_name0) if sc: server_name, server_url, user, password, port, usessl, level, connections, retention, useserver\ = sc if useserver: try: self.logger.debug(whoami() + "Opening connection # " + str(conn_nr) + "to server " + server_name) if usessl: nntpobj = nntplib.NNTP_SSL( server_url, user=user, password=password, ssl_context=context, port=port, readermode=True) else: # this is just for ginzicut preloading if user.lower( ) == "ginzicut" or password.lower( ) == "ginzicut": nntpobj = nntplib.NNTP(server_url, port=port) else: nntpobj = nntplib.NNTP( server_url, user=user, password=password, readermode=True, port=port) self.logger.debug(whoami() + "Opened Connection #" + str(conn_nr) + " on server " + server_name0) result = nntpobj self.all_connections[idx] = (sn, cn, rt, nntpobj) break except Exception as e: self.logger.error(whoami() + "Server " + server_name0 + " connect error: " + str(e)) self.all_connections[idx] = (sn, cn, rt, None) break else: self.logger.error( whoami() + "Cannot get server config for server: " + server_name0) self.all_connections[idx] = (sn, cn, rt, None) break return result
def download_article(self, article_name, article_age): bytesdownloaded = 0 info0 = None if self.mode == "sanitycheck": try: resp, number, message_id = self.nntpobj.stat(article_name) if article_name != message_id: status = -1 else: status = 1 except Exception as e: self.logger.error(whoami() + str(e) + self.idn + " for article " + article_name) status = -1 return status, 0, 0 if self.server_retention < article_age * 0.95: self.logger.warning(whoami() + "Retention on " + self.server_name + " not sufficient for article " + article_name) return -1, 0, None try: resp, info = self.nntpobj.body(article_name) if resp.startswith("222"): status = 1 info0 = [inf + b"\r\n" if not inf.endswith(b"\r\n") else inf for inf in info.lines] bytesdownloaded = len(b''.join(info.lines)) else: self.logger.warning(whoami() + resp + ": could not find " + article_name + " on " + self.idn) status = 0 # nntpError 4xx - Command was syntactically correct but failed for some reason except nntplib.NNTPTemporaryError as e: errcode = e.response.strip()[:3] if errcode == "400": # server quits, new connection has to be established status = -2 else: status = 0 self.logger.warning(whoami() + e.response + ": could not find " + article_name + " on " + self.idn) # nntpError 5xx - Command unknown error except nntplib.NNTPPermanentError as e: errcode = e.response.strip()[:3] if errcode in ["503", "502"]: # timeout, closing connection status = -2 else: status = 0 self.logger.warning(whoami() + e.response + ": could not find " + article_name + " on " + self.idn) status = 0 except nntplib.NNTPError as e: status = 0 self.logger.warning(whoami() + e.response + ": could not find " + article_name + " on " + self.idn) except KeyboardInterrupt: status = -3 except socket.timeout: status = -2 self.logger.warning(whoami() + "socket.timeout on " + self.idn) except AttributeError as e: status = -2 self.logger.warning(whoami() + str(e) + ": " + article_name + " on " + self.idn) except BrokenPipeError as e: status = -2 self.logger.warning(whoami() + str(e) + ": " + article_name + " on " + self.idn) except Exception as e: if "write to closed file" in str(e): status = -2 else: status = 0 self.logger.warning(whoami() + str(e) + ": " + article_name + " on " + self.idn) # self.bandwidth_bytes += bytesdownloaded return status, bytesdownloaded, info0