def resume_threads(self):
if self.threads:
self.logger.debug(whoami() + "Resuming threads")
for t, _ in self.threads:
t.paused = False
else:
self.logger.debug(whoami() + "Starting threads")
self.start_threads()
def sighandler_unrar(self, a, b):
global TERMINATED
try:
os.chdir(self.wd)
except Exception as e:
self.logger.warning(whoami() + str(e))
self.logger.info(whoami() + "terminating ...")
TERMINATED = True
def close_all_connections(self):
for idx, (sn, cn, rt, nobj) in enumerate(self.all_connections):
if nobj:
try:
nobj.quit()
self.all_connections[idx] = (sn, cn, rt, None)
self.logger.warning(whoami() + "Closed connection #" +
str(cn) + " on " + sn)
except Exception as e:
self.logger.warning(whoami() + "Cannot quit server " + sn +
": " + str(e))
def start_threads(self):
if not self.threads:
self.logger.debug(whoami() + "starting download threads")
self.init_servers()
for sn, scon, _, _ in self.all_connections:
t = ConnectionWorker((sn, scon), self.articlequeue, self.port, self.servers,
self.cfg, self.logger)
self.threads.append((t, time.time()))
t.start()
else:
self.logger.debug(whoami() + "threads already started")
def close_connection(self, server_name0, conn_nr):
res = False
for idx, (sn, cn, rt, nobj) in enumerate(self.all_connections):
if sn == server_name0 and cn == conn_nr:
if nobj:
try:
self.logger.warning(whoami() + "Closed connection #" +
str(cn) + " on " + sn)
self.all_connections[idx] = (sn, cn, rt, None)
nobj.quit()
res = True
except Exception as e:
self.logger.error(whoami() + "Server " + server_name0 +
" close error: " + str(e))
res = False
break
return res
def update_fmodtime_nzbfiles(nzbfilelist, dirs, logger):
for nzbfile in nzbfilelist:
nzbfile_full = dirs["nzb"] + nzbfile
try:
f = open(nzbfile_full, "a") # trigger inotify "MODIFY"
f.write("<!--modified-->\n")
f.close()
except Exception as e:
logger.warning(whoami() + str(e))
return
def remove_nzbdirs(deleted_nzbs, dirs, pwdb, logger, removenzbfile=True):
for deleted_nzb in deleted_nzbs:
nzbdirname = re.sub(r"[.]nzb$", "", deleted_nzb,
flags=re.IGNORECASE) + "/"
if removenzbfile:
# delete nzb from .ginzibix/nzb
try:
os.rename(dirs["nzb"] + deleted_nzb,
dirs["nzb"] + deleted_nzb + ".bak")
logger.debug(whoami() + ": renamed NZB " + deleted_nzb +
" to .bak")
except Exception as e:
logger.warning(whoami() + str(e))
# remove incomplete/$nzb_name
try:
shutil.rmtree(dirs["incomplete"] + nzbdirname)
logger.debug(whoami() + ": deleted incomplete dir for " +
deleted_nzb)
except Exception as e:
logger.warning(whoami() + str(e))
def stop_threads(self):
if not self.threads:
self.logger.debug(whoami() + "no threads running, exiting ...")
return
try:
self.logger.debug(whoami() + "stopping download threads + servers")
for t, _ in self.threads:
t.stop()
t.last_downloaded_ts = None
for t, _ in self.threads:
t.join()
del self.threads
self.threads = []
if self.servers:
self.servers.close_all_connections()
del self.servers
self.servers = None
self.logger.debug(whoami() + "all threads / servers stopped")
except Exception as e:
self.logger.warning(whoami() + str(e))
def scan_for_par2(notrenamedfiles, logger):
p2obj0 = None
p2basename0 = None
for fn, _ in notrenamedfiles:
ptype0 = par2lib.check_for_par_filetype(fn)
if ptype0 == 1:
p2obj0 = par2lib.Par2File(fn)
p2basename0 = fn.split(".par2")[0]
logger.debug(whoami() + "Found .par2 in _downloaded0: " +
fn.split("/")[-1])
break
return p2obj0, p2basename0
def retry_connect(self):
idx = 0
self.logger.debug(whoami() + "Server " + self.idn + " connecting ...")
while idx < 5 and self.running and not self.paused:
try:
self.servers.close_connection(self.name, self.conn_nr)
except Exception as e:
self.logger.warning(whoami() + str(e) + ": cannot close " + self.idn)
self.nntpobj = self.servers.open_connection(self.name, self.conn_nr)
if self.nntpobj:
self.logger.debug(whoami() + "Server " + self.idn + " connected!")
self.last_timestamp = time.time()
self.connectionstate = 1
self.server_name, self.server_url, self.server_user, self.server_password, self.server_port,\
self.server_usessl, self.server_level, self.server_connections, self.server_retention,\
self.useserver = self.servers.get_single_server_config(self.connection[0])
self.wait_running(1)
return
self.logger.warning(whoami() + "Could not connect to server " + self.idn + ", will retry in 5 sec.")
self.wait_running(2)
if not self.running or self.paused:
break
idx += 1
if not self.running:
self.logger.warning(whoami() + "No connection retries anymore due to exiting")
else:
self.logger.error(whoami() + "Connect retries to " + self.idn + " failed!")
self.connectionstate = -1
def rename_and_move_rarandremainingfiles_old(p2obj, notrenamedfiles,
source_dir, dest_dir, pwdb,
renamer_result_queue,
filewrite_lock, logger):
if p2obj:
rarfileslist = [(fn, md5) for fn, md5 in p2obj.md5_16khash()
if par2lib.get_file_type(fn) == "rar"]
notrenamedfiles0 = notrenamedfiles[:]
# rarfiles
for a_name, a_md5 in notrenamedfiles0:
pp = (a_name, a_md5)
try:
r_name = [fn for fn, r_md5 in rarfileslist
if r_md5 == a_md5][0]
if not r_name:
continue
if r_name != a_name:
with filewrite_lock:
shutil.copyfile(source_dir + a_name, dest_dir + r_name)
else:
with filewrite_lock:
shutil.copyfile(source_dir + a_name, dest_dir + a_name)
r_name = a_name
# oldft = pwdb.db_file_get_orig_filetype(a_name)
oldft = pwdb.exc("db_file_get_orig_filetype", [a_name], {})
# pwdb.db_file_set_renamed_name(a_name, r_name)
pwdb.exc("db_file_set_renamed_name", [a_name, r_name], {})
pwdb.exc("db_file_set_file_type", [a_name, "rar"], {})
renamer_result_queue.put(
(r_name, dest_dir + r_name, "rar", a_name, oldft))
# os.rename(source_dir + a_name, source_dir + a_name + ".renamed")
with filewrite_lock:
os.remove(source_dir + a_name)
notrenamedfiles.remove(pp)
except IndexError:
pass
except Exception as e:
logger.warning(whoami() + str(e))
for a_name, a_md5 in notrenamedfiles:
with filewrite_lock:
shutil.copyfile(source_dir + a_name, dest_dir + a_name)
ft = par2lib.get_file_type(a_name)
# pwdb.db_file_set_renamed_name(a_name, a_name)
pwdb.exc("db_file_set_renamed_name", [a_name, a_name], {})
# pwdb.db_file_set_file_type(a_name, ft)
pwdb.exc("db_file_set_file_type", [a_name, ft], {})
renamer_result_queue.put((a_name, dest_dir + a_name, ft, a_name, ft))
# os.rename(source_dir + a_name, source_dir + a_name + ".renamed")
with filewrite_lock:
os.remove(source_dir + a_name)
def make_allfilelist_wait(pwdb, dirs, logger, timeout0):
# immediatley get allfileslist
try:
nzbname = pwdb.exc("make_allfilelist",
[dirs["incomplete"], dirs["nzb"]], {})
if nzbname:
logger.debug(whoami() + "no timeout, got nzb " + nzbname +
" immediately!")
return nzbname
elif timeout0 and timeout0 <= -1:
return None
except Exception as e:
logger.warning(whoami() + str(e))
return None
# setup inotify
logger.debug(whoami() + "waiting for new nzb with timeout=" +
str(timeout0))
t0 = time.time()
if not timeout0:
delay0 = 5
else:
delay0 = 1
while True:
try:
nzbname = pwdb.exc("make_allfilelist",
[dirs["incomplete"], dirs["nzb"]], {})
except Exception as e:
logger.warning(whoami() + str(e))
if nzbname:
logger.debug(whoami() + "new nzb found in db, queuing ...")
return nzbname
if timeout0:
if time.time() - t0 > timeout0 / 1000:
break
time.sleep(delay0)
return None
def multipartrar_repair(directory, parvolname, pwdb, nzbname, logger):
cwd0 = os.getcwd()
os.chdir(directory)
logger.info(whoami() + "checking if repair possible for " + parvolname)
pwdb.exc("db_msg_insert",
[nzbname, "checking if repair is possible", "info"], {})
ssh = subprocess.Popen(['par2verify', parvolname],
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
sshres = ssh.stdout.readlines()
repair_is_required = False
repair_is_possible = False
exitstatus = 0
for ss in sshres:
ss0 = ss.decode("utf-8")
if "Repair is required" in ss0:
repair_is_required = True
if "Repair is possible" in ss0:
repair_is_possible = True
if repair_is_possible and repair_is_required:
logger.info(whoami() +
"repair is required and possible, performing par2repair")
pwdb.exc("db_msg_insert", [
nzbname, "repair is required and possible, performing par2repair",
"info"
], {})
# repair
ssh = subprocess.Popen(['par2repair', parvolname],
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
sshres = ssh.stdout.readlines()
repair_complete = False
for ss in sshres:
ss0 = ss.decode("utf-8")
if "Repair complete" in ss0:
repair_complete = True
if not repair_complete:
exitstatus = -1
logger.error(whoami() + "could not repair")
else:
logger.info(whoami() + "repair success!!")
exitstatus = 1
elif repair_is_required and not repair_is_possible:
logger.error(whoami() + "repair is required but not possible!")
pwdb.exc("db_msg_insert",
[nzbname, "repair is required but not possible!", "error"],
{})
exitstatus = -1
elif not repair_is_required and not repair_is_possible:
logger.error(whoami() + "repair is not required - all OK!")
exitstatus = 1
os.chdir(cwd0)
return exitstatus
def rename_and_move_rarandremainingfiles(p2list, notrenamedfiles, source_dir,
dest_dir, pwdb, renamer_result_queue,
filewrite_lock, logger):
for _, _, _, rarfileslist in p2list:
notrenamedfiles0 = notrenamedfiles[:]
# rarfiles
for fullname, shortname, a_md5 in notrenamedfiles0:
pp = (fullname, shortname, a_md5)
try:
r_name = [fn for fn, r_md5 in rarfileslist
if r_md5 == a_md5][0]
if not r_name:
continue
if r_name != shortname:
with filewrite_lock:
shutil.copyfile(source_dir + shortname,
dest_dir + r_name)
else:
with filewrite_lock:
shutil.copyfile(source_dir + shortname,
dest_dir + shortname)
r_name = shortname
oldft = pwdb.exc("db_file_get_orig_filetype", [shortname], {})
pwdb.exc("db_file_set_renamed_name", [shortname, r_name], {})
pwdb.exc("db_file_set_file_type", [shortname, "rar"], {})
renamer_result_queue.put(
(r_name, dest_dir + r_name, "rar", shortname, oldft))
with filewrite_lock:
os.remove(source_dir + shortname)
notrenamedfiles.remove(pp)
except IndexError:
pass
except Exception as e:
logger.warning(whoami() + str(e))
for fullname, shortname, a_md5 in notrenamedfiles:
with filewrite_lock:
shutil.copyfile(source_dir + shortname, dest_dir + shortname)
ft = par2lib.get_file_type(fullname, inspect=True)
oldft = pwdb.exc("db_file_get_orig_filetype", [shortname], {})
pwdb.exc("db_file_set_renamed_name", [shortname, shortname], {})
pwdb.exc("db_file_set_file_type", [shortname, ft], {})
renamer_result_queue.put(
(shortname, dest_dir + shortname, ft, shortname, oldft))
with filewrite_lock:
os.remove(source_dir + shortname)
def scan_renamed_dir(renamed_dir, p2obj, filewrite_lock, logger):
# get all files in renamed
p2list = []
with filewrite_lock:
rn = []
p2obj0 = p2obj
p2basename0 = None
for fn in glob.glob(renamed_dir + "*") + glob.glob(renamed_dir + ".*"):
fn0 = fn.split("/")[-1]
ptype0 = par2lib.check_for_par_filetype(fn)
if ptype0 == 1:
p2obj0 = par2lib.Par2File(fn)
p2basename0 = fn.split(".par2")[0]
logger.debug(whoami() + "Found .par2 in _renamed0: " + fn0)
p2list.append((p2obj0, p2basename0))
rn.append(fn0)
return rn, p2obj0, p2basename0
def get_not_yet_renamed_files_new(dir0, pwdb, filewrite_lock, logger):
nrf = []
with filewrite_lock:
for fn in glob.glob(dir0 + "*") + glob.glob(dir0 + ".*"):
fn0 = fn.split("/")[-1]
rn = pwdb.exc("db_file_get_renamed_name", [fn0], {})
is_renamed = rn and (rn != "N/A")
if not is_renamed:
nrf.append((fn0, par2lib.calc_file_md5hash_16k(fn0)))
p2list = []
with filewrite_lock:
for fn in glob.glob(dir0 + "*") + fn in glob.glob(dir0 + ".*"):
fn0 = fn.split("/")[-1]
ptype0 = par2lib.check_for_par_filetype(fn)
if ptype0 == 1:
p2obj0 = par2lib.Par2File(fn)
p2basename0 = fn.split(".par2")[0]
logger.debug(whoami() + "Found .par2 in _renamed0: " + fn0)
p2list.append((p2obj0, p2basename0))
return nrf, p2list
def process_next_unrar_child_pass(event_idle, child, logger):
str0 = ""
timeout = False
while True:
try:
a = child.read_nonblocking(timeout=120).decode("utf-8")
str0 += a
except pexpect.exceptions.EOF:
break
except pexpect.exceptions.TIMEOUT:
timeout = True
break
except Exception as e:
logger.warning(whoami() + str(e))
if str0[-6:] == "[Q]uit":
break
if timeout:
statmsg = "pexpect.timeout exceeded"
status = -3
else:
status = 1
statmsg = ""
if "WARNING: You need to start extraction from a previous volume" in str0:
child.close(force=True)
statmsg = "WARNING: You need to start extraction from a previous volume"
status = -5
elif "error" in str0:
if "packed data checksum" in str0:
statmsg = "packed data checksum error (= corrupt rar!)"
status = -1
elif "- checksum error" in str0:
statmsg = "checksum error (= rar is missing!)"
status = -2
else:
statmsg = "unknown error"
status = -3
else:
if "All OK" in str0:
status = 0
statmsg = "All OK"
return status, statmsg, str0
def stop(self):
self.logger.debug(whoami() + "setting event_stopped")
self.event_stopped.set()
def decode_articles(mp_work_queue0, mp_loggerqueue, filewrite_lock):
setproctitle("gzbx." + os.path.basename(__file__))
logger = mplogging.setup_logger(mp_loggerqueue, __file__)
logger.info(whoami() + "starting article decoder process")
sh = SigHandler_Decoder(logger)
signal.signal(signal.SIGINT, sh.sighandler)
signal.signal(signal.SIGTERM, sh.sighandler)
pwdb = PWDBSender()
bytesfinal = bytearray()
while not TERMINATED:
res0 = None
decoder_set_idle(True)
while not TERMINATED:
try:
res0 = mp_work_queue0.get_nowait()
break
except (queue.Empty, EOFError):
pass
except Exception as e:
logger.warning(whoami() + str(e))
time.sleep(0.1)
if not res0 or TERMINATED:
logger.info(whoami() + "exiting decoder process!")
break
decoder_set_idle(False)
infolist, save_dir, filename, filetype = res0
logger.debug(whoami() + "starting decoding for " + filename)
# del bytes0
bytesfinal = bytearray()
status = 0 # 1: ok, 0: wrong yenc structure, -1: no crc32, -2: crc32 checksum error, -3: decoding error
statusmsg = "ok"
# ginzyenc
full_filename = save_dir + filename
i = 0
for info in infolist:
try:
lastline = info[-1].decode("latin-1")
m = re.search('size=(.\d+?) ', lastline)
if m:
size = int(m.group(1))
except Exception as e:
logger.warning(whoami() + str(e) + ", guestimate size ...")
size = int(sum(len(i) for i in info.lines) * 1.1)
try:
decoded_data, output_filename, crc, crc_yenc, crc_correct = ginzyenc.decode_usenet_chunks(
info, size)
#if filename.endswith("part01.rar") and i == 3:
# pass
#else:
bytesfinal.extend(decoded_data)
except Exception as e:
logger.warning(whoami() + str(e) + ": cannot perform ginzyenc")
status = -3
statusmsg = "ginzyenc decoding error!"
# continue decoding, maybe it can be repaired ...?
i += 1
logger.debug(whoami() + "decoding for " + filename + ": success!")
try:
with filewrite_lock:
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
with open(full_filename, "wb") as f0:
f0.write(bytesfinal)
f0.flush()
f0.close()
except Exception as e:
statusmsg = "file_error"
logger.error(whoami() + str(e) + " in file " + filename)
status = -4
logger.info(whoami() + filename + " decoded with status " +
str(status) + " / " + statusmsg)
pwdbstatus = 2
if status in [-3, -4]:
pwdbstatus = -1
try:
# pwdb.db_file_update_status(filename, pwdbstatus)
pwdb.exc("db_file_update_status", [filename, pwdbstatus], {})
logger.debug(whoami() + "updated DB for " + filename +
", db.status=" + str(pwdbstatus))
except Exception as e:
logger.error(whoami() + str(e) + ": cannot update DB for " +
filename)
i += 1
logger.debug(whoami() + "exited!")
def sighandler(self, a, b):
global TERMINATED
self.logger.info(whoami() + "terminating ...")
TERMINATED = True
def get_password(directory, pw_file, nzbname0, logger, get_pw_direct=False):
if directory[-1] != "/":
directory += "/"
rars = get_sorted_rar_list(directory)
if not rars:
return None
rarname0 = rars[0][1]
rarname = rarname0.split("/")[-1]
nzbname = nzbname0.split(".nzb")[0]
logger.debug(whoami() + "trying to get password")
cwd0 = os.getcwd()
if get_pw_direct:
gg = re.search(r"}}.nzb$", nzbname0, flags=re.IGNORECASE)
if gg:
try:
pw0 = nzbname0[:gg.start()].split("{{")[-1]
os.chdir(directory)
if test_password(pw0, rarname):
logger.info(whoami() + "Found PW for NZB " + nzbname +
": " + pw0)
os.chdir(cwd0)
return pw0
else:
logger.warning(whoami() +
"provided password was not correct!")
except Exception as e:
logger.debug(whoami() + str(e) +
": cannot get pw from nzb string")
os.chdir(cwd0)
# PW file format:
# a) pw
# pw
# pw
# b) filename1 <:::> pw1
# filename2 <:::> pw2
if not pw_file:
return None
logger.debug(whoami() + "reading passwords in " + pw_file)
try:
with open(pw_file, "r") as f0:
pw_list = f0.readlines()
except Exception as e:
logger.warning(whoami() + str(e) + ": cannot open/read pw file")
return None
cwd0 = os.getcwd()
os.chdir(directory)
pwlist = [pw.rstrip("\n") for pw in pw_list]
PW = None
# first try with <:::> if exists
logger.info(whoami() + "trying specified password entries <:::> ...")
for pw in pwlist:
if "<:::>" not in pw:
continue
fn0 = pw.split("<:::>")[0].lstrip(" ").rstrip(" ")
fn0 = fn0.split(".nzb")[0]
pw0 = pw.split("<:::>")[1].lstrip(" ").rstrip(" ")
pw0 = pw0.split(".nzb")[0]
if fn0 != nzbname:
continue
logger.debug(whoami() + "Trying with entry: " + fn0 + " / " + pw0 +
" for NZB " + nzbname)
if test_password(pw0, rarname):
PW = pw0
logger.info(whoami() + "Found PW for NZB " + nzbname + ": " + PW)
break
if PW:
os.chdir(cwd0)
return PW
# try
logger.info(whoami() + "trying free password file entries ...")
for pw in pwlist:
if "<:::>" in pw:
continue
if test_password(pw, rarname):
PW = pw
logger.info(whoami() + "Found PW for NZB " + nzbname + ": " + PW)
break
os.chdir(cwd0)
return PW
def postprocess_nzb(nzbname, articlequeue, resultqueue, mp_work_queue, pipes,
mpp0, mp_events, cfg, verifiedrar_dir, unpack_dir, nzbdir,
rename_dir, main_dir, download_dir, dirs, pw_file,
mp_loggerqueue):
setproctitle("gzbx." + os.path.basename(__file__))
pwdb = PWDBSender()
if pwdb.exc("db_nzb_getstatus", [nzbname], {}) in [4, -4]:
sys.exit()
logger = setup_logger(mp_loggerqueue, __file__)
logger.debug(whoami() + "starting ...")
nzbdirname = re.sub(r"[.]nzb$", "", nzbname, flags=re.IGNORECASE) + "/"
incompletedir = dirs["incomplete"] + nzbdirname
if not os.path.isdir(incompletedir):
logger.error(whoami() + "no incomplete_dir, aborting postprocessing")
pwdb.exc("db_nzb_update_status", [nzbname, -4], {})
sys.exit()
mpp = mpp0.copy()
sh = SigHandler_Postprocessing(logger)
signal.signal(signal.SIGINT, sh.sighandler_postprocessing)
signal.signal(signal.SIGTERM, sh.sighandler_postprocessing)
stop_wait(nzbname, dirs, pwdb)
event_verifieridle = mp_events["verifier"]
event_unrareridle = mp_events["unrarer"]
pwdb.exc("db_msg_insert", [nzbname, "starting postprocess", "info"], {})
logger.debug(whoami() + "starting clearing queues & pipes")
# clear pipes
do_mpconnections(pipes, "clearqueues", None)
try:
for key, item in pipes.items():
if pipes[key][0].poll():
pipes[key][0].recv()
except Exception as e:
logger.error(whoami() + str(e))
pwdb.exc("db_nzb_update_status", [nzbname, -4], {})
pwdb.exc("db_msg_insert",
[nzbname, "postprocessing/clearing pipes failed!", "error"],
{})
sys.exit()
logger.debug(whoami() + "clearing queues & pipes done!")
# join decoder
if mpp_is_alive(mpp, "decoder"):
t0 = time.time()
timeout_reached = False
try:
while mp_work_queue.qsize() > 0:
if time.time() - t0 > 60 * 3:
timeout_reached = True
break
time.sleep(0.5)
except Exception as e:
logger.debug(whoami() + str(e))
timeout_reached = True
if timeout_reached:
logger.warning(
whoami() +
"Timeout reached/error in joining decoder, terminating decoder ..."
)
kill_mpp(mpp, "decoder")
logger.info(whoami() + "decoder terminated!")
try:
while not mp_work_queue.empty():
mp_work_queue.get_no_wait()
except Exception:
pass
stop_wait(nzbname, dirs, pwdb)
# --- PAR_VERIFIER ---
verifystatus = pwdb.exc("db_nzb_get_verifystatus", [nzbname], {})
all_rars_are_verified, _ = pwdb.exc("db_only_verified_rars", [nzbname], {})
renamed_rar_files = pwdb.exc("get_all_renamed_rar_files", [nzbname], {})
# verifier not running, status = 0 --> start & wait
if not mpp_is_alive(
mpp, "verifier"
) and verifystatus == 0 and not all_rars_are_verified and renamed_rar_files:
logger.info(
whoami() +
"there are files to check by par_verifier, starting par_verifier ..."
)
p2 = pwdb.exc("get_renamed_p2", [rename_dir, nzbname], {})
if p2:
pvmode = "verify"
else:
pvmode = "copy"
mpp_verifier = mp.Process(target=par_verifier.par_verifier,
args=(
pipes["verifier"][1],
rename_dir,
verifiedrar_dir,
main_dir,
mp_loggerqueue,
nzbname,
pvmode,
event_verifieridle,
cfg,
))
mpp_verifier.start()
mpp["verifier"] = mpp_verifier
time.sleep(1)
stop_wait(nzbname, dirs, pwdb)
verifystatus = pwdb.exc("db_nzb_get_verifystatus", [nzbname], {})
# verifier is running, status == 1 (running) or -2 -> wait/join
if mpp_is_alive(mpp, "verifier") and verifystatus in [1, -2]:
logger.info(whoami() + "Waiting for par_verifier to complete")
try:
# kill par_verifier in deadlock
while True:
mpp_join(mpp, "verifier", timeout=5)
if mpp_is_alive(mpp, "verifier"):
# if not finished, check if idle longer than 5 sec -> deadlock!!!
t0 = time.time()
while event_verifieridle.is_set(
) and time.time() - t0 < 30:
time.sleep(0.5)
if time.time() - t0 >= 30:
logger.info(whoami() +
"Verifier deadlock, killing verifier!")
kill_mpp(mpp, "verifier")
break
else:
continue
else:
break
except Exception as e:
logger.warning(whoami() + str(e))
mpp_join(mpp, "verifier")
mpp["verifier"] = None
logger.debug(whoami() + "par_verifier completed/terminated!")
# if verifier is running but wrong status, or correct status but not running, exit
elif mpp_is_alive(mpp, "verifier") or verifystatus == 1:
pwdb.exc("db_nzb_update_status", [nzbname, -4], {})
pwdb.exc("db_msg_insert",
[nzbname, "par_verifier status inconsistency", "error"], {})
logger.debug(
whoami() +
"something is wrong with par_verifier, exiting postprocessor")
logger.info(whoami() + "postprocess of NZB " + nzbname + " failed!")
if mpp_is_alive(mpp, "verifier"):
logger.debug(whoami() + "terminating par_verifier")
kill_mpp(mpp, "verifier")
logger.info(whoami() + "verifier terminated!")
sys.exit()
stop_wait(nzbname, dirs, pwdb)
# --- UNRARER ---
# already checked if pw protected?
is_pw_checked = pwdb.exc("db_nzb_get_ispw_checked", [nzbname], {})
if not is_pw_checked:
logger.debug(whoami() + "testing if pw protected")
ispw = passworded_rars.is_rar_password_protected(
verifiedrar_dir, logger)
pwdb.exc("db_nzb_set_ispw_checked", [nzbname, True], {})
if ispw == 0 or ispw == -2 or ispw == -3:
logger.warning(
whoami() +
"cannot test rar if pw protected, something is wrong: " +
str(ispw) + ", exiting ...")
pwdb.exc("db_msg_insert", [
nzbname, "postprocessing failed due to pw test not possible",
"error"
], {})
pwdb.exc("db_nzb_update_status", [nzbname, -4], {})
sys.exit()
elif ispw == 1:
pwdb.exc("db_nzb_set_ispw", [nzbname, True], {})
pwdb.exc("db_msg_insert",
[nzbname, "rar archive is password protected", "warning"],
{})
logger.info(whoami() + "rar archive is pw protected")
elif ispw == -1:
pwdb.exc("db_nzb_set_ispw", [nzbname, False], {})
pwdb.exc(
"db_msg_insert",
[nzbname, "rar archive is NOT password protected", "info"], {})
logger.info(whoami() + "rar archive is NOT pw protected")
ispw = pwdb.exc("db_nzb_get_ispw", [nzbname], {})
unrarernewstarted = False
if ispw:
get_pw_direct0 = False
try:
get_pw_direct0 = (
cfg["OPTIONS"]["GET_PW_DIRECTLY"].lower() == "yes")
except Exception as e:
logger.warning(whoami() + str(e))
if pwdb.exc("db_nzb_get_password", [nzbname], {}) == "N/A":
logger.info(whoami() +
"Trying to get password from file for NZB " + nzbname)
pwdb.exc("db_msg_insert",
[nzbname, "trying to get password", "info"], {})
pw = passworded_rars.get_password(verifiedrar_dir,
pw_file,
nzbname,
logger,
get_pw_direct=get_pw_direct0)
if pw:
logger.info(whoami() + "Found password " + pw + " for NZB " +
nzbname)
pwdb.exc("db_msg_insert",
[nzbname, "found password " + pw, "info"], {})
pwdb.exc("db_nzb_set_password", [nzbname, pw], {})
else:
pw = pwdb.exc("db_nzb_get_password", [nzbname], {})
if not pw:
pwdb.exc("db_msg_insert", [
nzbname,
"Provided password was not correct / no password found in PW file! ",
"error"
], {})
logger.error(whoami() + "Cannot find password for NZB " + nzbname +
"in postprocess, exiting ...")
pwdb.exc("db_nzb_update_status", [nzbname, -4], {})
mpp["unrarer"] = None
# sighandler.mpp = mpp
sys.exit()
event_unrareridle = mp.Event()
# passing p2list to unrarer
mpp_unrarer = mp.Process(target=partial_unrar.partial_unrar,
args=(
verifiedrar_dir,
unpack_dir,
nzbname,
mp_loggerqueue,
pw,
event_unrareridle,
cfg,
))
unrarernewstarted = True
mpp_unrarer.start()
mpp["unrarer"] = mpp_unrarer
# sighandler.mpp = self.mpp
# start unrarer if never started and ok verified/repaired
elif not mpp["unrarer"]:
logger.debug(whoami() + "checking if unrarer should be started")
try:
verifystatus = pwdb.exc("db_nzb_get_verifystatus", [nzbname], {})
unrarstatus = pwdb.exc("db_nzb_get_unrarstatus", [nzbname], {})
except Exception as e:
logger.warning(whoami() + str(e))
logger.debug(whoami() + "verifystatus: " + str(verifystatus) +
" / unrarstatus: " + str(unrarstatus))
if (verifystatus > 0 or verifystatus == -2) and unrarstatus == 0:
try:
logger.debug(whoami() +
"unrarer passiv until now, starting ...")
event_unrareridle = mp.Event()
# passing p2list to unrarer
mpp_unrarer = mp.Process(target=partial_unrar.partial_unrar,
args=(
verifiedrar_dir,
unpack_dir,
nzbname,
mp_loggerqueue,
None,
event_unrareridle,
cfg,
))
unrarernewstarted = True
mpp_unrarer.start()
mpp["unrarer"] = mpp_unrarer
except Exception as e:
logger.warning(whoami() + str(e))
finalverifierstate = (pwdb.exc("db_nzb_get_verifystatus", [nzbname], {})
in [0, 2])
stop_wait(nzbname, dirs, pwdb)
# join unrarer
if mpp_is_alive(mpp, "unrarer"):
if finalverifierstate:
logger.info(whoami() + "Waiting for unrar to complete")
while True:
# try to join unrarer
mpp_join(mpp, "unrarer", timeout=5)
isalive = mpp_is_alive(mpp, "unrarer")
if isalive:
# if not finished, check if idle longer than 5 sec -> deadlock!!!
t0 = time.time()
timeout0 = 99999999 if unrarernewstarted else 120 * 2
while event_unrareridle.is_set(
) and time.time() - t0 < timeout0:
time.sleep(0.5)
if time.time() - t0 >= timeout0:
logger.info(whoami() +
"Unrarer deadlock, killing unrarer!")
kill_mpp(mpp, "unrarer")
break
else:
logger.debug(
whoami() +
"Unrarer not idle, waiting before terminating")
stop_wait(nzbname, dirs, pwdb)
time.sleep(0.5)
continue
else:
break
else:
logger.info(whoami() +
"Repair/unrar not possible, killing unrarer!")
kill_mpp(mpp, "unrarer")
logger.debug(whoami() + "unrarer completed/terminated!")
stop_wait(nzbname, dirs, pwdb)
# get status
finalverifierstate = (pwdb.exc("db_nzb_get_verifystatus", [nzbname], {})
in [0, 2])
finalnonrarstate = pwdb.exc("db_allnonrarfiles_getstate", [nzbname], {})
finalrarstate = (pwdb.exc("db_nzb_get_unrarstatus", [nzbname], {})
in [0, 2])
logger.info(whoami() + "Finalverifierstate: " + str(finalverifierstate) +
" / Finalrarstate: " + str(finalrarstate) +
" / Finalnonrarstate: " + str(finalnonrarstate))
if finalrarstate and finalnonrarstate and finalverifierstate:
pwdb.exc("db_msg_insert", [nzbname, "unrar/par-repair ok!", "success"],
{})
logger.info(whoami() + "unrar/par-repair of NZB " + nzbname +
" success!")
else:
pwdb.exc("db_nzb_update_status", [nzbname, -4], {})
pwdb.exc("db_msg_insert",
[nzbname, "unrar/par-repair failed!", "error"], {})
logger.info(whoami() + "postprocess of NZB " + nzbname + " failed!")
sys.exit()
stop_wait(nzbname, dirs, pwdb)
# copy to complete
logger.info(whoami() + "starting copy-to-complete")
pwdb.exc("db_msg_insert",
[nzbname, "copying & cleaning directories", "info"], {})
complete_dir = make_complete_dir(dirs, nzbdir, logger)
if not complete_dir:
pwdb.exc("db_nzb_update_status", [nzbname, -4], {})
pwdb.exc("db_msg_insert", [nzbname, "postprocessing failed!", "error"],
{})
logger.info("Cannot create complete_dir for " + nzbname +
", exiting ...")
pwdb.exc("db_msg_insert", [nzbname, "postprocessing failed!", "error"],
{})
sys.exit()
# move all non-rar/par2/par2vol files from renamed to complete
for f00 in glob.glob(rename_dir + "*") + glob.glob(rename_dir + ".*"):
logger.debug(whoami() + "renamed_dir: checking " + f00 + " / " +
str(os.path.isdir(f00)))
if os.path.isdir(f00):
logger.debug(f00 + "is a directory, skipping")
continue
f0 = f00.split("/")[-1]
file0type = pwdb.exc("db_file_getftype_renamed", [f0], {})
logger.debug(whoami() + "Moving/deleting " + f0)
if not file0type:
gg = re.search(r"[0-9]+[.]rar[.]+[0-9]", f0, flags=re.IGNORECASE)
if gg:
try:
os.remove(f00)
logger.debug(whoami() + "Removed rar.x file " + f0)
except Exception as e:
pwdb.exc("db_nzb_update_status", [nzbname, -4], {})
logger.warning(whoami() + str(e) +
": cannot remove corrupt rar file!")
else: # if unknown file (not in db) move to complete anyway
try:
shutil.move(f00, complete_dir)
logger.debug(whoami() + "moved " + f00 + " to " +
complete_dir)
except Exception as e:
logger.warning(whoami() + str(e) +
": cannot move unknown file to complete!")
continue
if file0type in ["rar", "par2", "par2vol"]:
try:
os.remove(f00)
logger.debug(whoami() + "removed rar/par2 file " + f0)
except Exception as e:
pwdb.exc("db_nzb_update_status", [nzbname, -4], {})
logger.warning(whoami() + str(e) +
": cannot remove rar/par2 file!")
else:
try:
shutil.move(f00, complete_dir)
logger.debug(whoami() + "moved non-rar/non-par2 file " + f0 +
" to complete")
except Exception as e:
pwdb.exc("db_nzb_update_status", [nzbname, -4], {})
logger.warning(whoami() + str(e) +
": cannot move non-rar/non-par2 file " + f00 +
"!")
stop_wait(nzbname, dirs, pwdb)
# remove download_dir
try:
shutil.rmtree(download_dir)
except Exception as e:
pwdb.exc("db_nzb_update_status", [nzbname, -4], {})
logger.warning(whoami() + str(e) + ": cannot remove download_dir!")
# move content of unpack dir to complete
logger.debug(whoami() + "moving unpack_dir to complete: " + unpack_dir)
for f00 in glob.glob(unpack_dir + "*") + glob.glob(unpack_dir + ".*"):
logger.debug(whoami() + "unpack_dir: checking " + f00 + " / " +
str(os.path.isdir(f00)))
d0 = f00.split("/")[-1]
logger.debug(whoami() + "Does " + complete_dir + d0 +
" already exist?")
if os.path.isfile(complete_dir + d0):
try:
logger.debug(whoami() + complete_dir + d0 +
" already exists, deleting!")
os.remove(complete_dir + d0)
except Exception:
logger.debug(whoami() + f00 +
" already exists but cannot delete")
pwdb.exc("db_nzb_update_status", [nzbname, -4], {})
break
else:
logger.debug(whoami() + complete_dir + d0 + " does not exist!")
if not os.path.isdir(f00):
try:
shutil.move(f00, complete_dir)
logger.debug(whoami() + ": moved " + f00 + " to " +
complete_dir)
except Exception as e:
pwdb.exc("db_nzb_update_status", [nzbname, -4], {})
logger.warning(whoami() + str(e) +
": cannot move unrared file to complete dir!")
else:
if os.path.isdir(complete_dir + d0):
try:
shutil.rmtree(complete_dir + d0)
logger.debug(whoami() + ": removed tree " + complete_dir +
d0)
except Exception as e:
pwdb.exc("db_nzb_update_status", [nzbname, -4], {})
logger.warning(whoami() + str(e) +
": cannot remove unrared dir in complete!")
try:
logger.debug(whoami() + "copying tree " + f00 + " to " +
complete_dir + d0)
shutil.copytree(f00, complete_dir + d0)
logger.debug(whoami() + "copied tree " + f00 + " to " +
complete_dir + d0)
except Exception as e:
pwdb.exc("db_nzb_update_status", [nzbname, -4], {})
logger.warning(whoami() + str(e) +
": cannot move non-rar/non-par2 file!")
# remove unpack_dir
logger.debug(whoami() +
": removing unpacl_dir, verified_rardir and incomplete_dir")
if pwdb.exc("db_nzb_getstatus", [nzbname], {}) != -4:
try:
shutil.rmtree(unpack_dir)
shutil.rmtree(verifiedrar_dir)
logger.debug(whoami() + ": removed " + unpack_dir +
verifiedrar_dir)
except Exception as e:
pwdb.exc("db_nzb_update_status", [nzbname, -4], {})
logger.warning(whoami() + str(e) +
": cannot remove unpack_dir / verifiedrar_dir")
# remove incomplete_dir
if pwdb.exc("db_nzb_getstatus", [nzbname], {}) != -4:
try:
shutil.rmtree(main_dir)
logger.debug(whoami() + ": removed " + main_dir)
except Exception as e:
pwdb.exc("db_nzb_update_status", [nzbname, -4], {})
logger.warning(whoami() + str(e) +
": cannot remove incomplete_dir!")
# finalize
if pwdb.exc("db_nzb_getstatus", [nzbname], {}) == -4:
logger.info(whoami() + "Copy/Move of NZB " + nzbname + " failed!")
else:
logger.info(whoami() + "Copy/Move of NZB " + nzbname + " success!")
pwdb.exc("db_nzb_update_status", [nzbname, 4], {})
logger.info(whoami() + "Postprocess of NZB " + nzbname + " success!")
sys.exit()
def par_verifier(child_pipe, renamed_dir, verifiedrar_dir, main_dir,
mp_loggerqueue, nzbname, pvmode, event_idle, cfg):
setproctitle("gzbx." + os.path.basename(__file__))
logger = mplogging.setup_logger(mp_loggerqueue, __file__)
logger.debug(whoami() + "starting ...")
sh = SigHandler_Verifier(logger)
signal.signal(signal.SIGINT, sh.sighandler_verifier)
signal.signal(signal.SIGTERM, sh.sighandler_verifier)
pwdb = PWDBSender()
event_idle.clear()
if pvmode == "verify":
# p2 = pwdb.get_renamed_p2(renamed_dir, nzbname)
try:
p2list = pwdb.exc("db_p2_get_p2list", [nzbname], {})
p2 = P2(p2list)
except Exception as e:
logger.warning(whoami() + str(e))
# pwdb.db_nzb_update_verify_status(nzbname, 1)
pwdb.exc("db_nzb_update_verify_status", [nzbname, 1], {})
# a: verify all unverified files in "renamed"
unverified_rarfiles = None
try:
# unverified_rarfiles = pwdb.get_all_renamed_rar_files(nzbname)
unverified_rarfiles = pwdb.exc("get_all_renamed_rar_files", [nzbname],
{})
except Exception as e:
logger.debug(whoami() + str(e) +
": no unverified rarfiles met in first run, skipping!")
doloadpar2vols = False
if pvmode == "verify" and not p2:
logger.debug(whoami() + "no par2 file found")
if pvmode == "verify" and unverified_rarfiles and p2:
logger.debug(whoami() + "verifying all unchecked rarfiles")
for filename, f_origname in unverified_rarfiles:
f_short = filename.split("/")[-1]
md5 = calc_file_md5hash(renamed_dir + filename)
md5match = [(pmd5 == md5) for pname, pmd5 in p2.filenames()
if pname == filename]
if False in md5match:
logger.warning(whoami() +
" error in md5 hash match for file " + f_short)
pwdb.exc("db_msg_insert", [
nzbname, "error in md5 hash match for file " + f_short,
"warning"
], {})
pwdb.exc("db_nzb_update_verify_status", [nzbname, -2], {})
pwdb.exc("db_file_update_parstatus", [f_origname, -1], {})
child_pipe.send(True)
else:
logger.info(whoami() + f_short +
" md5 hash match ok, copying to verified_rar dir")
pwdb.exc("db_msg_insert", [
nzbname, f_short +
" md5 hash match ok, copying to verified_rar dir ", "info"
], {})
shutil.copy(renamed_dir + filename, verifiedrar_dir)
pwdb.exc("db_file_update_parstatus", [f_origname, 1], {})
elif (pvmode == "verify" and not p2) or (pvmode == "copy"):
logger.info(whoami() + "copying all rarfiles")
for filename, f_origname in unverified_rarfiles:
f_short = filename.split("/")[-1]
sfvcheck = pwdb.exc("db_nzb_check_sfvcrc32",
[nzbname, renamed_dir, renamed_dir + filename],
{})
if sfvcheck == -1:
logger.warning(whoami() + " error in crc32 check for file " +
f_short)
pwdb.exc("db_msg_insert", [
nzbname, "error in crc32 check for file " + f_short,
"warning"
], {})
pwdb.exc("db_nzb_update_verify_status", [nzbname, -2], {})
pwdb.exc("db_file_update_parstatus", [f_origname, -1], {})
child_pipe.send(True)
continue
logger.debug(whoami() + "copying " + f_short +
" to verified_rar dir")
pwdb.exc("db_msg_insert", [
nzbname, "copying " + f_short + " to verified_rar dir ", "info"
], {})
shutil.copy(renamed_dir + filename, verifiedrar_dir)
pwdb.exc("db_file_update_parstatus", [f_origname, 1], {})
# b: inotify renamed_dir
inotify = inotify_simple.INotify()
watch_flags = inotify_simple.flags.CREATE | inotify_simple.flags.DELETE | inotify_simple.flags.MODIFY | inotify_simple.flags.DELETE_SELF
inotify.add_watch(renamed_dir, watch_flags)
while not TERMINATED:
# allparstatus = pwdb.db_file_getallparstatus(0)
allparstatus = pwdb.exc("db_file_getallparstatus", [0], {})
if 0 not in allparstatus:
event_idle.clear()
logger.info(whoami() +
"all renamed rars checked, exiting par_verifier")
break
events = get_inotify_events(inotify)
event_idle.set()
if events or 0 in allparstatus:
event_idle.clear()
if pvmode == "verify" and not p2:
try:
p2list = pwdb.exc("db_p2_get_p2list", [nzbname], {})
p2 = P2(p2list)
except Exception as e:
logger.debug(whoami() + str(e))
if pvmode == "verify" and p2:
for rar in glob.glob(renamed_dir +
"*") + glob.glob(renamed_dir + ".*"):
rar0 = rar.split("/")[-1]
f0 = pwdb.exc("db_file_get_renamed", [rar0], {})
if not f0:
continue
f0_origname, f0_renamedname, f0_ftype = f0
if not f0_ftype == "rar":
continue
if pwdb.exc("db_file_getparstatus", [rar0],
{}) == 0 and f0_renamedname != "N/A":
f_short = f0_renamedname.split("/")[-1]
md5 = calc_file_md5hash(renamed_dir + rar0)
md5match = [(pmd5 == md5)
for pname, pmd5 in p2.filenames()
if pname == f0_renamedname]
#print(f0_renamedname, md5, " : ", p2.filenames())
#print(md5match)
#print("-" * 80)
if True in md5match:
logger.info(
whoami() + f_short +
" md5 hash match ok, copying to verified_rar dir"
)
pwdb.exc("db_msg_insert", [
nzbname, f_short +
" md5 hash match ok, copying to verified_rar dir ",
"info"
], {})
shutil.copy(renamed_dir + f0_renamedname,
verifiedrar_dir)
pwdb.exc("db_file_update_parstatus",
[f0_origname, 1], {})
elif False in md5match:
logger.warning(
whoami() +
"error in md5 hash match for file " + f_short)
pwdb.exc("db_msg_insert", [
nzbname, "error in md5 hash match for file " +
f_short, "warning"
], {})
pwdb.exc("db_nzb_update_verify_status",
[nzbname, -2], {})
pwdb.exc("db_file_update_parstatus",
[f0_origname, -1], {})
child_pipe.send(True)
else: # if no match at all in p2list -> try sfvcheck / or just copy
sfvcheck = pwdb.exc("db_nzb_check_sfvcrc32",
[nzbname, renamed_dir, rar],
{})
if sfvcheck == -1:
logger.warning(
whoami() +
" error in crc32 check for file " + rar0)
pwdb.exc("db_msg_insert", [
nzbname, "error in crc32 check for file " +
rar0, "warning"
], {})
pwdb.exc("db_nzb_update_verify_status",
[nzbname, -2], {})
pwdb.exc("db_file_update_parstatus",
[f0_origname, -1], {})
child_pipe.send(True)
continue
if pwdb.exc("db_file_getparstatus", [rar0],
{}) == 0 and f0_renamedname != "N/A":
logger.debug(whoami() +
"no md5 check, copying " +
f0_renamedname.split("/")[-1] +
" to verified_rar dir")
pwdb.exc("db_msg_insert", [
nzbname, "no md5 check, copying " +
f0_renamedname.split("/")[-1] +
" to verified_rar dir", "info"
], {})
shutil.copy(renamed_dir + f0_renamedname,
verifiedrar_dir)
pwdb.exc("db_file_update_parstatus",
[f0_origname, 1], {})
# free rars or copy mode?
elif (pvmode == "verify" and not p2) or (pvmode == "copy"):
# maybe we can check via sfv file?
for file0full in glob.glob(renamed_dir +
"*") + glob.glob(renamed_dir +
".*"):
file0short = file0full.split("/")[-1]
ft = pwdb.exc("db_file_getftype_renamed", [file0short], {})
if ft == "rar":
rar0 = file0short
f0 = pwdb.exc("db_file_get_renamed", [rar0], {})
if not f0:
continue
f0_origname, f0_renamedname, f0_ftype = f0
sfvcheck = pwdb.exc("db_nzb_check_sfvcrc32",
[nzbname, renamed_dir, file0full],
{})
if sfvcheck == -1:
logger.warning(whoami() +
" error in crc32 check for file " +
rar0)
pwdb.exc("db_msg_insert", [
nzbname, "error in crc32 check for file " +
rar0, "warning"
], {})
pwdb.exc("db_nzb_update_verify_status",
[nzbname, -2], {})
pwdb.exc("db_file_update_parstatus",
[f0_origname, -1], {})
child_pipe.send(True)
continue
if pwdb.exc("db_file_getparstatus", [rar0],
{}) == 0 and f0_renamedname != "N/A":
logger.debug(whoami() + "copying " +
f0_renamedname.split("/")[-1] +
" to verified_rar dir")
pwdb.exc("db_msg_insert", [
nzbname,
"copying " + f0_renamedname.split("/")[-1] +
" to verified_rar dir", "info"
], {})
shutil.copy(renamed_dir + f0_renamedname,
verifiedrar_dir)
pwdb.exc("db_file_update_parstatus",
[f0_origname, 1], {})
allrarsverified, rvlist = pwdb.exc("db_only_verified_rars", [nzbname],
{})
if allrarsverified:
break
time.sleep(1)
if TERMINATED:
logger.info(whoami() + "terminated!")
sys.exit()
logger.debug(whoami() + "all rars are checked!")
corruptrars = pwdb.exc("get_all_corrupt_rar_files", [nzbname], {})
if not corruptrars:
logger.debug(whoami() +
"rar files ok, no repair needed, exiting par_verifier")
pwdb.exc("db_nzb_update_verify_status", [nzbname, 2], {})
elif p2list and corruptrars:
pwdb.exc("db_msg_insert", [nzbname, "repairing rar files", "info"], {})
logger.info(whoami() + "par2vol files present, repairing ...")
allok = True
allfound = True
corruptrars_1 = [c1 for c1, _ in corruptrars]
corruptrars_2 = [c2 for _, c2 in corruptrars]
for _, fnshort, fnlong, rarfiles in p2list:
rarf_match = [
rarf for rarf, _ in rarfiles
if rarf in corruptrars_1 or rarf in corruptrars_2
]
if len(rarf_match) == 0:
allfound = False
continue
lrar = str(len(rarfiles))
pwdb.exc("db_msg_insert",
[nzbname, "performing par2verify for " + fnshort, "info"],
{})
ssh = subprocess.Popen(['par2verify', fnlong],
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
sshres = ssh.stdout.readlines()
repair_is_required = False
repair_is_possible = False
for ss in sshres:
ss0 = ss.decode("utf-8")
if "Repair is required" in ss0:
repair_is_required = True
if "Repair is possible" in ss0:
repair_is_possible = True
if not repair_is_required:
pwdb.exc("db_msg_insert", [
nzbname, "par2verify for " + fnshort +
": repair is not required!", "info"
], {})
res0 = 1
elif repair_is_required and not repair_is_possible:
pwdb.exc("db_msg_insert", [
nzbname, "par2verify for " + fnshort +
": repair is required but not possible", "error"
], {})
res0 = -1
elif repair_is_required and repair_is_possible:
pwdb.exc("db_msg_insert", [
nzbname, "par2verify for " + fnshort +
": repair is required and possible, repairing files",
"info"
], {})
logger.info(
whoami() +
"repair is required and possible, performing par2repair")
# repair
ssh = subprocess.Popen(['par2repair', fnlong],
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
sshres = ssh.stdout.readlines()
repair_complete = False
for ss in sshres:
ss0 = ss.decode("utf-8")
if "Repair complete" in ss0:
repair_complete = True
if not repair_complete:
res0 = -1
else:
res0 = 1
# res0 = multipartrar_repair(renamed_dir, fnshort, pwdb, nzbname, logger)
else:
res0 = -1
if res0 != 1:
allok = False
logger.error(whoami() + "repair failed for " + lrar +
"rarfiles in " + fnshort)
pwdb.exc("db_msg_insert", [
nzbname, "rar file repair failed for " + lrar +
" rarfiles in " + fnshort + "!", "error"
], {})
else:
logger.info(whoami() + "repair success for " + lrar +
"rarfiles in " + fnshort)
pwdb.exc("db_msg_insert", [
nzbname, "rar file repair success for " + lrar +
" rarfiles in " + fnshort + "!", "info"
], {})
if not allfound:
allok = False
logger.error(
whoami() +
"cannot attempt one or more par2repairs due to missing par2 file(s)!"
)
pwdb.exc("db_msg_insert", [
nzbname,
"cannot attempt one or more par2repairs due to missing par2 file(s)!",
"error"
], {})
if allok:
logger.info(whoami() + "repair success")
pwdb.exc("db_nzb_update_verify_status", [nzbname, 2], {})
# copy all no yet copied rars to verifiedrar_dir
for c_origname, c_renamedname in corruptrars:
logger.info(whoami() + "copying " + c_renamedname +
" to verifiedrar_dir")
pwdb.exc("db_file_update_parstatus", [c_origname, 1], {})
pwdb.exc("db_file_update_status", [c_origname, 2], {})
shutil.copy(renamed_dir + c_renamedname, verifiedrar_dir)
else:
logger.error(whoami() + "repair failed!")
pwdb.exc("db_nzb_update_verify_status", [nzbname, -1], {})
for _, c_origname in corruptrars:
pwdb.exc("db_file_update_parstatus", [c_origname, -2], {})
else:
pwdb.exc("db_msg_insert", [
"nzbname", "rar file repair failed, no par files available",
"error"
], {})
logger.warning(
whoami() +
"some rars are corrupt but cannot repair (no par2 files)")
pwdb.exc("db_nzb_update_verify_status", [nzbname, -1], {})
logger.info(whoami() + "terminated!")
sys.exit()
def run(self):
self.logger.info(whoami() + self.idn + " thread starting !")
timeout = 2
self.tt_pause_started = None
# connect to push-socket
self.context = zmq.Context()
self.socket = self.context.socket(zmq.PUSH)
self.socket.connect("tcp://127.0.0.1:%d" % self.port)
while self.running:
self.download_done = True
if self.paused:
if not self.tt_pause_started:
self.tt_pause_started = time.time()
elif time.time() - self.tt_pause_started > self.connection_idle_time and self.nntpobj:
if self.servers.close_connection(self.name, self.conn_nr):
self.logger.info(whoami() + self.idn + " connection idle, closed!")
self.nntpobj = None
self.connectionstate = -1
else:
self.logger.info(whoami() + self.idn + " connection non existent, closed!")
self.nntpobj = None
self.connectionstate = -1
time.sleep(0.25)
continue
else:
self.tt_pause_started = None
if not self.running:
break
try:
article = self.articlequeue.pop()
except (queue.Empty, EOFError, IndexError):
time.sleep(0.1)
continue
except Exception as e:
self.logger.warning(whoami() + str(e) + ": problem in clearing article queue")
time.sleep(0.1)
continue
# avoid ctrl-c to interrup downloading itself
self.download_done = False
if not self.nntpobj:
self.retry_connect()
filename, age, filetype, nr_articles, art_nr, art_name, remaining_servers1 = article
if not remaining_servers1:
self.socket.send(pickle.dumps(article + (None,)))
continue
if self.name not in remaining_servers1[0] or not self.nntpobj:
self.articlequeue.append((filename, age, filetype, nr_articles, art_nr, art_name, remaining_servers1))
time.sleep(0.1)
continue
if not self.nntpobj:
self.wait_running(3)
continue
status, bytesdownloaded, info = self.download_article(art_name, age)
# if ctrl-c - exit thread
if status == -3 or not self.running:
break
# if download successfull - put to resultqueue
elif status == 1:
self.last_downloaded_ts = time.time()
timeout = 2
self.bytesdownloaded += bytesdownloaded
self.socket.send(pickle.dumps((filename, age, filetype, nr_articles, art_nr, art_name, self.name, info, True)))
# if 400 error
elif status == -2:
# disconnect
self.logger.warning(whoami() + self.idn + " server connection error, reconnecting ...")
self.connectionstate = -1
try:
name, conn_nr = self.connection
if self.servers.close_connection(name, conn_nr):
self.nntpobj = None
except Exception:
pass
self.nntpobj = None
# take next server
next_servers = self.remove_from_remaining_servers(self.name, remaining_servers1)
next_servers.append([self.name]) # add current server to end of list
self.logger.debug(whoami() + "Requeuing " + art_name + " on server " + self.idn)
# requeue
self.articlequeue.append((filename, age, filetype, nr_articles, art_nr, art_name, next_servers))
self.wait_running(timeout)
timeout *= 2
if timeout > 30:
timeout = 2
continue
# if article could not be found on server / retention not good enough - requeue to other server
elif status in [0, -1]:
timeout = 2
next_servers = self.remove_from_remaining_servers(self.name, remaining_servers1)
if not next_servers:
self.logger.error(whoami() + "Download finally failed on server " + self.idn + ": for article " + art_name + " " + str(next_servers))
self.socket.send(pickle.dumps((filename, age, filetype, nr_articles, art_nr, art_name, [], "failed", True)))
else:
self.logger.debug(whoami() + "Download failed on server " + self.idn + ": for article " + art_name + ", queueing: "
+ str(next_servers))
self.articlequeue.append((filename, age, filetype, nr_articles, art_nr, art_name, next_servers))
self.logger.info(whoami() + self.idn + " exited!")
def partial_unrar(directory, unpack_dir, nzbname, mp_loggerqueue, password,
event_idle, cfg):
setproctitle("gzbx." + os.path.basename(__file__))
logger = mplogging.setup_logger(mp_loggerqueue, __file__)
logger.debug(whoami() + "starting ...")
pwdb = PWDBSender()
event_idle.clear()
cwd0 = os.getcwd()
sh = SigHandler_Unrar(cwd0, logger)
signal.signal(signal.SIGINT, sh.sighandler_unrar)
signal.signal(signal.SIGTERM, sh.sighandler_unrar)
try:
os.chdir(directory)
except FileNotFoundError:
os.mkdir(directory)
logger.info(whoami() + "started partial_unrar")
# get already present rar files
rar_sortedlist0 = None
while not TERMINATED:
rar_sortedlist0 = passworded_rars.get_sorted_rar_list(directory)
# todo: what to do if does not finish here?
if rar_sortedlist0:
break
time.sleep(1)
if TERMINATED:
logger.info(whoami() + "terminated!")
return
rar_sortedlist = []
# das muss besser gehen!!! mit rarlist als parameter von main/postproc
for r1, r2 in rar_sortedlist0:
try:
rar_sortedlist.append((r1, r2.split("/")[-1]))
except Exception as e:
logger.debug(whoami() + whoami() + ": " + str(e))
pwdb.exc("db_nzb_update_unrar_status", [nzbname, 1], {})
nextrarname = rar_sortedlist[0][1]
# first valid rar_sortedlist in place, start unrar!
if password:
cmd = "unrar x -y -o+ -p" + password + " '" + directory + nextrarname + "' '" + unpack_dir + "'"
logger.debug(whoami() + "rar archive is passworded, executing " + cmd)
pwdb.exc("db_msg_insert",
[nzbname, "unraring pw protected rar archive", "info"], {})
status = 1
child = pexpect.spawn(cmd)
status, statmsg, str0 = process_next_unrar_child_pass(
event_idle, child, logger)
if status < 0:
logger.info(whoami() + nextrarname + ": " + statmsg)
pwdb.exc("db_msg_insert",
[nzbname, "unrar " + nextrarname + " failed!", "error"],
{})
else:
pwdb.exc("db_msg_insert",
[nzbname, "checking for double packed rars", "info"], {})
# check if double packed
try:
child.kill(signal.SIGKILL)
except Exception:
pass
is_double_packed, fn = check_double_packed(unpack_dir)
if is_double_packed:
pwdb.exc("db_msg_insert", [
nzbname, "rars are double packed, starting unrar 2nd run",
"warning"
], {})
logger.debug(whoami() + "rars are double packed, executing " +
cmd)
# unrar without pausing!
cmd = "unrar x -y -o+ -p" + password + " '" + fn + "' '" + unpack_dir + "'"
child = pexpect.spawn(cmd)
status, statmsg, str0 = process_next_unrar_child_pass(
event_idle, child, logger)
if status < 0:
logger.info(whoami() + "2nd pass: "******"db_msg_insert",
[nzbname, "unrar 2nd pass failed!", "error"], {})
elif status == 0:
statmsg = "All OK"
status = 0
pwdb.exc("db_msg_insert", [
nzbname, "unrar success 2nd pass for all rar files!",
"info"
], {})
logger.info(whoami() +
"unrar success 2nd pass for all rar files!")
logger.debug(whoami() +
"deleting all rar files in unpack_dir")
delete_all_rar_files(unpack_dir, logger)
elif status == 1:
status = -3
statmsg = "unknown error"
logger.info(whoami() + "2nd pass: "******" / " +
str0)
pwdb.exc("db_msg_insert",
[nzbname, "unrar 2nd pass failed!", "error"], {})
else:
statmsg = "All OK"
status = 0
pwdb.exc("db_msg_insert",
[nzbname, "unrar success for all rar files!", "info"],
{})
logger.info(whoami() + "unrar success for all rar files!")
else:
cmd = "unrar x -y -o+ -vp '" + directory + nextrarname + "' '" + unpack_dir + "'"
logger.debug(whoami() + "rar archive is NOT passworded, executing " +
cmd)
pwdb.exc("db_msg_insert", [nzbname, "unraring rar archive", "info"],
{})
child = pexpect.spawn(cmd)
status = 1 # 1 ... running, 0 ... exited ok, -1 ... rar corrupt, -2 ..missing rar, -3 ... unknown error
while not TERMINATED:
oldnextrarname = nextrarname.split("/")[-1]
#oldfoldersize = folder_size(path=unpack_dir)
#old_t0 = time.time()
status, statmsg, str0 = process_next_unrar_child_pass(
event_idle, child, logger)
#newfoldersize = folder_size(path=unpack_dir)
#delta_size = (newfoldersize - oldfoldersize) / (1024 * 1024)
#delta_t = time.time() - old_t0
#print(delta_size / delta_t, "M decompressed per sec")
if status < 0:
logger.info(whoami() + nextrarname + ": " + statmsg)
pwdb.exc(
"db_msg_insert",
[nzbname, "unrar " + oldnextrarname + " failed!", "error"],
{})
break
logger.info(whoami() + nextrarname + ": unrar success!")
if status == 0:
pwdb.exc("db_msg_insert",
[nzbname, "checking for double packed rars", "info"],
{})
# check if double packed
try:
child.kill(signal.SIGKILL)
except Exception:
pass
is_double_packed, fn = check_double_packed(unpack_dir)
if is_double_packed:
pwdb.exc("db_msg_insert", [
nzbname,
"rars are double packed, starting unrar 2nd run",
"warning"
], {})
cmd = "unrar x -y -o+ '" + fn + "' '" + unpack_dir + "'"
logger.debug(whoami() +
"rars are double packed, executing " + cmd)
# unrar without pausing!
child = pexpect.spawn(cmd)
status, statmsg, str0 = process_next_unrar_child_pass(
event_idle, child, logger)
if status < 0:
logger.info(whoami() + "2nd pass: "******"db_msg_insert",
[nzbname, "unrar 2nd pass failed!", "error"],
{})
break
if status == 0:
statmsg = "All OK"
status = 0
pwdb.exc("db_msg_insert", [
nzbname,
"unrar success 2nd pass for all rar files!", "info"
], {})
logger.info(
whoami() +
"unrar success 2nd pass for all rar files!")
logger.debug(whoami() +
"deleting all rar files in unpack_dir")
delete_all_rar_files(unpack_dir, logger)
break
if status == 1:
status = -3
statmsg = "unknown error"
logger.info(whoami() + "2nd pass: "******" / " +
str0)
pwdb.exc("db_msg_insert",
[nzbname, "unrar 2nd pass failed!", "error"],
{})
break
else:
statmsg = "All OK"
status = 0
pwdb.exc(
"db_msg_insert",
[nzbname, "unrar success for all rar files!", "info"],
{})
logger.info(whoami() + "unrar success for all rar files!")
break
try:
gg = re.search(r"Insert disk with ", str0, flags=re.IGNORECASE)
gend = gg.span()[1]
nextrarname = str0[gend:-19]
except Exception as e:
logger.warning(whoami() + str(e) + ", unknown error")
statmsg = "unknown error in re evalution"
status = -4
pwdb.exc(
"db_msg_insert",
[nzbname, "unrar " + oldnextrarname + " failed!", "error"],
{})
break
pwdb.exc(
"db_msg_insert",
[nzbname, "unrar " + oldnextrarname + " success!", "info"], {})
logger.debug(whoami() + "Waiting for next rar: " + nextrarname)
# first, if .r00
try:
nextrar_number = int(
re.search(r"\d{0,9}$", nextrarname).group())
nextrar_wo_number = nextrarname.rstrip("0123456789")
except Exception:
nextrar_wo_number = nextrarname
nextrar_number = -1
# if part01.rar
if nextrar_number == -1:
try:
nextrar_number = int(
nextrarname.split(".part")[-1].split(".rar")[0])
nextrar_wo_number = nextrarname.rstrip(".rar").rstrip(
"0123456789").rstrip("part").rstrip(".")
except Exception:
nextrar_wo_number = nextrarname
nextrar_number = -1
gotnextrar = False
nextrar_short = nextrarname.split("/")[-1]
# todo: hier deadlock/unendliches Warten im Postprocess vermeiden, wenn rar nicht auftaucht!
event_idle.set()
while not gotnextrar and not TERMINATED:
time.sleep(1)
for f0 in glob.glob(directory + "*") + glob.glob(directory +
".*"):
if nextrarname == f0:
gotnextrar = True
# now check if we waited too long for next rar0 - but how?
if not gotnextrar and nextrar_number != -1:
for f0 in glob.glob(directory +
"*") + glob.glob(directory + ".*"):
f0_number = -1
f0_wo_number = f0
try:
f0_number = int(re.search(r"\d{0,9}$", f0).group())
f0_wo_number = f0.rstrip("0123456789")
except Exception:
try:
f0_number = int(
f0.split(".part")[-1].split(".rar")[0])
f0_wo_number = f0.rstrip(".rar").rstrip(
"0123456789").rstrip("part").rstrip(".")
except Exception:
continue
if f0_number == -1:
continue
if f0_wo_number == nextrar_wo_number and f0_number > nextrar_number:
pwdb.exc("db_msg_insert", [
nzbname,
"unrar waiting for next rar, but next rar " +
nextrar_short +
" seems to be skipped, you may want to interrupt ...",
"warning"
], {})
break
event_idle.clear()
if TERMINATED:
break
time.sleep(1) # achtung hack!
child.sendline("C")
try:
child.kill(signal.SIGKILL)
except Exception:
pass
if TERMINATED:
logger.info(whoami() + "exited!")
else:
logger.info(whoami() + str(status) + " " + statmsg)
try:
os.chdir(cwd0)
if status == 0:
pwdb.exc("db_nzb_update_unrar_status", [nzbname, 2], {})
elif status == -5:
pwdb.exc("db_nzb_update_unrar_status", [nzbname, -2], {})
else:
pwdb.exc("db_nzb_update_unrar_status", [nzbname, -1], {})
except Exception as e:
logger.warning(whoami() + str(e))
event_idle.clear()
logger.info(whoami() + "exited!")
sys.exit()
def run(self):
self.socket.setsockopt(zmq.LINGER, 0)
socketurl = "tcp://" + self.host + ":" + self.port
self.socket.connect(socketurl)
dl_running = True
while not self.event_stopped.wait(self.delay):
# some button pressed, of which main.py should be informed?
GLib.idle_add(self.gui.update_logs_and_lists)
try:
queue_elem = self.guiqueue.get_nowait()
self.guiqueue.task_done()
except (queue.Empty, EOFError, ValueError):
queue_elem = None
except Exception as e:
self.logger.error(whoami() + str(e))
queue_elem = None
if queue_elem:
elem_type, elem_val = queue_elem
if elem_type == "order_changed":
msg0 = "SET_NZB_ORDER"
msg0_val = [nzb[0] for nzb in self.appdata.nzbs]
elif elem_type == "interrupted":
msg0 = "SET_NZB_INTERRUPT"
msg0_val = [nzb[0] for nzb in self.appdata.nzbs]
elif elem_type == "closeall":
msg0 = "SET_CLOSEALL"
msg0_val = elem_val # non-empty if apply / restart!
self.appdata.closeall = True
elif elem_type == "nzb_added":
msg0 = "NZB_ADDED"
msg0_val, add_button = elem_val
elif elem_type == "deleted_from_history":
msg0 = "DELETED_FROM_HISTORY"
msg0_val = elem_val
elif elem_type == "reprocess_from_start":
msg0 = "REPROCESS_FROM_START"
msg0_val = elem_val
elif elem_type == "reprocess_from_last":
msg0 = "REPROCESS_FROM_LAST"
msg0_val = elem_val
elif elem_type == "dl_running":
msg0_val = None
dl_running_new = elem_val
if dl_running != dl_running_new:
dl_running = dl_running_new
if dl_running:
msg0 = "SET_RESUME"
else:
msg0 = "SET_PAUSE"
else:
msg0 = None
else:
msg0 = None
if msg0:
try:
self.socket.send_pyobj((msg0, msg0_val))
datatype, datarec = self.socket.recv_pyobj()
except Exception as e:
self.logger.error(whoami() + str(e))
if elem_type == "nzb_added":
add_button.set_sensitive(True)
elif elem_type == "closeall":
with self.lock:
self.appdata.closeall = False
self.logger.debug(
whoami() +
"received main closeall confirm, shutting down guipoller"
)
elif elem_type in ["order_changed", "interrupted"]:
GLib.idle_add(self.toggle_buttons)
self.logger.debug(whoami() +
"order changed/interrupted ok!")
elif elem_type in [
"deleted_from_history", "reprocess_from_start",
"reprocess_from_last"
]:
GLib.idle_add(self.toggle_buttons_history)
self.logger.debug(whoami() +
"deleted_from_history/reprocess ok!")
else:
self.logger.error(whoami() +
"cannot interpret element in guiqueue")
else:
try:
self.socket.send_pyobj(("REQ", None))
datatype, datarec = self.socket.recv_pyobj()
if datatype == "NOOK":
continue
elif datatype == "DL_DATA":
data, server_config, dl_running, nzb_status_string, \
article_health, dlconfig, gb_downloaded, server_ts = datarec
try:
GLib.idle_add(self.update_mainwindow, data,
server_config, dl_running,
nzb_status_string, article_health,
dlconfig, gb_downloaded, server_ts)
continue
except Exception as e:
self.logger.debug(whoami() + str(e))
except Exception as e:
self.logger.error(whoami() + str(e))
# close socket, join queue & exit guipoller
self.logger.debug(whoami() + "closing socket")
try:
self.socket.close()
self.context.term()
except Exception:
self.logger.warning(whoami())
self.logger.debug(whoami() + "joining gui_queue")
while True:
try:
queue_elem = self.guiqueue.get_nowait()
self.guiqueue.task_done()
except (queue.Empty, EOFError, ValueError):
break
except Exception as e:
self.logger.error(whoami() + str(e))
break
self.guiqueue.join()
self.logger.info(whoami() + "exiting")
sys.exit()
def mpconnector(child_pipe, cfg, server_ts, mp_loggerqueue):
setproctitle("gzbx." + os.path.basename(__file__))
logger = mplogging.setup_logger(mp_loggerqueue, __file__)
logger.debug(whoami() + "starting mpconnector process")
sh = SigHandler_MPconnector(logger)
signal.signal(signal.SIGINT, sh.sighandler_mpconnector)
signal.signal(signal.SIGTERM, sh.sighandler_mpconnector)
# setup ports + zmq streamer device
worker_port = 37100
connections_port = worker_port + 1
while is_port_in_use(worker_port) or is_port_in_use(connections_port):
worker_port += 1
connections_port = worker_port + 1
# streamerdevice gathers results downloaded from usenet servers and
# keeps them ready to be delivered to mpconnector (which is the only worker).
# this is not working with ProcessDevice -> stuck on sys.exit!??
streamerdevice = ThreadDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH)
streamerdevice.bind_in("tcp://127.0.0.1:%d" % connections_port)
streamerdevice.bind_out("tcp://127.0.0.1:%d" % worker_port)
streamerdevice.setsockopt_in(zmq.IDENTITY, b"PULL")
streamerdevice.setsockopt_out(zmq.IDENTITY, b"PUSH")
streamerdevice.start()
# setup socket for worker
context = zmq.Context()
socket = context.socket(zmq.PULL)
socket.connect("tcp://127.0.0.1:%d" % worker_port)
# poller to check if streamerdevice is empty
poller = zmq.Poller()
poller.register(socket, zmq.POLLIN)
thr_articlequeue = deque()
ct = ConnectionThreads(cfg, thr_articlequeue, connections_port, server_ts, logger)
cmdlist = ("start", "stop", "pause", "resume", "reset_timestamps", "reset_timestamps_bdl",
"get_downloaded_per_server", "exit", "clearqueues", "connection_thread_health", "get_server_config",
"set_tmode_sanitycheck", "set_tmode_download", "get_level_servers", "clear_articlequeue",
"queues_empty", "clear_resultqueue", "len_articlequeue", "push_articlequeue", "pull_resultqueue",
"push_entire_articlequeue", "pull_entire_resultqueue", "get_bytesdownloaded")
quit_via_cmdexit = False
while not TERMINATED:
try:
cmd, param = child_pipe.recv()
except Exception as e:
logger.warning(whoami() + str(e))
continue
if cmd in cmdlist:
result = True
if cmd == "push_articlequeue":
try:
thr_articlequeue.append(param)
except Exception:
result = None
elif cmd == "push_entire_articlequeue":
try:
for article0 in param:
thr_articlequeue.append(article0)
except Exception:
result = None
elif cmd == "pull_entire_resultqueue":
result = get_from_streamingdevice(socket, onlyfirst=False)
logger.debug(whoami() + "len_articlequeue: " + str(len(ct.articlequeue)))
elif cmd == "pull_resultqueue":
result = get_from_streamingdevice(socket, onlyfirst=True)
elif cmd == "start":
ct.start_threads()
elif cmd == "queues_empty":
socks = dict(poller.poll(timeout=10))
streamerdevice_empty = True
if socket in socks and socks[socket] == zmq.POLLIN:
streamerdevice_empty = False
result = (len(ct.articlequeue) == 0) and streamerdevice_empty
elif cmd == "get_bytesdownloaded":
result = ct.get_bytesdownloaded()
elif cmd == "len_articlequeue":
try:
result = int(len(ct.articlequeue))
except Exception:
result = 0
elif cmd == "clear_articlequeue":
ct.articlequeue.clear()
elif cmd == "clear_resultqueue":
get_from_streamingdevice(socket, onlyfirst=False)
elif cmd == "stop":
ct.stop_threads()
elif cmd == "resume":
ct.resume_threads()
elif cmd == "pause":
ct.pause_threads()
elif cmd == "reset_timestamps":
ct.reset_timestamps()
elif cmd == "reset_timestamps_bdl":
ct.reset_timestamps_bdl()
elif cmd == "get_downloaded_per_server":
result = ct.get_downloaded_per_server()
elif cmd == "connection_thread_health":
result = ct.connection_thread_health()
elif cmd == "get_server_config":
result = ct.get_server_config()
elif cmd == "set_tmode_sanitycheck":
for t, _ in ct.threads:
t.mode = "sanitycheck"
elif cmd == "set_tmode_download":
for t, _ in ct.threads:
t.mode = "download"
elif cmd == "get_level_servers":
le_serv0 = []
try:
retention = param
for level, serverlist in ct.level_servers.items():
level_servers = serverlist
le_dic = {}
for le in level_servers:
_, _, _, _, _, _, _, _, age, _ = ct.servers.get_single_server_config(le)
le_dic[le] = age
les = [le for le in level_servers if le_dic[le] > retention * 0.9]
le_serv0.append(les)
except Exception:
pass
result = le_serv0
elif cmd == "exit":
quit_via_cmdexit = True
break
elif cmd == "clearqueues":
ct.articlequeue.clear()
get_from_streamingdevice(socket, onlyfirst=False)
child_pipe.send(result)
else:
child_pipe.send(None)
logger.debug(whoami() + "shutting down ...")
ct.stop_threads()
get_from_streamingdevice(socket, onlyfirst=False)
if quit_via_cmdexit:
child_pipe.send(result)
def sighandler_mpconnector(self, a, b):
self.logger.info(whoami() + "terminating ...")
global TERMINATED
TERMINATED = True
def open_connection(self, server_name0, conn_nr):
result = None
for idx, (sn, cn, rt, nobj) in enumerate(self.all_connections):
if sn == server_name0 and cn == conn_nr:
if nobj:
return nobj
else:
context = ssl.SSLContext(ssl.PROTOCOL_TLS)
sc = self.get_single_server_config(server_name0)
if sc:
server_name, server_url, user, password, port, usessl, level, connections, retention, useserver\
= sc
if useserver:
try:
self.logger.debug(whoami() +
"Opening connection # " +
str(conn_nr) + "to server " +
server_name)
if usessl:
nntpobj = nntplib.NNTP_SSL(
server_url,
user=user,
password=password,
ssl_context=context,
port=port,
readermode=True)
else:
# this is just for ginzicut preloading
if user.lower(
) == "ginzicut" or password.lower(
) == "ginzicut":
nntpobj = nntplib.NNTP(server_url,
port=port)
else:
nntpobj = nntplib.NNTP(
server_url,
user=user,
password=password,
readermode=True,
port=port)
self.logger.debug(whoami() +
"Opened Connection #" +
str(conn_nr) +
" on server " + server_name0)
result = nntpobj
self.all_connections[idx] = (sn, cn, rt,
nntpobj)
break
except Exception as e:
self.logger.error(whoami() + "Server " +
server_name0 +
" connect error: " + str(e))
self.all_connections[idx] = (sn, cn, rt, None)
break
else:
self.logger.error(
whoami() +
"Cannot get server config for server: " +
server_name0)
self.all_connections[idx] = (sn, cn, rt, None)
break
return result
def download_article(self, article_name, article_age):
bytesdownloaded = 0
info0 = None
if self.mode == "sanitycheck":
try:
resp, number, message_id = self.nntpobj.stat(article_name)
if article_name != message_id:
status = -1
else:
status = 1
except Exception as e:
self.logger.error(whoami() + str(e) + self.idn + " for article " + article_name)
status = -1
return status, 0, 0
if self.server_retention < article_age * 0.95:
self.logger.warning(whoami() + "Retention on " + self.server_name + " not sufficient for article " + article_name)
return -1, 0, None
try:
resp, info = self.nntpobj.body(article_name)
if resp.startswith("222"):
status = 1
info0 = [inf + b"\r\n" if not inf.endswith(b"\r\n") else inf for inf in info.lines]
bytesdownloaded = len(b''.join(info.lines))
else:
self.logger.warning(whoami() + resp + ": could not find " + article_name + " on " + self.idn)
status = 0
# nntpError 4xx - Command was syntactically correct but failed for some reason
except nntplib.NNTPTemporaryError as e:
errcode = e.response.strip()[:3]
if errcode == "400":
# server quits, new connection has to be established
status = -2
else:
status = 0
self.logger.warning(whoami() + e.response + ": could not find " + article_name + " on " + self.idn)
# nntpError 5xx - Command unknown error
except nntplib.NNTPPermanentError as e:
errcode = e.response.strip()[:3]
if errcode in ["503", "502"]:
# timeout, closing connection
status = -2
else:
status = 0
self.logger.warning(whoami() + e.response + ": could not find " + article_name + " on " + self.idn)
status = 0
except nntplib.NNTPError as e:
status = 0
self.logger.warning(whoami() + e.response + ": could not find " + article_name + " on " + self.idn)
except KeyboardInterrupt:
status = -3
except socket.timeout:
status = -2
self.logger.warning(whoami() + "socket.timeout on " + self.idn)
except AttributeError as e:
status = -2
self.logger.warning(whoami() + str(e) + ": " + article_name + " on " + self.idn)
except BrokenPipeError as e:
status = -2
self.logger.warning(whoami() + str(e) + ": " + article_name + " on " + self.idn)
except Exception as e:
if "write to closed file" in str(e):
status = -2
else:
status = 0
self.logger.warning(whoami() + str(e) + ": " + article_name + " on " + self.idn)
# self.bandwidth_bytes += bytesdownloaded
return status, bytesdownloaded, info0
