def signal_handler(signal, frame):
sys.stdout.write('\n')
sys.stdout.write("Stopping threads...")
sys.stdout.flush()
thread.stop_threads(threads)
sys.stdout.write("done\n")
sys.exit(0)
def signal_handler(signal, frame):
sys.stdout.write('\n')
sys.stdout.write("Stopping threads...")
sys.stdout.flush()
thread.stop_threads(threads)
sys.stdout.write("done\n")
sys.exit(0)
def main(nzb_fn, config):
threads = []
files = []
seg_count = 0
bytes_total = 0
segments = Queue.Queue()
missing = Queue.Queue()
if nzb_fn.endswith('.gz'):
with open(nzb_fn) as f:
nzb = BytesIO(GzipFile('', 'r', 0, f).read())
else:
with open(nzb_fn) as f:
nzb = BytesIO(f.read())
# Listen for exit
def signal_handler(signal, frame):
sys.stdout.write('\n')
sys.stdout.write("Stopping threads...")
sys.stdout.flush()
thread.stop_threads(threads)
sys.stdout.write("done\n")
sys.exit(0)
# TODO: Listen to other signals
signal.signal(signal.SIGINT, signal_handler)
# Determine server priority. We do this by considering two types of
# servers: primary and backup. Primary servers will be used to check for
# all segments. If a segment is missing from all available primary servers
# then backup servers will be used to check for the missing segments. Only
# after a segment is verified as missing from all primary and backup servers
# do we consider is a missing segment.
primary_servers = []
backup_servers = []
num_connections = 0
for host, settings in config.items():
if settings.get("backup", False):
backup_servers.append(host)
else:
primary_servers.append(host)
num_connections += settings.get("connections", 0)
print("Found %d primary host%s" %
(len(primary_servers), len(primary_servers) != 1 and "s" or ""))
for host in primary_servers:
print(" %s" % host)
print("Found %d backup host%s" %
(len(backup_servers), len(backup_servers) != 1 and "s" or ""))
for host in backup_servers:
print(" %s" % host)
#print("Creating %d threads" % num_connections)
priority = 0
for host in primary_servers:
config[host]["priority"] = priority
priority += 1
for host in backup_servers:
config[host]["priority"] = priority
priority += 1
# Spawn some threads
tid = 0
for host, settings in config.items():
server = Server(host, settings)
for i in range(settings.get("connections", 0)):
try:
t = thread.SegmentCheckerThread(tid, segments, missing, server)
t.setDaemon(True)
t.start()
threads.append(t)
tid += 1
except:
break
print("Created %d/%d threads" % (tid, num_connections))
# Parse NZB and populate the Queue
print("Parsing NZB: %s" % nzb_fn)
for event, elem in iterparse(nzb, events=("start", "end")):
if event == "start" and elem.tag.endswith('file'):
files.append(elem.get('subject'))
if event == "end" and elem.tag.endswith('segment'):
bytes = int(elem.get('bytes', 0))
bytes_total += bytes
segments.put((files[-1], '<%s>' % elem.text, bytes))
seg_count += 1
size, unit = get_size(bytes_total)
print("Found %d files and %d segments totaling %s %s" %
(len(files), seg_count, size, unit))
pbar = ProgressBar(segments, missing)
while not segments.empty():
pbar.update()
time.sleep(0.1)
pbar.finish()
missing.join()
num_missing = missing.qsize()
if num_missing > 0:
missing_bytes = 0
print("Result: missing %d/%d segments; %0.2f%% complete" %
(num_missing, seg_count,
((seg_count - num_missing) / seg_count * 100.00)))
while not missing.empty():
f, seg, bytes = missing.get()
missing_bytes += bytes
print('\tfile="%s", segment="%s"' % (f, seg))
size, unit = get_size(missing_bytes)
print("Missing %s %s" % (size, unit))
else:
print("Result: all %d segments available" % seg_count)
thread.stop_threads(threads)
def main(nzb, num_connections, nntp_kwargs):
threads = []
files = []
seg_count = 0
bytes_total = 0
segments = Queue.Queue()
missing = Queue.Queue()
# Listen for exit
def signal_handler(signal, frame):
sys.stdout.write('\n')
sys.stdout.write("Stopping threads...")
sys.stdout.flush()
thread.stop_threads(threads)
sys.stdout.write("done\n")
sys.exit(0)
# TODO: Listen to other signals
signal.signal(signal.SIGINT, signal_handler)
# Spawn some threads
for i in range(num_connections):
try:
t = thread.SegmentCheckerThread(i, segments, missing, nntp_kwargs)
t.setDaemon(True)
t.start()
threads.append(t)
except:
break
print "Created %d threads" % (i+1)
# Parse NZB and populate the Queue
print "Parsing NZB: %s" % nzb
for event, elem in iterparse(nzb, events=("start", "end")):
if event == "start" and elem.tag.endswith('file'):
files.append(elem.get('subject'))
if event == "end" and elem.tag.endswith('segment'):
bytes = int(elem.get('bytes',0))
bytes_total += bytes
segments.put((files[-1], '<%s>' % elem.text, bytes))
seg_count += 1
size, unit = get_size(bytes_total)
print "Found %d files and %d segments totalling %s %s" % (len(files), seg_count, size, unit)
pbar = ProgressBar(segments, missing)
while not segments.empty():
pbar.update()
time.sleep(0.1)
pbar.finish()
missing.join()
num_missing = missing.qsize()
if num_missing > 0:
missing_bytes = 0
print "Result: missing %d/%d segments; %0.2f%% complete" % (num_missing, seg_count, ((seg_count-num_missing)/seg_count * 100.00))
while not missing.empty():
f, seg, bytes = missing.get()
missing_bytes += bytes
print '\tfile="%s", segment="%s"' % (f, seg)
size, unit = get_size(missing_bytes)
print "Missing %s %s" % (size, unit)
else:
print "Result: all %d segments available" % seg_count
thread.stop_threads(threads)
def main(nzb, config):
threads = []
files = []
seg_count = 0
bytes_total = 0
segments = Queue.Queue()
missing = Queue.Queue()
# Listen for exit
def signal_handler(signal, frame):
sys.stdout.write('\n')
sys.stdout.write("Stopping threads...")
sys.stdout.flush()
thread.stop_threads(threads)
sys.stdout.write("done\n")
sys.exit(0)
# TODO: Listen to other signals
signal.signal(signal.SIGINT, signal_handler)
# Determine server priority. We do this by considering two types of
# servers: primary and backup. Primary servers will be used to check for
# all segments. If a segment is missing from all available primary servers
# then backup servers will be used to check for the missing segments. Only
# after a segment is verified as missing from all primary and backup servers
# do we consider is a missing segment.
primary_servers = []
backup_servers = []
num_connections = 0
for host, settings in config.items():
if settings.get("backup", False):
backup_servers.append(host)
else:
primary_servers.append(host)
num_connections += settings.get("connections", 0)
print "Found %d primary host%s" % (len(primary_servers), len(primary_servers) != 1 and "s" or "")
for host in primary_servers:
print " ", host
print "Found %d backup host%s" % (len(backup_servers), len(backup_servers) != 1 and "s" or "")
for host in backup_servers:
print " ", host
#print "Creating %d threads" % num_connections
priority = 0
for host in primary_servers:
config[host]["priority"] = priority
priority += 1
for host in backup_servers:
config[host]["priority"] = priority
priority += 1
# Spawn some threads
tid = 0
for host, settings in config.items():
server = Server(host, settings)
for i in range(settings.get("connections", 0)):
try:
t = thread.SegmentCheckerThread(tid, segments, missing, server)
t.setDaemon(True)
t.start()
threads.append(t)
tid += 1
except:
break
print "Created %d/%d threads" % (tid, num_connections)
# Parse NZB and populate the Queue
print "Parsing NZB: %s" % nzb
for event, elem in iterparse(nzb, events=("start", "end")):
if event == "start" and elem.tag.endswith('file'):
files.append(elem.get('subject'))
if event == "end" and elem.tag.endswith('segment'):
bytes = int(elem.get('bytes',0))
bytes_total += bytes
segments.put((files[-1], '<%s>' % elem.text, bytes))
seg_count += 1
size, unit = get_size(bytes_total)
print "Found %d files and %d segments totaling %s %s" % (len(files), seg_count, size, unit)
pbar = ProgressBar(segments, missing)
while not segments.empty():
pbar.update()
time.sleep(0.1)
pbar.finish()
missing.join()
num_missing = missing.qsize()
if num_missing > 0:
missing_bytes = 0
print "Result: missing %d/%d segments; %0.2f%% complete" % (num_missing, seg_count, ((seg_count-num_missing)/seg_count * 100.00))
while not missing.empty():
f, seg, bytes = missing.get()
missing_bytes += bytes
print '\tfile="%s", segment="%s"' % (f, seg)
size, unit = get_size(missing_bytes)
print "Missing %s %s" % (size, unit)
else:
print "Result: all %d segments available" % seg_count
thread.stop_threads(threads)
def main(nzb, num_connections, nntp_kwargs):
threads = []
files = []
seg_count = 0
bytes_total = 0
segments = Queue.Queue()
missing = Queue.Queue()
# Listen for exit
def signal_handler(signal, frame):
sys.stdout.write('\n')
sys.stdout.write("Stopping threads...")
sys.stdout.flush()
thread.stop_threads(threads)
sys.stdout.write("done\n")
sys.exit(0)
# TODO: Listen to other signals
signal.signal(signal.SIGINT, signal_handler)
# Spawn some threads
for i in range(num_connections):
try:
t = thread.SegmentCheckerThread(i, segments, missing, nntp_kwargs)
t.setDaemon(True)
t.start()
threads.append(t)
except:
break
print "Created %d threads" % (i + 1)
# Parse NZB and populate the Queue
print "Parsing NZB: %s" % nzb
for event, elem in iterparse(nzb, events=("start", "end")):
if event == "start" and elem.tag.endswith('file'):
files.append(elem.get('subject'))
if event == "end" and elem.tag.endswith('segment'):
bytes = int(elem.get('bytes', 0))
bytes_total += bytes
segments.put((files[-1], '<%s>' % elem.text, bytes))
seg_count += 1
size, unit = get_size(bytes_total)
print "Found %d files and %d segments totalling %s %s" % (
len(files), seg_count, size, unit)
pbar = ProgressBar(segments, missing)
while not segments.empty():
pbar.update()
time.sleep(0.1)
pbar.finish()
missing.join()
num_missing = missing.qsize()
if num_missing > 0:
missing_bytes = 0
print "Result: missing %d/%d segments; %0.2f%% complete" % (
num_missing, seg_count,
((seg_count - num_missing) / seg_count * 100.00))
while not missing.empty():
f, seg, bytes = missing.get()
missing_bytes += bytes
print '\tfile="%s", segment="%s"' % (f, seg)
size, unit = get_size(missing_bytes)
print "Missing %s %s" % (size, unit)
else:
print "Result: all %d segments available" % seg_count
thread.stop_threads(threads)
