def signal_handler(signal, frame): sys.stdout.write('\n') sys.stdout.write("Stopping threads...") sys.stdout.flush() thread.stop_threads(threads) sys.stdout.write("done\n") sys.exit(0)
def main(nzb_fn, config): threads = [] files = [] seg_count = 0 bytes_total = 0 segments = Queue.Queue() missing = Queue.Queue() if nzb_fn.endswith('.gz'): with open(nzb_fn) as f: nzb = BytesIO(GzipFile('', 'r', 0, f).read()) else: with open(nzb_fn) as f: nzb = BytesIO(f.read()) # Listen for exit def signal_handler(signal, frame): sys.stdout.write('\n') sys.stdout.write("Stopping threads...") sys.stdout.flush() thread.stop_threads(threads) sys.stdout.write("done\n") sys.exit(0) # TODO: Listen to other signals signal.signal(signal.SIGINT, signal_handler) # Determine server priority. We do this by considering two types of # servers: primary and backup. Primary servers will be used to check for # all segments. If a segment is missing from all available primary servers # then backup servers will be used to check for the missing segments. Only # after a segment is verified as missing from all primary and backup servers # do we consider is a missing segment. primary_servers = [] backup_servers = [] num_connections = 0 for host, settings in config.items(): if settings.get("backup", False): backup_servers.append(host) else: primary_servers.append(host) num_connections += settings.get("connections", 0) print("Found %d primary host%s" % (len(primary_servers), len(primary_servers) != 1 and "s" or "")) for host in primary_servers: print(" %s" % host) print("Found %d backup host%s" % (len(backup_servers), len(backup_servers) != 1 and "s" or "")) for host in backup_servers: print(" %s" % host) #print("Creating %d threads" % num_connections) priority = 0 for host in primary_servers: config[host]["priority"] = priority priority += 1 for host in backup_servers: config[host]["priority"] = priority priority += 1 # Spawn some threads tid = 0 for host, settings in config.items(): server = Server(host, settings) for i in range(settings.get("connections", 0)): try: t = thread.SegmentCheckerThread(tid, segments, missing, server) t.setDaemon(True) t.start() threads.append(t) tid += 1 except: break print("Created %d/%d threads" % (tid, num_connections)) # Parse NZB and populate the Queue print("Parsing NZB: %s" % nzb_fn) for event, elem in iterparse(nzb, events=("start", "end")): if event == "start" and elem.tag.endswith('file'): files.append(elem.get('subject')) if event == "end" and elem.tag.endswith('segment'): bytes = int(elem.get('bytes', 0)) bytes_total += bytes segments.put((files[-1], '<%s>' % elem.text, bytes)) seg_count += 1 size, unit = get_size(bytes_total) print("Found %d files and %d segments totaling %s %s" % (len(files), seg_count, size, unit)) pbar = ProgressBar(segments, missing) while not segments.empty(): pbar.update() time.sleep(0.1) pbar.finish() missing.join() num_missing = missing.qsize() if num_missing > 0: missing_bytes = 0 print("Result: missing %d/%d segments; %0.2f%% complete" % (num_missing, seg_count, ((seg_count - num_missing) / seg_count * 100.00))) while not missing.empty(): f, seg, bytes = missing.get() missing_bytes += bytes print('\tfile="%s", segment="%s"' % (f, seg)) size, unit = get_size(missing_bytes) print("Missing %s %s" % (size, unit)) else: print("Result: all %d segments available" % seg_count) thread.stop_threads(threads)
def main(nzb, num_connections, nntp_kwargs): threads = [] files = [] seg_count = 0 bytes_total = 0 segments = Queue.Queue() missing = Queue.Queue() # Listen for exit def signal_handler(signal, frame): sys.stdout.write('\n') sys.stdout.write("Stopping threads...") sys.stdout.flush() thread.stop_threads(threads) sys.stdout.write("done\n") sys.exit(0) # TODO: Listen to other signals signal.signal(signal.SIGINT, signal_handler) # Spawn some threads for i in range(num_connections): try: t = thread.SegmentCheckerThread(i, segments, missing, nntp_kwargs) t.setDaemon(True) t.start() threads.append(t) except: break print "Created %d threads" % (i+1) # Parse NZB and populate the Queue print "Parsing NZB: %s" % nzb for event, elem in iterparse(nzb, events=("start", "end")): if event == "start" and elem.tag.endswith('file'): files.append(elem.get('subject')) if event == "end" and elem.tag.endswith('segment'): bytes = int(elem.get('bytes',0)) bytes_total += bytes segments.put((files[-1], '<%s>' % elem.text, bytes)) seg_count += 1 size, unit = get_size(bytes_total) print "Found %d files and %d segments totalling %s %s" % (len(files), seg_count, size, unit) pbar = ProgressBar(segments, missing) while not segments.empty(): pbar.update() time.sleep(0.1) pbar.finish() missing.join() num_missing = missing.qsize() if num_missing > 0: missing_bytes = 0 print "Result: missing %d/%d segments; %0.2f%% complete" % (num_missing, seg_count, ((seg_count-num_missing)/seg_count * 100.00)) while not missing.empty(): f, seg, bytes = missing.get() missing_bytes += bytes print '\tfile="%s", segment="%s"' % (f, seg) size, unit = get_size(missing_bytes) print "Missing %s %s" % (size, unit) else: print "Result: all %d segments available" % seg_count thread.stop_threads(threads)
def main(nzb, config): threads = [] files = [] seg_count = 0 bytes_total = 0 segments = Queue.Queue() missing = Queue.Queue() # Listen for exit def signal_handler(signal, frame): sys.stdout.write('\n') sys.stdout.write("Stopping threads...") sys.stdout.flush() thread.stop_threads(threads) sys.stdout.write("done\n") sys.exit(0) # TODO: Listen to other signals signal.signal(signal.SIGINT, signal_handler) # Determine server priority. We do this by considering two types of # servers: primary and backup. Primary servers will be used to check for # all segments. If a segment is missing from all available primary servers # then backup servers will be used to check for the missing segments. Only # after a segment is verified as missing from all primary and backup servers # do we consider is a missing segment. primary_servers = [] backup_servers = [] num_connections = 0 for host, settings in config.items(): if settings.get("backup", False): backup_servers.append(host) else: primary_servers.append(host) num_connections += settings.get("connections", 0) print "Found %d primary host%s" % (len(primary_servers), len(primary_servers) != 1 and "s" or "") for host in primary_servers: print " ", host print "Found %d backup host%s" % (len(backup_servers), len(backup_servers) != 1 and "s" or "") for host in backup_servers: print " ", host #print "Creating %d threads" % num_connections priority = 0 for host in primary_servers: config[host]["priority"] = priority priority += 1 for host in backup_servers: config[host]["priority"] = priority priority += 1 # Spawn some threads tid = 0 for host, settings in config.items(): server = Server(host, settings) for i in range(settings.get("connections", 0)): try: t = thread.SegmentCheckerThread(tid, segments, missing, server) t.setDaemon(True) t.start() threads.append(t) tid += 1 except: break print "Created %d/%d threads" % (tid, num_connections) # Parse NZB and populate the Queue print "Parsing NZB: %s" % nzb for event, elem in iterparse(nzb, events=("start", "end")): if event == "start" and elem.tag.endswith('file'): files.append(elem.get('subject')) if event == "end" and elem.tag.endswith('segment'): bytes = int(elem.get('bytes',0)) bytes_total += bytes segments.put((files[-1], '<%s>' % elem.text, bytes)) seg_count += 1 size, unit = get_size(bytes_total) print "Found %d files and %d segments totaling %s %s" % (len(files), seg_count, size, unit) pbar = ProgressBar(segments, missing) while not segments.empty(): pbar.update() time.sleep(0.1) pbar.finish() missing.join() num_missing = missing.qsize() if num_missing > 0: missing_bytes = 0 print "Result: missing %d/%d segments; %0.2f%% complete" % (num_missing, seg_count, ((seg_count-num_missing)/seg_count * 100.00)) while not missing.empty(): f, seg, bytes = missing.get() missing_bytes += bytes print '\tfile="%s", segment="%s"' % (f, seg) size, unit = get_size(missing_bytes) print "Missing %s %s" % (size, unit) else: print "Result: all %d segments available" % seg_count thread.stop_threads(threads)
def main(nzb, num_connections, nntp_kwargs): threads = [] files = [] seg_count = 0 bytes_total = 0 segments = Queue.Queue() missing = Queue.Queue() # Listen for exit def signal_handler(signal, frame): sys.stdout.write('\n') sys.stdout.write("Stopping threads...") sys.stdout.flush() thread.stop_threads(threads) sys.stdout.write("done\n") sys.exit(0) # TODO: Listen to other signals signal.signal(signal.SIGINT, signal_handler) # Spawn some threads for i in range(num_connections): try: t = thread.SegmentCheckerThread(i, segments, missing, nntp_kwargs) t.setDaemon(True) t.start() threads.append(t) except: break print "Created %d threads" % (i + 1) # Parse NZB and populate the Queue print "Parsing NZB: %s" % nzb for event, elem in iterparse(nzb, events=("start", "end")): if event == "start" and elem.tag.endswith('file'): files.append(elem.get('subject')) if event == "end" and elem.tag.endswith('segment'): bytes = int(elem.get('bytes', 0)) bytes_total += bytes segments.put((files[-1], '<%s>' % elem.text, bytes)) seg_count += 1 size, unit = get_size(bytes_total) print "Found %d files and %d segments totalling %s %s" % ( len(files), seg_count, size, unit) pbar = ProgressBar(segments, missing) while not segments.empty(): pbar.update() time.sleep(0.1) pbar.finish() missing.join() num_missing = missing.qsize() if num_missing > 0: missing_bytes = 0 print "Result: missing %d/%d segments; %0.2f%% complete" % ( num_missing, seg_count, ((seg_count - num_missing) / seg_count * 100.00)) while not missing.empty(): f, seg, bytes = missing.get() missing_bytes += bytes print '\tfile="%s", segment="%s"' % (f, seg) size, unit = get_size(missing_bytes) print "Missing %s %s" % (size, unit) else: print "Result: all %d segments available" % seg_count thread.stop_threads(threads)