def gits_download(self, url, output="/tmp", threads=20):
if not self.output:
self.output = output
results = self.index(url, output=output)
if not results:
return
args = [((i[0], i[1]), {}) for i in self.giturls]
# ... and build a WorkRequest object for each item in data
requests = makeRequests(self.callback,
args,
self.print_result,
self.handle_exception)
main = ThreadPool(threads)
for req in requests:
main.putRequest(req)
print "Work request #%s added." % req.requestID
i = 0
while True:
try:
main.poll()
print "Main thread working...",
print "(active worker threads: %i)" % (
threading.activeCount()-1, )
if i == 10:
print "**** Adding 3 more worker threads..."
main.createWorkers(3)
if i == 20:
print "**** Dismissing 2 worker threads..."
main.dismissWorkers(2)
i += 1
except KeyboardInterrupt:
print "**** Interrupted!"
break
except NoResultsPending:
print "**** No pending results."
break
if main.dismissedWorkers:
print "Joining all dismissed worker threads..."
main.joinAllDismissedWorkers()
backlog = 5
size = 1024
threads = 5
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((host,port))
s.listen(backlog)
sock = [s]
main = ThreadPool(threads)
main.createWorkers(threads)
while True:
if exit == 1:
main.joinAllDismissedWorkers()
sys.exit()
else:
requests = makeRequests(use_client, sock)
for req in requests:
main.putRequest(req)
updates = makeRequests(update_client, sock)
for update in updates:
main.putRequest(update)
time.sleep(0.5)
main.poll()
s.close
if main.dismissedWorkers:
print("Joining all dismissed worker threads...")
main.joinAllDismissedWorkers()
# ...and wait for the results to arrive in the result queue
# by using ThreadPool.wait(). This would block until results for
# all work requests have arrived:
# main.wait()
# instead we can poll for results while doing something else:
i = 0
while True:
try:
time.sleep(0.5)
main.poll()
print "Main thread working...",
print "(active worker threads: %i)" % (threading.activeCount()-1, )
if i == 10:
print "**** Adding 3 more worker threads..."
main.createWorkers(3)
if i == 20:
print "**** Dismissing 2 worker threads..."
main.dismissWorkers(2)
i += 1
except KeyboardInterrupt:
print "**** Interrupted!"
break
except NoResultsPending:
print "**** No pending results."
break
if main.dismissedWorkers:
print "Joining all dismissed worker threads..."
main.joinAllDismissedWorkers()
