def main(): from optparse import OptionParser parser = OptionParser(usage="usage: %prog [options]") parser.add_option("-D", "--debug", dest="debug", help="debug mode.", action="store_true", default=False) parser.add_option("-c", "--cpool", dest="cpool_size", help="size of coroutine pool.", metavar="<size>", type="int", default=512) opts, args = parser.parse_args() sockets = [] pool = coros.CoroutinePool(max_size=opts.cpool_size) if len(args) == 0: sockets.append(fcgi_stdin_sock()) else: for arg in args: # todo: add support for UNIX sockets addr = arg.replace('[', '').replace(']', '').rsplit(':', 1) if len(addr) != 2: parser.error( 'arguments must be IPv4 or IPv6 addresses including port') addr = (addr[0], int(addr[1])) sockets.append(api.tcp_listener(addr)) run(sockets, handle_connection, pool)
def test_a_buncha_stuff(self): pool = coros.CoroutinePool(max_size=10) waiters = [] for i in range(0, 9): waiters.append(pool.execute(sender_loop, i)) for waiter in waiters: waiter.wait()
def test_contention(self): from greentest import tpool_test prox = tpool.Proxy(tpool_test) pool = coros.CoroutinePool(max_size=4) waiters = [] waiters.append(pool.execute(lambda: self.assertEquals(prox.one, 1))) waiters.append(pool.execute(lambda: self.assertEquals(prox.two, 2))) waiters.append(pool.execute(lambda: self.assertEquals(prox.three, 3))) for waiter in waiters: waiter.wait()
def __init__(self, queue=None, max_size=DEFAULT_MAX_CONC_REQUESTS): self.log = log.get_logger('zenq.server.native:%x' % (id(self), )) # An initial queue may be provided; this might help with durable queues # (i.e. those that save their state to disk and can restore it on load). self.queue = queue or Queue() # The client pool is a pool of coroutines which doesn't allow more than # max_size coroutines to be running 'at the same time' (although # strictly speaking they never do anyway). In this case it represents # the maximum number of clients that may be connected at once. self.client_pool = coros.CoroutinePool(max_size=max_size) self.socket = None
queue.put(feed['url']) while not queue.empty(): latestDateTime = crawlFeedOnePage(feed, queue, crawlDateTime, latestWarc, warcDateTime, warcDateTime, tempdir) print "Finished crawling %s, whose feed was last updated on %s" % (feed['domain'], latestDateTime.isoformat()) os.rmdir(tempdir) latestWarc.destroy() ### Create a new warc everytime instead of adding to an old one #renameWarc(warcFileName, feed['domain'], domain_warc_dir, latestDateTime) # __main__ #_______________________________________________________________________________ assert os.path.exists(config['warc_dir']) writeLockFile(config['warc_dir']) crawlDateTime = datetime.datetime.utcnow() pool = coros.CoroutinePool(max_size=4) waiters = [] for feed in feeds: waiters.append(pool.execute(crawlDomain, feed, crawlDateTime)) #wait until all feeds have been fetched for waiter in waiters: waiter.wait() rmLockFile(config['warc_dir'])