def test_wait_for_ssh(self, mock_conn): """Test the _wait_for_ssh() helper func.""" # if the connection attempts fail mock_conn.return_value = False q = g_queue.JoinableQueue(items=[('dummy.host', 1234), ('host', 1234)]) self.ssh_mod._wait_for_ssh(q, 'ssh', 'cmd', timeout=0, attempts=10) self.assertEqual(mock_conn.call_count, 10) # check that "task_done()" has been invoked as many times as elements # have been read from the queue with self.assertRaises(ValueError): q.task_done() # if the connection attempt is successful mock_conn.reset_mock() mock_conn.return_value = True q = g_queue.JoinableQueue(items=[('dummy.host', 1234), ('host', 1234)]) self.ssh_mod._wait_for_ssh(q, 'ssh', 'cmd', timeout=0, attempts=5) self.assertEqual(mock_conn.call_count, 1) # first two attempts fail, then new endpoint info is taken from the # queue and the new attempt is successful mock_conn.reset_mock() # connection attempt successful only at the third (host_B) attempt mock_conn.side_effect = lambda host, port: host == 'host_B' q = g_queue.JoinableQueue(items=[('dummy.host', 1234), ('host_A', 1234), ('host_B', 1234)]) self.ssh_mod._wait_for_ssh(q, 'ssh', 'cmd', timeout=0, attempts=5) self.assertEqual(mock_conn.call_count, 3) with self.assertRaises(ValueError): q.task_done()
def __init__(self, num_workers, callable, poll_timeout_secs=10, name=None, queue_size=None): """ Args: num_workers - The max number of concurrent greenlets in this pool. callable - Python callable that will be invoked with each task input. The callable should return the tuple that will be passed as input to all registered consumer pools, or None if nothing should be passed to consumer pools. poll_timeout_secs - How often the pool will check to see if it has been signaled to finish. name - An optional name for the worker pool; used in logging. queue_size - The size of the underlying input queue. If this is None, the size is unbounded, and adding a task input to this pool will never block the calling thread. If queue_size > 0, the queue may get filled up, in which case, adding a task input will block the calling thread. """ self.name = name self.poll_timeout_secs = poll_timeout_secs self.callable = callable self.queue = queue.JoinableQueue(maxsize=queue_size) self.driver = gevent.Greenlet(self._Driver) self.finish_signaled = gevent.event.Event() self.pool = pool.Pool(num_workers) self.consumer_pools = [] self.producer_pools = [] self.producer_pools_finished = 0 self.counter_time_since_add = None self.counter_num_inputs_added = None self.counter_queue_size_at_processing_time = None
def __init__(self, write_key=None, host=None, debug=False, max_queue_size=10000, send=True, on_error=None): require('write_key', write_key, string_types) self.queue = queue.JoinableQueue(max_queue_size) self.consumer = Consumer(self.queue, write_key, host=host, on_error=on_error) self.write_key = write_key self.on_error = on_error self.debug = debug self.send = send if debug: self.log.setLevel(logging.DEBUG) # if we've disabled sending, just don't start the consumer if send: # On program exit, allow the consumer thread to exit cleanly. # This prevents exceptions and a messy shutdown when the interpreter is # destroyed before the daemon thread finishes execution. However, it # is *not* the same as flushing the queue! To guarantee all messages # have been delivered, you'll still need to call flush(). atexit.register(self.join) self.consumer.start()
def _wait_for_app(ssh, app, command, queue=None): """Use websockets to wait for the app to start""" # JoinableQueue is filled with a dummy item otherwise queue.join() unblocks # immediately wo/ actually letting the ws_loop and _wait_for_ssh to run. queue = queue or g_queue.JoinableQueue(items=[('dummy.host', 1234)]) def on_message(result, queue=queue): """Callback to process trace message.""" _LOGGER.debug('Endpoint trase msg: %r', result) queue.put((result['host'], result['port'])) return False def on_error(result): """Callback to process errors.""" click.echo('Error: %s' % result['_error'], err=True) try: gevent.spawn(_wait_for_ssh, queue, ssh, command) gevent.spawn( ws_client.ws_loop, context.GLOBAL.ws_api(), { 'topic': '/endpoints', 'filter': app, 'proto': 'tcp', 'endpoint': 'ssh' }, False, on_message, on_error) queue.join() except ws_client.WSConnectionError: cli.bad_exit('Could not connect to any Websocket APIs')
def __init__(self): # Allow enqueuing of several API calls worth of work, which # right now allow 1000 key deletions per job. self.PAGINATION_MAX = 1000 self._q = queue.JoinableQueue(self.PAGINATION_MAX * 10) self._worker = gevent.spawn(self._work) self._parent_greenlet = gevent.getcurrent() self.closing = False
def test_task_done(self): channel = queue.JoinableQueue(0) X = object() gevent.spawn(channel.put, X) result = channel.get() assert result is X, (result, X) assert channel.unfinished_tasks == 1, channel.unfinished_tasks channel.task_done() assert channel.unfinished_tasks == 0, channel.unfinished_tasks
def test_task_done(self): channel = queue.JoinableQueue() X = object() gevent.spawn(channel.put, X) result = channel.get() self.assertIs(result, X) self.assertEqual(1, channel.unfinished_tasks) channel.task_done() self.assertEqual(0, channel.unfinished_tasks)
def test_queue_task_done(self): # Test to make sure a queue task completed successfully. q = Queue.JoinableQueue() # self.type2test() # XXX the same test in subclasses try: q.task_done() except ValueError: pass else: self.fail("Did not detect task count going negative")
def test_queue_join(self): # Test that a queue join()s successfully, and before anything else # (done twice for insurance). q = Queue.JoinableQueue() # self.type2test() # XXX the same test in subclass self.queue_join_test(q) self.queue_join_test(q) try: q.task_done() except ValueError: pass else: self.fail("Did not detect task count going negative")
def test_queue_task_done_with_items(self): # Passing items to the constructor allows for as # many task_done calls. Joining before all the task done # are called returns false # XXX the same test in subclass l = [1, 2, 3] q = Queue.JoinableQueue(items=l) for i in l: self.assertFalse(q.join(timeout=0.001)) self.assertEqual(i, q.get()) q.task_done() try: q.task_done() except ValueError: pass else: self.fail("Did not detect task count going negative") self.assertTrue(q.join(timeout=0.001))
def test_gevent_based_observer(temporary_dir): from populus.utils.observers.observers_gevent import DirWatcher import gevent from gevent import queue file_path_a = os.path.join(temporary_dir, 'file-a.txt') ensure_file_exists(file_path_a) change_queue = queue.JoinableQueue() def change_cb(*args): change_queue.put(args) watcher = DirWatcher(temporary_dir, change_cb) watcher.start() def empty_queue(): while True: try: gevent.sleep(1) change_queue.get(False) except queue.Empty: break else: change_queue.task_done() def assert_event(*expected): gevent.sleep(1) actual = change_queue.get(block=True, timeout=2) assert actual == expected change_queue.task_done() empty_queue() # write existing root file with open(file_path_a, 'w') as file_a: file_a.write('test') assert_event(file_path_a, 'modified')
import urllib2 def fetch_title(url): print "Running %s" % url data = urllib2.urlopen(url).read() print "Parsing %s" % url d = feedparser.parse(data) print "Parsed %s" % d.feed.get('title', '') return d.feed.get('title', '') def worker(): while True: url = q.get() try: fetch_title(url) finally: q.task_done() if __name__ == '__main__': q = queue.JoinableQueue() for i in range(5): gevent.spawn(worker) for url in "http://www.43folders.com/rss.xml/nhttp://feeds.feedburner.com/43folders/nhttp://www.43folders.com/rss.xml/nhttp://feeds.feedburner.com/43folders/nhttp://feeds.feedburner.com/AMinuteWithBrendan/nhttp://feeds.feedburner.com/AMinuteWithBrendan/nhttp://www.asianart.org/feeds/Lectures,Classes,Symposia.xml/nhttp://www.asianart.org/feeds/Performances.xml/nhttp://feeds.feedburner.com/ajaxian/nhttp://ajaxian.com/index.xml/nhttp://al3x.net/atom.xml/nhttp://feeds.feedburner.com/AmericanDrink/nhttp://feeds.feedburner.com/eod_full/nhttp://feeds.feedburner.com/typepad/notes/nhttp://feeds.dashes.com/AnilDash/nhttp://rss.sciam.com/assignment-impossible/feed/nhttp://blogs.scientificamerican.com/assignment-impossible//nhttp://feeds.feedburner.com/Beautiful-Pixels/nhttp://feeds.feedburner.com/Beautiful-Pixels/nhttp://www.betabeat.com/feed/".split('/n'): print "Spawning: %s" % url q.put(url) q.join() # block until all tasks are done
def test_issue_45(self): """Test that join() exits immediately if not jobs were put into the queue""" self.switch_expected = False q = queue.JoinableQueue() q.join()
def make_put_interrupt(queue): class TestPutInterrupt(GenericGetTestCase): Timeout = Full def wait(self, timeout): while not queue.full(): queue.put(1) return queue.put(2, timeout=timeout) TestPutInterrupt.__name__ += '_' + queue.__class__.__name__ return TestPutInterrupt for obj in [ queue.Queue(1), queue.JoinableQueue(1), queue.LifoQueue(1), queue.PriorityQueue(1), queue.Channel() ]: klass = make_put_interrupt(obj) globals()[klass.__name__] = klass del klass, obj del GenericGetTestCase if __name__ == '__main__': main()
globals()[klass.__name__] = klass del klass, queue_type def make_put_interrupt(queue): class TestPutInterrupt(GenericGetTestCase): Timeout = Full def wait(self, timeout): while not queue.full(): queue.put(1) return queue.put(2, timeout=timeout) TestPutInterrupt.__name__ += '_' + queue.__class__.__name__ return TestPutInterrupt for obj in [queue.Queue(1), queue.JoinableQueue(1), queue.LifoQueue(1), queue.PriorityQueue(1), queue.Channel()]: klass = make_put_interrupt(obj) globals()[klass.__name__] = klass del klass, obj del GenericGetTestCase if __name__ == '__main__': main()
def __init__(self): self._queue = queue.JoinableQueue() self._all_tasks = {}
class Task(object): def __init__(self, func, *args, **kwargs): self.args = args self.kwargs = kwargs self.func = func def __call__(self): self.func(*self.args, **self.kwargs) def consumer(): while True: try: task = _task_queue.get() except queue.Empty: pass else: try: task() finally: _task_queue.task_done() def add_task(task): _task_queue.put_nowait(task) _running_tasks = {} _task_queue = queue.JoinableQueue() _consumer = gevent.spawn(consumer)
from ..compat import py3k #assert not py3k, "this script requires python 2." import psycogreen.gevent from ..profile import avg_rec_rate from .. import util import psycopg2 from sqlalchemy.engine import url import gevent from gevent import queue psycogreen.gevent.patch_psycopg() options = None work_queue = queue.JoinableQueue() monitor = avg_rec_rate() connect = None def setup(opt): global options options = opt db_url = url.make_url(options.dburl) global connect def connect(): conn = psycopg2.connect( user=db_url.username, password=db_url.password, host=db_url.host,