def __init__(self, server_url, login_params): self.running = True self.agent = True self.cmd_out_queue = [] self.cmd_in_queue = [] self.out_queue = Queue() self.in_queue = Queue() self.server_url = server_url self.login_params = login_params Thread.__init__(self)
def setupDatamodel(self, datamodel): self.datamodel = datamodel self.doc.datamodel = datamodel_mapping[datamodel]() self.dm = self.doc.datamodel self.dm.response = Queue() self.dm.websocket = Queue() self.dm["__event"] = None # self.dm["_x"]["sessions"] = {} if datamodel != "xpath": self.dm["In"] = self.interpreter.In
def __init__(self): self.running = True self.exited = False self.cancelled = False self.configuration = OrderedSet() self.internalQueue = Queue() self.externalQueue = Queue() self.statesToInvoke = OrderedSet() self.historyValue = {} self.dm = None self.invokeId = None self.parentId = None self.logger = None
def test_blocks_on_pool(self): waiter = Queue(0) def greedy(): self.pool.get() self.pool.get() self.pool.get() self.pool.get() # No one should be waiting yet. self.assertEquals(self.pool.waiting(), 0) # The call to the next get will unschedule this routine. self.pool.get() # So this put should never be called. waiter.put('Failed!') killable = eventlet.spawn(greedy) # no one should be waiting yet. self.assertEquals(self.pool.waiting(), 0) ## Wait for greedy eventlet.sleep(0) ## Greedy should be blocking on the last get self.assertEquals(self.pool.waiting(), 1) ## Send will never be called, so balance should be 0. self.assertFalse(not waiter.full()) eventlet.kill(killable)
def handler(self, ws): self.queue = Queue() while True: m = ws.wait() # import ipdb; ipdb.set_trace() if m is None: break self.queue.put(m)
def test_connection_pooling(self): with patch('swift.common.memcached.socket') as mock_module: # patch socket, stub socket.socket, mock sock mock_sock = mock_module.socket.return_value # track clients waiting for connections connected = [] connections = Queue() errors = [] def wait_connect(addr): connected.append(addr) sleep(0.1) # yield val = connections.get() if val is not None: errors.append(val) mock_sock.connect = wait_connect memcache_client = memcached.MemcacheRing(['1.2.3.4:11211'], connect_timeout=10) # sanity self.assertEquals(1, len(memcache_client._client_cache)) for server, pool in memcache_client._client_cache.items(): self.assertEqual(2, pool.max_size) # make 10 requests "at the same time" p = GreenPool() for i in range(10): p.spawn(memcache_client.set, 'key', 'value') for i in range(3): sleep(0.1) self.assertEqual(2, len(connected)) # give out a connection connections.put(None) # at this point, only one connection should have actually been # created, the other is in the creation step, and the rest of the # clients are not attempting to connect. we let this play out a # bit to verify. for i in range(3): sleep(0.1) self.assertEqual(2, len(connected)) # finish up, this allows the final connection to be created, so # that all the other clients can use the two existing connections # and no others will be created. connections.put(None) connections.put('nono') self.assertEqual(2, len(connected)) p.waitall() self.assertEqual(2, len(connected)) self.assertEqual(0, len(errors), "A client was allowed a third connection") connections.get_nowait() self.assertTrue(connections.empty())
def __init__(self, chunk, conn, write_timeout=None, **_kwargs): self._chunk = chunk self._conn = conn self.failed = False self.bytes_transferred = 0 self.checksum = hashlib.md5() self.write_timeout = write_timeout or io.CHUNK_TIMEOUT # we use eventlet Queue to pass data to the send coroutine self.queue = Queue(io.PUT_QUEUE_DEPTH)
def _restart(self): self._pkt_trans = PacketTransceiver(self._node_id, self._dp, self._flood_ports) demux, trans = build_transceiver(self._cps, self._pkt_trans) for cpid in self._cps: q = Queue() self._qs[cpid] = q if os.path.exists('/common/pp'): v = PushPullECSMgr(self._node_id, q, self._topo, trans[cpid]) else: v = FloodECSMgr(self._node_id, q, self._topo, trans[cpid]) spawn(v.run)
def test_putting_to_queue(self): timer = eventlet.Timeout(0.1) try: size = 2 self.pool = IntPool(min_size=0, max_size=size) queue = Queue() results = [] def just_put(pool_item, index): self.pool.put(pool_item) queue.put(index) for index in xrange(size + 1): pool_item = self.pool.get() eventlet.spawn(just_put, pool_item, index) for _ in range(size+1): x = queue.get() results.append(x) self.assertEqual(sorted(results), range(size + 1)) finally: timer.cancel()
def take_action(self, parsed_args): self.log.debug('take_action(%s)', parsed_args) digits = self.app.client_manager.get_meta1_digits() workers_count = parsed_args.workers conf = {'namespace': self.app.client_manager.namespace} if parsed_args.proxy: conf.update({'proxyd_url': parsed_args.proxy}) else: ns_conf = load_namespace_conf(conf['namespace']) proxy = ns_conf.get('proxy') conf.update({'proxyd_url': proxy}) workers = list() with green.ContextPool(workers_count) as pool: pile = GreenPile(pool) prefix_queue = Queue(16) # Prepare some workers for i in range(workers_count): w = WarmupWorker(conf, self.log) workers.append(w) pile.spawn(w.run, prefix_queue) # Feed the queue trace_increment = 0.01 trace_next = trace_increment sent, total = 0, float(count_prefixes(digits)) for prefix in generate_prefixes(digits): sent += 1 prefix_queue.put(prefix) # Display the progression ratio = float(sent) / total if ratio >= trace_next: self.log.info("... %d%%", int(ratio * 100.0)) trace_next += trace_increment self.log.debug("Send the termination marker") prefix_queue.join() self.log.info("All the workers are done")
def test_exhaustion(self): waiter = Queue(0) def consumer(): gotten = None try: gotten = self.pool.get() finally: waiter.put(gotten) eventlet.spawn(consumer) one, two, three, four = ( self.pool.get(), self.pool.get(), self.pool.get(), self.pool.get()) self.assertEquals(self.pool.free(), 0) # Let consumer run; nothing will be in the pool, so he will wait eventlet.sleep(0) # Wake consumer self.pool.put(one) # wait for the consumer self.assertEquals(waiter.get(), one)
"/home/pi/craftbeerpi3/modules/plugins/FermentWifiPlugin/roda.txt", "w") file.write("nao") file.close() os.system("reboot") else: file.close() import paho.mqtt.client as mqtt a_ativado = 0 r_ativado = 0 cache = {} q = Queue() cbpi.gpio_compressors = [] cbpi.gpio_compressors2 = [] client = None mqttc = mqtt.Client() mqttc.connect("localhost", 1883, 60) mqttc.loop_start() @cbpi.actor class Resfriador_FermentWifi(ActorBase): key0 = Property.Text(label="Nome do FermentWifi (ex: FW_0000)",
def _decode_segments(self, fragment_iterators): """ Reads from fragments and yield full segments """ # we use eventlet Queue to read fragments queues = [] # each iterators has its queue for _j in range(len(fragment_iterators)): queues.append(Queue(1)) def put_in_queue(fragment_iterator, queue): """ Coroutine to read the fragments from the iterator """ try: for fragment in fragment_iterator: # put the read fragment in the queue queue.put(fragment) # the queues are of size 1 so this coroutine blocks # until we decode a full segment except GreenletExit: # ignore pass except green.ChunkReadTimeout: logger.error("Timeout on reading") except Exception: logger.exception("Exception on reading") finally: queue.resize(2) # put None to indicate the decoding loop # this is over queue.put(None) # close the iterator fragment_iterator.close() # we use eventlet GreenPool to manage the read of fragments with green.ContextPool(len(fragment_iterators)) as pool: # spawn coroutines to read the fragments for fragment_iterator, queue in zip(fragment_iterators, queues): pool.spawn(put_in_queue, fragment_iterator, queue) # main decoding loop while True: data = [] # get the fragments from the queues for queue in queues: fragment = queue.get() queue.task_done() data.append(fragment) if not all(data): # one of the readers returned None # impossible to read segment break # actually decode the fragments into a segment try: segment = self.storage_method.driver.decode(data) except exceptions.ECError: # something terrible happened logger.exception("ERROR decoding fragments") raise yield segment
def main(): args = options() global ACCOUNT, PROXY, QUEUE, NS, VERBOSE, TIMEOUT global COUNTERS, ELECTIONS ACCOUNT = args.account NS = args.namespace VERBOSE = args.verbose TIMEOUT = args.timeout PROXY = ObjectStorageApi(NS) ELECTIONS = AtomicInteger() num_worker_threads = int(args.max_worker) print("Using %d workers" % num_worker_threads) total_objects = {'size': 0, 'files': 0, 'elapsed': 0} total_containers = {'size': 0, 'files': 0, 'elapsed': 0} for path in args.path: path = path.rstrip('/') if '/' in path: bucket, path = path.split('/', 1) else: bucket = path path = "" containers = [] QUEUE = Queue() pool = eventlet.GreenPool(num_worker_threads) for i in range(num_worker_threads): pool.spawn(worker_objects) COUNTERS = AtomicInteger() _bucket = container_hierarchy(bucket, path) # we don't use placeholders, we use prefix path as prefix for entry in full_list(prefix=container_hierarchy(bucket, path)): name, _files, _size, _ = entry if name != _bucket and not name.startswith(_bucket + '%2F'): continue if _files: QUEUE.put(name) containers.append(name) # we have to wait all objects print("Waiting flush of objects") report = args.report while not QUEUE.empty(): ts = time.time() while time.time() - ts < report and not QUEUE.empty(): time.sleep(1) diff = time.time() - ts val = COUNTERS.reset() elections = ELECTIONS.reset() print("Objects: %5.2f / Size: %5.2f" % (val[0] / diff, val[1] / diff), "Elections failed: %5.2f/s total: %d" % (elections[0] / diff, ELECTIONS.total()[0]), " " * 20, end='\r') sys.stdout.flush() print("Waiting end of workers") QUEUE.join() val = COUNTERS.total() total_objects['files'] += val[0] total_objects['size'] += val[1] total_objects['elapsed'] += COUNTERS.time() COUNTERS = AtomicInteger() QUEUE = Queue() for i in range(num_worker_threads): pool.spawn(worker_container) print("We have to delete", len(containers), "containers") for container in containers: QUEUE.put(container) while not QUEUE.empty(): ts = time.time() while time.time() - ts < report and not QUEUE.empty(): time.sleep(1) diff = time.time() - ts val = COUNTERS.reset() elections = ELECTIONS.reset() print("Containers: %5.2f" % (val[0] / diff), "Elections failed: %5.2f/s total: %d" % (elections[0] / diff, ELECTIONS.total()[0]), " " * 20, end='\r') sys.stdout.flush() QUEUE.join() val = COUNTERS.total() total_containers['files'] += val[0] total_containers['size'] += val[1] total_containers['elapsed'] += COUNTERS.time() print(""" Objects: - ran during {o[elapsed]:5.2f} - {o[files]} objects removed (size {size}) - {o_file_avg:5.2f} objects/s ({o_size_avg} avg. size/s) """.format(o=total_objects, size=show(total_objects['size'], True), o_file_avg=total_objects['files'] / total_objects['elapsed'], o_size_avg=show(total_objects['size'] / total_objects['elapsed'], True))) print(""" Containers: - ran during {o[elapsed]:5.2f} - {o[files]} containers - {o_file_avg:5.2f} containers/s """.format(o=total_containers, o_file_avg=total_containers['files'] / total_containers['elapsed'])) print("Elections failed: %d" % ELECTIONS.total()[0])
def __call__(self): """ :return httplib.HTTP(S)Connection in success, and webob.exc.HTTPException in failure """ if self.headers.has_key('content-length'): if int(self.headers['content-length']) >= MAX_FILE_SIZE: return HTTPRequestEntityTooLarge(request=self.req) parsed = urlparse(self.url) if self.proxy: proxy_parsed = urlparse(self.proxy) if self._proxy_request_check(parsed.path): host, port = self.split_netloc(proxy_parsed) path = self.url ssl = True if proxy_parsed.scheme == 'https' else False else: host, port = self.split_netloc(parsed) path = parsed.path ssl = True if parsed.scheme == 'https' else False self.headers['host'] = '%s:%s' % (host, port) if self.method == 'PUT' and len(parsed.path.split('/')) >= 5: if self.headers.has_key('content-length') and int( self.headers['content-length']) != 0: if not self.headers.has_key('expect'): self.headers['expect'] = '100-continue' chunked = self.req.headers.get('transfer-encoding') if isinstance(self.req.environ['wsgi.input'], str): reader = self.req.environ['wsgi.input'].read data_source = iter(lambda: reader(self.chunk_size), '') else: data_source = self.req.environ['wsgi.input'] bytes_transferred = 0 try: conn = self._connect_put_node(host, port, self.method, path, headers=self.headers, query_string=parsed.query, ssl=ssl) if not conn: return HTTPServiceUnavailable(request=self.req) with ContextPool(1) as pool: conn.failed = False conn.queue = Queue(10) pool.spawn(self._send_file, conn, path) while True: with ChunkReadTimeout(self.client_timeout): try: chunk = next(data_source) except StopIteration: if chunked: conn.queue.put('0\r\n\r\n') break except TypeError, err: self.logger.info('Chunk Read Error: %s' % err) break except Exception, err: self.logger.info('Chunk Read Error: %s' % err) return HTTPServerError(request=self.req) bytes_transferred += len(chunk) if bytes_transferred > MAX_FILE_SIZE: return HTTPRequestEntityTooLarge(request=self.req) if not conn.failed: conn.queue.put('%x\r\n%s\r\n' % (len(chunk), chunk) if chunked else chunk)
def __call__(self): """ :return httplib.HTTP(S)Connection in success, and webob.exc.HTTPException in failure """ if self.headers.has_key('content-length'): if int(self.headers['content-length']) >= MAX_FILE_SIZE: return HTTPRequestEntityTooLarge(request=self.req) parsed = urlparse(self.url) if self.proxy: proxy_parsed = urlparse(self.proxy) if self._proxy_request_check(parsed.path): host, port = self.split_netloc(proxy_parsed) path = self.url else: host, port = self.split_netloc(parsed) path = parsed.path self.headers['host'] = '%s:%s' % (host, port) if self.method == 'PUT' and len(parsed.path.split('/')) == 5: chunked = self.req.headers.get('transfer-encoding') reader = self.req.environ['wsgi.input'].read data_source = iter(lambda: reader(self.chunk_size), '') bytes_transferred = 0 # pile = GreenPile() # pile.spawn(self._connect_server, host, port, self.method, path, self.headers, parsed.query) # conns = [conn for conn in pile if conn] # conn = conns[0] try: with ConnectionTimeout(self.conn_timeout): conn = http_connect_raw(host, port, self.method, path, headers=self.headers, query_string=parsed.query) with ContextPool(1) as pool: conn.failed = False conn.queue = Queue(10) pool.spawn(self._send_file, conn, path) while True: with ChunkReadTimeout(self.client_timeout): try: chunk = next(data_source) except StopIteration: if chunked: conn.queue.put('0\r\n\r\n') break except TypeError, err: self.logger.info('Chunk Read Error: %s' % err) break except Exception, err: self.logger.info('Chunk Read Error: %s' % err) return HTTPServerError(request=self.req) bytes_transferred += len(chunk) if bytes_transferred > MAX_FILE_SIZE: return HTTPRequestEntityTooLarge(request=self.req) if not conn.failed: conn.queue.put('%x\r\n%s\r\n' % (len(chunk), chunk) if chunked else chunk)
def start(self, pool): # we use eventlet Queue to pass data to the send coroutine self.queue = Queue(io.PUT_QUEUE_DEPTH) # spawn the send coroutine pool.spawn(self._send)
def _get_queue(self): from eventlet import Queue return Queue()