def xreader(): in_queue = Queue(buffer_size) out_queue = Queue(buffer_size) out_order = [0] # start a read worker in a thread target = order_read_worker if order else read_worker t = Thread(target=target, args=(reader, in_queue)) t.daemon = True t.start() # start several handle_workers target = order_handle_worker if order else handle_worker args = (in_queue, out_queue, mapper, out_order) if order else (in_queue, out_queue, mapper) workers = [] for i in range(process_num): worker = Thread(target=target, args=args) worker.daemon = True workers.append(worker) for w in workers: w.start() sample = out_queue.get() while not isinstance(sample, XmapEndSignal): yield sample sample = out_queue.get() finish = 1 while finish < process_num: sample = out_queue.get() if isinstance(sample, XmapEndSignal): finish += 1 else: yield sample
def put_tile(self, tile, img): params = dict(tile=tile, img=None if img is None else img.copy(), uuid=self.uuid.hex, db_path=self.tilestor_path) writer = TilestorWriter.getInstance() if self.async_writing: answer_queue = Queue(maxsize=1) params['answer_queue'] = answer_queue try: writer.put(params) except TileWriterQueueException as exc: _logger.error( "Failed to put tile {} to tile cache for resource {}. {}". format(params['tile'], self.resource_id, exc.message), exc_info=True) if self.async_writing: try: answer_queue.get() except Exception: pass
def test_blocked_cancel(any_executor, request): if "sync" in request.node.name: skip("test not applicable with sync executor") to_fn = Queue(1) from_fn = Queue(1) def fn(): to_fn.get() from_fn.put(None) to_fn.get() return 123 future = any_executor.submit(fn) # Wait until fn is certainly running to_fn.put(None) from_fn.get() # Since the function is in progress, # it should NOT be possible to cancel it assert_that(not future.cancel(), str(future)) # assert_soon since, by blocking on from_fn.get(), we only guarantee # that the future is running from the innermost executor/future's point # of view, but this may not have propagated to the outermost future yet assert_soon(lambda: assert_that(future.running(), str(future))) # Re-check cancel after running() is true assert_that(not future.cancel(), str(future)) # Let fn proceed and the future should be able to complete to_fn.put(None) assert_that(future.result(TIMEOUT), equal_to(123))
class ThreadManager(six.with_metaclass(Singleton, object)): """ ThreadManager provides thread on demand """ NUM_THREAD = 4 # Default number of threads def __init__(self, num_thread=NUM_THREAD): """ Create num_thread Threads """ self.queue = Queue() self.thread_list = [] for i in range(num_thread): t = Thread(target=worker, args=(self.queue, )) t.setDaemon(True) t.start() self.thread_list.append(t) def add_task(self, func, params): """ Add a task to perform :param func: function to call :param params : tuple of parameters """ self.queue.put((func, params)) def clear(self): """ clear pending task """ while (not self.queue.empty()): self.queue.get()
def test(add_after_start, generator): queue = Queue() e = {} em = element_updater.ElementUpdater() async def fn(element): element.setdefault('called', []).append(True) queue.put(True) count = 1 if generator: count = 2 old_fn = fn async def fn(element): for _ in range(2): await old_fn(element) yield updater = element_updater.Updater(element=e, updater=fn) if add_after_start: em.start() em.add(updater) else: em.add(updater) em.start() for _ in range(count): queue.get(timeout=3) assert e['called'] == [True] * count
def generator_to_async_generator(get_iterable): """ Turn a generator or iterable into an async generator. This works by running the generator in a background thread. The new async generator will yield both `Future` objects as well as the original items. :param get_iterable: Function that returns a generator or iterable when called. """ q = Queue() f = Future() l = RLock() quitting = False def runner(): """ Consume the generator in background thread. When items are received, they'll be pushed to the queue and the Future is set. """ for item in get_iterable(): with l: q.put(item) if not f.done(): f.set_result(None) # When this async generator was cancelled (closed), stop this # thread. if quitting: break with l: if not f.done(): f.set_result(None) # Start background thread. done_f = run_in_executor(runner, _daemon=True) try: while not done_f.done(): # Wait for next item(s): yield Future. yield From(f) # Items received. Yield all items so far. with l: while not q.empty(): yield AsyncGeneratorItem(q.get()) f = Future() # Yield final items. while not q.empty(): yield q.get() finally: # When this async generator is closed (GeneratorExit exception, stop # the background thread as well. - we don't need that anymore.) quitting = True
def _maybeFixReactorThreadRace(self): # pragma: no cover # XXX For some obscure reason, this is needed in order to have the # reactor properly wait for the shutdown sequence. It's probably # a race between this thread and the reactor thread. Needs # investigation. spin = Queue() self.reactor.callFromThread(self.reactor.callLater, 0, spin.put, None) spin.get(timeout=self.timeout)
def run_load_stats(state, *args, **kwargs): """ Run load_stats() in a thread """ queue = Queue() thread_args = [state, queue] thread_args.extend(args) thread = Thread(target=load_stats, args=thread_args, kwargs=kwargs) thread.daemon = True thread.start() # Wait until the thread is definitely started queue.get(True) return queue
class SubscribeListener(SubscribeCallback): def __init__(self): self.connected = False self.connected_event = Event() self.disconnected_event = Event() self.presence_queue = Queue() self.message_queue = Queue() def status(self, pubnub, status): if utils.is_subscribed_event( status) and not self.connected_event.is_set(): self.connected_event.set() elif utils.is_unsubscribed_event( status) and not self.disconnected_event.is_set(): self.disconnected_event.set() def message(self, pubnub, message): self.message_queue.put(message) def presence(self, pubnub, presence): self.presence_queue.put(presence) def wait_for_connect(self): if not self.connected_event.is_set(): self.connected_event.wait() else: raise Exception("the instance is already connected") def wait_for_disconnect(self): if not self.disconnected_event.is_set(): self.disconnected_event.wait() else: raise Exception("the instance is already disconnected") def wait_for_message_on(self, *channel_names): channel_names = list(channel_names) while True: env = self.message_queue.get() self.message_queue.task_done() if env.channel in channel_names: return env else: continue def wait_for_presence_on(self, *channel_names): channel_names = list(channel_names) while True: env = self.presence_queue.get() self.presence_queue.task_done() if env.channel in channel_names: return env else: continue
def data_reader(): r = reader() q = Queue(maxsize=size) t = Thread(target=read_worker, args=( r, q, )) t.daemon = True t.start() e = q.get() while e != end: yield e e = q.get()
class SubscribeListener(SubscribeCallback): def __init__(self): self.connected = False self.connected_event = Event() self.disconnected_event = Event() self.presence_queue = Queue() self.message_queue = Queue() def status(self, pubnub, status): if utils.is_subscribed_event(status) and not self.connected_event.is_set(): self.connected_event.set() elif utils.is_unsubscribed_event(status) and not self.disconnected_event.is_set(): self.disconnected_event.set() def message(self, pubnub, message): self.message_queue.put(message) def presence(self, pubnub, presence): self.presence_queue.put(presence) def wait_for_connect(self): if not self.connected_event.is_set(): self.connected_event.wait() else: raise Exception("the instance is already connected") def wait_for_disconnect(self): if not self.disconnected_event.is_set(): self.disconnected_event.wait() else: raise Exception("the instance is already disconnected") def wait_for_message_on(self, *channel_names): channel_names = list(channel_names) while True: env = self.message_queue.get() self.message_queue.task_done() if env.channel in channel_names: return env else: continue def wait_for_presence_on(self, *channel_names): channel_names = list(channel_names) while True: env = self.presence_queue.get() self.presence_queue.task_done() if env.channel in channel_names: return env else: continue
class WinPty(object): def __init__(self, stdin): self._s = stdin self._q = Queue() def _read_next_char(stdin, queue): while True: char = stdin.read(1) # potentially blocking read if char: queue.put(char) else: break self._t = Thread(target=_read_next_char, args=(self._s, self._q)) self._t.daemon = True self._t.start() # read characters asynchronously from stdin def read(self, blksize=-1, timeout=1): buf = StringIO() count = 0 try: while count < blksize or blksize == -1: next = self._q.get(block=timeout is not None, timeout=timeout) buf.write(next) count = count + 1 except Empty: pass return buf.getvalue()
class Called(object): def __init__(self, obj, attr, request): self.obj = obj self.attr = attr self.request = request self._queue = Queue() self.orig = getattr(self.obj, self.attr) setattr(self.obj, self.attr, self._call) def restore(self): if hasattr(self, "orig"): setattr(self.obj, self.attr, self.orig) def _call(self, *args, **kwargs): try: rv = self.orig(*args, **kwargs) except Exception as e: self._queue.put((args, kwargs, e)) raise else: self._queue.put((args, kwargs, rv)) return rv def get(self, timeout=1): assert timeout try: rv = self._queue.get(timeout=timeout) except Empty: pytest.fail("no item received within %s seconds" % timeout) if isinstance(rv[2], Exception): raise rv[2]
class FrameSaver( threading.Thread ): def __init__( self ): threading.Thread.__init__( self ) self.daemon = True self.name = 'FrameSaver' self.reset() def reset( self ): self.queue = Queue() def run( self ): self.reset() while 1: message = self.queue.get() if message[0] == 'Save': cmd, fileName, bib, t, frame = message #sys.stderr.write( 'save' ) PhotoFinish.SavePhoto( fileName, bib, t, frame ) self.queue.task_done() elif message[0] == 'Terminate': self.queue.task_done() self.reset() break def stop( self ): self.queue.put( ['Terminate'] ) self.join() def save( self, fileName, bib, t, frame ): self.queue.put( ['Save', fileName, bib, t, frame] )
def blockingCallFromThread(reactor, f, *args, **kwargs): """ Improved version of twisted's blockingCallFromThread that shows the complete stacktrace when an exception is raised on the reactor's thread. If being called from the reactor thread already, just return the result of execution of the callable. """ if isInIOThread(): return f(*args, **kwargs) else: queue = Queue() def _callFromThread(): result = defer.maybeDeferred(f, *args, **kwargs) result.addBoth(queue.put) reactor.callFromThread(_callFromThread) result = queue.get() if isinstance(result, Failure): other_thread_tb = traceback.extract_tb(result.getTracebackObject()) this_thread_tb = traceback.extract_stack() logger.error( "Exception raised on the reactor's thread %s: \"%s\".\n Traceback from this thread:\n%s\n" " Traceback from the reactor's thread:\n %s", result.type.__name__, result.getErrorMessage(), ''.join(traceback.format_list(this_thread_tb)), ''.join(traceback.format_list(other_thread_tb))) result.raiseException() return result
class Consumer(object): """ A consumer is registered to the read thread before a command is issued. If an output matches the consumer, the output will unblock the get() caller. """ def __init__(self, cmd, cid): self.cmd = cmd # type: MasterCommandSpec self.cid = cid # type: int self.__queue = Queue() def get_prefix(self): """ Get the prefix of the answer from the master. """ return self.cmd.output_action + bytearray([self.cid]) def consume(self, data, partial_result): # type: (bytearray, Optional[Result]) -> Tuple[int, Result, bool] """ Consume data. """ return self.cmd.consume_output(data, partial_result) def get(self, timeout): """ Wait until the master replies or the timeout expires. :param timeout: timeout in seconds :raises: :class`CommunicationTimedOutException` if master did not respond in time :returns: dict containing the output fields of the command """ try: return self.__queue.get(timeout=timeout) except Empty: raise CommunicationTimedOutException() def deliver(self, output): # type: (Result) -> None """ Deliver output to the thread waiting on get(). """ self.__queue.put(output)
def test_equipped(self): """ Created equipped worker that will use an internal Counter resource to keep track of the job count. """ results = Queue() def toolbox_factory(): return Counter() def worker_factory(job_queue): return workerpool.EquippedWorker(job_queue, toolbox_factory) pool = workerpool.WorkerPool(1, worker_factory=worker_factory) # Run 10 jobs for i in range(10): j = CountJob(results) pool.put(j) # Get 10 results for i in range(10): r = results.get() # Each result should be an incremented value self.assertEquals(r, i) pool.shutdown()
class _SBPQueueIterator(six.Iterator): """ Class for upstream iterators. Implements callable interface for adding messages into the queue, and iterable interface for getting them out. """ def __init__(self, maxsize): self._queue = Queue(maxsize) self._broken = False def __iter__(self): return self def __call__(self, msg, **metadata): self._queue.put((msg, metadata), False) def breakiter(self): self._broken = True self._queue.put(None, True, 1.0) def __next__(self): if self._broken and self._queue.empty(): raise StopIteration m = self._queue.get(True) if self._broken and m is None: raise StopIteration return m
def inner(self, *args, **kwargs): if self.use_post_event: # create ephemeral queue q = Queue() # create an invocation that calls the decorated function class Invocation(object): def __call__(killme): # when the invocation is called, we call the function and stick the result into the queue try: res = func(self, *args, **kwargs) except Exception as e: # if we got an exception, just queue that instead res = e q.put(res) # post this invocation to be called on the main thread at the next opportunity gdb.post_event(Invocation()) # now we wait until there's something in the queue, which indicates that the invocation has run and return # the result that was pushed onto the queue by the invocation res = q.get() # if we got an exception back from the posted event, raise it if isinstance(res, Exception): raise res return res else: return func(self, *args, **kwargs)
def make_buffer_for_iterator_with_thread(gen, n_workers, buffer_size): wait_time = 0.02 generator_queue = Queue() _stop = threading.Event() def generator_task(): while not _stop.is_set(): try: if generator_queue.qsize() < buffer_size: generator_output = next(gen) generator_queue.put(generator_output) else: time.sleep(wait_time) except (StopIteration, KeyboardInterrupt): _stop.set() return generator_threads = [threading.Thread(target=generator_task) for _ in range(n_workers)] for thread in generator_threads: thread.start() while not _stop.is_set() or not generator_queue.empty(): if not generator_queue.empty(): yield generator_queue.get() else: time.sleep(wait_time)
class Future(object): def __init__(self): self._result = Queue(maxsize=1) self._success_callback = None self._failure_callback = None def get(self, timeout=None): """ Return value on success, or raise exception on failure. """ result = None try: result = self._result.get(True, timeout=timeout) except Empty: raise Timeout() if isinstance(result, Failure): six.reraise(*result.exc_info) else: return result def on_success(self, callback): self._success_callback = callback def on_failure(self, callback): self._failure_callback = callback
class Executor(object): _INTERRUPT = object() def __init__(self, num_workers=1): super(Executor, self).__init__() self._queue = Queue() self._workers = [] for _ in range(num_workers): th = Thread(target=self._work) th.start() self._workers.append(th) def submit(self, task): self._queue.put(task) def shutdown(self): for _ in self._workers: self._queue.put(self._INTERRUPT) def join(self): for worker in self._workers: worker.join() def _work(self): while True: task = self._queue.get(block=True) if task is self._INTERRUPT: break try: task() except BaseException as e: logger.exception(e)
def acquireResource(self, namespace, name, lockType, timeout=None): """ Acquire a resource synchronously. :returns: a reference to the resource. """ if timeout is not None: try: timeout = int(timeout) except ValueError: raise TypeError("'timeout' must be number") resource = Queue() def callback(req, res): resource.put(res) request = self.registerResource(namespace, name, lockType, callback) request.wait(timeout) if not request.granted(): try: request.cancel() raise RequestTimedOutError("Request timed out. Could not " "acquire resource '%s.%s'" % (namespace, name)) except RequestAlreadyProcessedError: # We might have acquired the resource between 'wait' and # 'cancel' if request.canceled(): raise se.ResourceAcqusitionFailed() return resource.get()
class WorkerSetup: use_callback = False def __init__(self, request, testdir): self.request = request self.testdir = testdir self.events = Queue() def setup(self,): self.testdir.chdir() # import os ; os.environ['EXECNET_DEBUG'] = "2" self.gateway = execnet.makegateway() self.config = config = self.testdir.parseconfigure() putevent = self.use_callback and self.events.put or None class DummyMananger: specs = [0, 1] self.slp = WorkerController(DummyMananger, self.gateway, config, putevent) self.request.addfinalizer(self.slp.ensure_teardown) self.slp.setup() def popevent(self, name=None): while 1: if self.use_callback: data = self.events.get(timeout=WAIT_TIMEOUT) else: data = self.slp.channel.receive(timeout=WAIT_TIMEOUT) ev = EventCall(data) if name is None or ev.name == name: return ev print("skipping %s" % (ev,)) def sendcommand(self, name, **kwargs): self.slp.sendcommand(name, **kwargs)
def test_websocket_api(): client = boto3.client('apigatewayv2', endpoint_url=TEST_APIGATEWAYV2_URL) queue = Queue() msg = {'action': 'test-action'} async def start_client(uri): async with websockets.connect(uri) as websocket: print('Sending message to websocket') await websocket.send(json.dumps(msg)) print('Waiting for response message from websocket ...') result = await websocket.recv() print('Received message from websocket: %s' % result) queue.put(json.loads(result)) apis = client.get_apis()['Items'] api = [a for a in apis if 'localstack-websockets' in a['Name']][0] url = api['ApiEndpoint'] print('Connecting to websocket URL %s' % url) asyncio.get_event_loop().run_until_complete(start_client(url)) result = queue.get(timeout=3) result_body = result['body'] result_body = json.loads(result_body) assert result_body == msg
class Consumer(object): """ A consumer is registered to the read thread before a command is issued. If an output matches the consumer, the output will unblock the get() caller. """ def __init__(self, command, cid): # type: (CoreCommandSpec, int) -> None self.cid = cid self.command = command self._queue = Queue() # type: Queue[Dict[str, Any]] def get_hash(self): # type: () -> int """ Get an identification hash for this consumer. """ return Toolbox.hash(CoreCommunicator.START_OF_REPLY + bytearray([self.cid]) + self.command.response_instruction) def consume(self, payload): # type: (bytearray) -> None """ Consume payload. """ data = self.command.consume_response_payload(payload) self._queue.put(data) def get(self, timeout): # type: (Union[T_co, int]) -> Dict[str, Any] """ Wait until the Core replies or the timeout expires. :param timeout: timeout in seconds :returns: dict containing the output fields of the command """ try: return self._queue.get(timeout=timeout) except Empty: raise CommunicationTimedOutException( 'No Core data received in {0}s'.format(timeout))
def test_db(db_glob, num_thread): # python 2.7 does not support glob such as ".../**" to do recursively # to access all files under some directory, we have to implement ourselves. db = get_file_recursively(db_glob) # using python 3.5, a simple form: # db = glob(db_glob) print("{} contains {} items".format(db_glob, len(db))) index_queue = Queue() result_queue = Queue() workers = [Thread(target=worker, args=(db, index_queue, result_queue)) for _ in range(num_thread)] for w in workers: w.start() t1 = time.time() for i in range(len(db)): index_queue.put(i) total_len = 0 for i in range(len(db)): total_len += result_queue.get() t2 = time.time() for i in range(len(workers)): index_queue.put(None) print("{} processing {} items with {} threads uses {}s, avg {}/s, avg {}B/s, avg {}MiB/s".format(db_glob, len(db), num_thread, (t2 - t1), len(db) / (t2 - t1), total_len / (t2 - t1), total_len / (t2 - t1) / 1024 / 1024))
class BufferedIterator(six.Iterator): def __init__(self, source, buffer_size=2): assert buffer_size >= 2, "minimum buffer size is 2" # The effective buffer size is one larger, because the generation # process will generate one extra element and block until there is room # in the buffer. self.buffer = Queue(maxsize=buffer_size - 1) def populate_buffer(): try: for item in source: self.buffer.put((None, item)) except: self.buffer.put((sys.exc_info(), None)) else: self.buffer.put(DONE) thread = threading.Thread(target=populate_buffer) thread.daemon = True thread.start() def __iter__(self): return self def __next__(self): value = self.buffer.get() if value is DONE: raise StopIteration() exc_info, data = value if exc_info: six.reraise(*exc_info) return data
class Scheduler(object): def __init__(self): self.queue = Queue() self._filter_set = set() def add_request(self, request): if self._filter_request(request): self._filter_set.add(request.url) self.queue.put(request) def get_request(self): try: return self.queue.get(False) except: return False def _filter_request(self, request): """ 请求去重,并返回判断结果 """ # 如果请求的url地址不再去重集合中,那么返回True,表示允许添加到请求队列中 if request.url not in self._filter_set: return True else: # 否则,表示重复, 不允许添加 return False
class Scheduler(): '''完成调取器模块的封装''' def __init__(self): self.queue = Queue() def add_request(self, request): ''' 实现添加request到队列中 :param request: 请求对象 :return: None ''' # url去重 # self._filter_request(request) self.queue.put(request) def get_request(self): ''' 实现获取队列中的request对象 :return: 请求对象 ''' return self.queue.get() def _filter_request(self, request): ''' 实现对请求对象的去重 :param request: 请求对象 :return: bool ''' pass
class NonBlockingStreamReader: def __init__(self, stream): ''' stream: the stream to read from. Usually a process' stdout or stderr. ''' self._s = stream self._q = Queue() def _populateQueue(stream, queue): ''' Collect lines from 'stream' and put them in 'quque'. ''' while True: line = stream.readline() if line: queue.put(line) else: break self._t = Thread(target = _populateQueue, args = (self._s, self._q)) self._t.daemon = True self._t.start() #start collecting lines from the stream def readline(self, timeout = None): try: return self._q.get(block = timeout is not None, timeout = timeout) except Empty: return None
def run_with_timeout_and_stack(request, timeout): ''' interrupts evaluation after a given time period. provides a suitable stack environment. ''' # only use set_thread_stack_size if max recursion depth was changed via the environment variable # MATHICS_MAX_RECURSION_DEPTH. if it is set, we always use a thread, even if timeout is None, in # order to be able to set the thread stack size. if MAX_RECURSION_DEPTH > settings.DEFAULT_MAX_RECURSION_DEPTH: set_thread_stack_size(python_stack_size(MAX_RECURSION_DEPTH)) elif timeout is None: return request() queue = Queue(maxsize=1) # stores the result or exception thread = Thread(target=_thread_target, args=(request, queue)) thread.start() thread.join(timeout) if thread.is_alive(): raise TimeoutInterrupt() success, result = queue.get() if success: return result else: six.reraise(*result)
def iter_entries(handle): cd = pycdlib.PyCdlib() if hasattr(handle, 'seek') and handle.seekable(): handle.seek(0) cd.open_fp(handle) else: cd.open(handle) rock_ridge = cd.rock_ridge is not None joliet = cd.joliet_vd is not None joliet_only = joliet and not rock_ridge directories = Queue() directories.put(cd.get_entry('/', joliet_only)) while not directories.empty(): directory = directories.get() for child in directory.children: if not child.is_dot() and not child.is_dotdot(): if child.is_dir(): directories.put(child) yield child
class NonBlockingStreamReader: def __init__(self, stream): ''' stream: the stream to read from. Usually a process' stdout or stderr. ''' self._s = stream self._q = Queue() def _populateQueue(stream, queue): ''' Collect lines from 'stream' and put them in 'quque'. ''' while True: line = stream.readline() if line: queue.put(line) else: break self._t = Thread(target=_populateQueue, args=(self._s, self._q)) self._t.daemon = True self._t.start() #start collecting lines from the stream def readline(self, timeout=None): try: return self._q.get(block=timeout is not None, timeout=timeout) except Empty: return None
def main_random_cast(voter_url_file, plaintexts_file, nr_threads=2): if exists(plaintexts_file): m = "%s: file exists, will not overwrite" % (plaintexts_file, ) raise ValueError(m) with open(voter_url_file) as f: voter_urls = f.read().splitlines() total = len(voter_urls) inqueue = Queue(maxsize=total) outqueue = Queue(maxsize=total) for i, voter_url in enumerate(voter_urls): inqueue.put((i, total, voter_url)) #main_random_cast_thread(queue) threads = [ Thread(target=main_random_cast_thread, args=(inqueue, outqueue)) for _ in range(nr_threads) ] for t in threads: t.daemon = True t.start() plaintexts = [outqueue.get() for _ in range(total)] with open(plaintexts, 'w') as f: f.write(repr(plaintexts)) for t in threads: t.join()
def parallel_execute_stream(objects, func, get_deps): if get_deps is None: get_deps = _no_deps results = Queue() state = State(objects) while not state.is_done(): for event in feed_queue(objects, func, get_deps, results, state): yield event try: event = results.get(timeout=0.1) except Empty: continue # See https://github.com/docker/compose/issues/189 except thread.error: raise ShutdownException() obj, _, exception = event if exception is None: log.debug('Finished processing: {}'.format(obj)) state.finished.add(obj) else: log.debug('Failed: {}'.format(obj)) state.failed.add(obj) yield event
class TagGroup( object ): ''' Process groups of tag reads and return the best time estimated using quadratic regression. Stray reads are also detected if there is no quiet period for the tag. The first read time of each stray read is returned. ''' def __init__( self ): self.q = Queue() self.tagInfo = {} def add( self, antenna, tag, t, db ): self.q.put((antenna, tag, t, db)) def flush( self ): # Process all waiting reads. while 1: try: antenna, tag, t, db = self.q.get(False) except Empty: break try: self.tagInfo[tag].add( antenna, t, db ) except KeyError: self.tagInfo[tag] = TagGroupEntry( antenna, t, db ) self.q.task_done() def getReadsStrays( self, tNow=None, method=QuadraticRegressionMethod, antennaChoice=MostReadsChoice, removeOutliers=True ): ''' Returns two lists: reads = [(tag1, t1, sampleSize1, antennaID1), (tag2, t2, sampleSize2, , antennaID2), ...] strays = [(tagA, tFirstReadA), (tagB, tFirstReadB), ...] Each stray will be reported as a read the first time it is detected. ''' self.flush() trNow = datetimeToTr( tNow or datetime.now() ) reads, strays = [], [] toDelete = [] for tag, tge in six.iteritems(self.tagInfo): if trNow - tge.lastReadMax >= tQuiet: # Tag has left read range. if not tge.isStray: t, sampleSize, antennaID = tge.getBestEstimate(method, antennaChoice, removeOutliers) reads.append( (tag, t, sampleSize, antennaID) ) toDelete.append( tag ) elif tge.lastReadMax - tge.firstReadMin >= tStray: # This is a stray. t = trToDatetime( tge.firstReadMin ) if not tge.isStray: tge.setStray() reads.append( (tag, t, 1, 0) ) # Report stray first read time. strays.append( (tag, t) ) for tag in toDelete: del self.tagInfo[tag] reads.sort( key=operator.itemgetter(1,0)) strays.sort( key=operator.itemgetter(1,0) ) return reads, strays
class BaseQueue(object): def __init__(self): self.queue = Queue() def commit(self, *args, **kwargs): raise NotImplemented() def get(self, *args, **kwargs): return self.queue.get(*args, **kwargs)
def test_response_on_queue(self): g = MockGrader() pl = self._make_payload({ 'student_response': 'blah', 'grader_payload': json.dumps({ 'grader': 'correct' }) }) q = Queue() reply = g.process_item(pl, queue=q) popped = q.get() self.assertEqual(reply, popped) del pl['xqueue_body'] try: g.process_item(pl, queue=q) except Exception as e: popped = q.get() self.assertEqual(e, popped)
def parallel_execute(objects, obj_callable, msg_index, msg): """ For a given list of objects, call the callable passing in the first object we give it. """ stream = get_output_stream(sys.stdout) lines = [] errors = {} for obj in objects: write_out_msg(stream, lines, msg_index(obj), msg) q = Queue() def inner_execute_function(an_callable, parameter, msg_index): try: result = an_callable(parameter) except APIError as e: errors[msg_index] = e.explanation result = "error" except Exception as e: errors[msg_index] = e result = 'unexpected_exception' q.put((msg_index, result)) for an_object in objects: t = Thread( target=inner_execute_function, args=(obj_callable, an_object, msg_index(an_object)), ) t.daemon = True t.start() done = 0 total_to_execute = len(objects) while done < total_to_execute: try: msg_index, result = q.get(timeout=1) if result == 'unexpected_exception': raise errors[msg_index] if result == 'error': write_out_msg(stream, lines, msg_index, msg, status='error') else: write_out_msg(stream, lines, msg_index, msg) done += 1 except Empty: pass if errors: stream.write("\n") for error in errors: stream.write("ERROR: for {} {} \n".format(error, errors[error]))
def test_command_subprocess(self): """test_process | command_subprocess """ queue = Queue() self.assertRaises( SystemExit, shprocess.command_subprocess, queue, [sys.executable, "-c", "from __future__ import print_function;print('foo')"], ) line = queue.get(block=True, timeout=.1) self.assertEqual(to_unicode("foo"), to_unicode(line).rstrip())
class USBFtdiInterface(USBInterface): name = 'FtdiInterface' def __init__(self, app, phy, interface_number): super(USBFtdiInterface, self).__init__( app=app, phy=phy, interface_number=interface_number, interface_alternate=0, interface_class=USBClass.VendorSpecific, interface_subclass=0xff, interface_protocol=0xff, interface_string_index=0, endpoints=[ USBEndpoint( app=app, phy=phy, number=1, direction=USBEndpoint.direction_out, transfer_type=USBEndpoint.transfer_type_bulk, sync_type=USBEndpoint.sync_type_none, usage_type=USBEndpoint.usage_type_data, max_packet_size=0x40, interval=0, handler=self.handle_data_available ), USBEndpoint( app=app, phy=phy, number=3, direction=USBEndpoint.direction_in, transfer_type=USBEndpoint.transfer_type_bulk, sync_type=USBEndpoint.sync_type_none, usage_type=USBEndpoint.usage_type_data, max_packet_size=0x40, interval=0, handler=self.handle_ep3_buffer_available # at this point, we don't send data to the host ) ], ) self.txq = Queue() def handle_data_available(self, data): self.debug('received string (%d): %s' % (len(data), data)) reply = b'\x01\x00' + data self.txq.put(reply) def handle_ep3_buffer_available(self): if not self.txq.empty(): self.send_on_endpoint(3, self.txq.get())
def hangwatch(timeout, func, *args, **kwargs): def target(queue): try: func(*args, **kwargs) except Exception as e: queue.put(sys.exc_info()) queue.put(e) sys.exit() q = Queue() thread = threading.Thread(target=target, args = (q,)) thread.start() thread.join(timeout) if thread.is_alive(): raise RuntimeError('Operation did not terminate within {} seconds' .format(timeout)) if not q.empty(): info = q.get(block=False) e = q.get(block=False) eprint(''.join(traceback.format_exception(*info))) raise e
def test_changesize(self): "Change sizes and make sure pool doesn't work with no workers." pool = workerpool.WorkerPool(5) for i in range(5): pool.grow() self.assertEquals(pool.size(), 10) for i in range(10): pool.shrink() pool.wait() self.assertEquals(pool.size(), 0) # Make sure nothing is reading jobs anymore q = Queue() for i in range(5): pool.put(workerpool.SimpleJob(q, sum, [range(5)])) try: q.get(block=False) except Empty: pass # Success else: assert False, "Something returned a result, even though we are" "expecting no workers." pool.shutdown()
def parallel_execute_iter(objects, func, get_deps, limit): """ Runs func on objects in parallel while ensuring that func is ran on object only after it is ran on all its dependencies. Returns an iterator of tuples which look like: # if func returned normally when run on object (object, result, None) # if func raised an exception when run on object (object, None, exception) # if func raised an exception when run on one of object's dependencies (object, None, UpstreamError()) """ if get_deps is None: get_deps = _no_deps if limit is None: limiter = NoLimit() else: limiter = Semaphore(limit) results = Queue() state = State(objects) while True: feed_queue(objects, func, get_deps, results, state, limiter) try: event = results.get(timeout=0.1) except Empty: continue # See https://github.com/docker/compose/issues/189 except thread.error: raise ShutdownException() if event is STOP: break obj, _, exception = event if exception is None: log.debug('Finished processing: {}'.format(obj)) state.finished.add(obj) else: log.debug('Failed: {}'.format(obj)) state.failed.add(obj) yield event
def _handle_messages_threaded(self): # Handles messages in a threaded fashion. queue = Queue() def producer_loop(): # Read messages from file, and queue them for execution. for msg in self._read_next_message(): queue.put(msg) # Check if an error occurred. if self._done: break # Wait until the queue empties out to signal completion from the # producer's side. if not self._done: queue.join() self._done = True producer = Thread(name="Producer", target=producer_loop) # @note Previously, when trying to do `queue.clear()` in the consumer, # and `queue.join()` in the producer, there would be intermittent # deadlocks. By demoting the producer to a daemon, I (eric.c) have not # yet encountered a deadlock. producer.daemon = True producer.start() # Consume. # TODO(eric.cousineau): Trying to quit via Ctrl+C is awkward (but kinda # works). Is there a way to have `plt.pause` handle Ctrl+C differently? try: pause = self.scope_globals['pause'] while not self._done: # Process messages. while not queue.empty(): msg = queue.get() queue.task_done() self._execute_message(msg) # Spin busy for a bit, let matplotlib (or whatever) flush its # event queue. pause(0.01) except KeyboardInterrupt: # User pressed Ctrl+C. self._done = True print("Quitting") except Exception as e: # We encountered an error, and must stop. self._done = True self._had_error = True traceback.print_exc(file=sys.stderr) sys.stderr.write(" Stopping (--stop_on_error)\n")
class DnsCachingResolver(Thread): def __init__(self, cache_time=600.0, cache_fail_time=30.0): super(DnsCachingResolver, self).__init__() self._cache = {} self._cache_time = cache_time self._cache_fail_time = cache_fail_time self._resolve_queue = Queue() self.daemon = True self.start() def run(self): while True: hostname, attempt = self._resolve_queue.get() ips = self._do_resolve(hostname) if ips: self._cache[hostname] = (time.time(), ips) else: if attempt < 10: self.resolve_async(hostname, attempt + 1) time.sleep(1) def resolve(self, hostname): current_time = time.time() cached_time, ips = self._cache.get(hostname, (0, [])) time_passed = current_time - cached_time if time_passed > self._cache_time or (not ips and time_passed > self._cache_fail_time): new_ips = self._do_resolve(hostname) if new_ips: self._cache[hostname] = (current_time, new_ips) ips = new_ips return ips def resolve_async(self, hostname, attempt=0): self._resolve_queue.put((hostname, attempt)) @staticmethod def _do_resolve(hostname): try: ret = set() for r in socket.getaddrinfo(hostname, 0, 0, 0, socket.IPPROTO_TCP): if r[0] == socket.AF_INET6: ret.add('[{0}]'.format(r[4][0])) else: ret.add(r[4][0]) return list(ret) except socket.gaierror: logger.warning('failed to resolve host %s', hostname) return []
def map(self, fn, *seq): "Perform a map operation distributed among the workers. Will " "block until done." results = Queue() args = zip(*seq) for seq in args: j = SimpleJob(results, fn, seq) self.put(j) # Aggregate results r = [] for i in range(len(list(args))): r.append(results.get()) return r
def parallel_execute(objects, func, index_func, msg): """For a given list of objects, call the callable passing in the first object we give it. """ objects = list(objects) stream = get_output_stream(sys.stderr) writer = ParallelStreamWriter(stream, msg) for obj in objects: writer.initialize(index_func(obj)) q = Queue() # TODO: limit the number of threads #1828 for obj in objects: t = Thread( target=perform_operation, args=(func, obj, q.put, index_func(obj))) t.daemon = True t.start() done = 0 errors = {} while done < len(objects): try: msg_index, result = q.get(timeout=1) except Empty: continue if isinstance(result, APIError): errors[msg_index] = "error", result.explanation writer.write(msg_index, 'error') elif isinstance(result, Exception): errors[msg_index] = "unexpected_exception", result else: writer.write(msg_index, 'done') done += 1 if not errors: return stream.write("\n") for msg_index, (result, error) in errors.items(): stream.write("ERROR: for {} {} \n".format(msg_index, error)) if result == 'unexpected_exception': raise error
class AudioStreaming(object): def __init__(self, app, phy, tx_ep, rx_ep): self.app = app self.phy = phy self.tx_ep = tx_ep self.rx_ep = rx_ep self.txq = Queue() def buffer_available(self): if self.txq.empty(): self.phy.send_on_endpoint(self.tx_ep, b'\x00\x00\x00\x00\x00\x00\x00\x00') else: self.phy.send_on_endpoint(self.tx_ep, self.txq.get()) def data_available(self, data): self.app.logger.info('[AudioStreaming] Got %#x bytes on streaming endpoint' % (len(data)))
def _build_droot_impact(destroy_handler): droot = {} # destroyed view + nonview variables -> foundation impact = {} # destroyed nonview variable -> it + all views of it root_destroyer = {} # root -> destroyer apply for app in destroy_handler.destroyers: for output_idx, input_idx_list in app.op.destroy_map.items(): if len(input_idx_list) != 1: raise NotImplementedError() input_idx = input_idx_list[0] input = app.inputs[input_idx] # Find non-view variable which is ultimatly viewed by input. view_i = destroy_handler.view_i _r = input while _r is not None: r = _r _r = view_i.get(r) input_root = r if input_root in droot: raise InconsistencyError( "Multiple destroyers of %s" % input_root) droot[input_root] = input_root root_destroyer[input_root] = app # The code here add all the variables that are views of r into # an OrderedSet input_impact input_impact = OrderedSet() queue = Queue() queue.put(input_root) while not queue.empty(): v = queue.get() for n in destroy_handler.view_o.get(v, []): input_impact.add(n) queue.put(n) for v in input_impact: assert v not in droot droot[v] = input_root impact[input_root] = input_impact impact[input_root].add(input_root) return droot, impact, root_destroyer
class Events(threading.Thread): def __init__(self, callback): super(Events, self).__init__() self.queue = Queue() # http://stackoverflow.com/a/20598791 self.daemon = False self.callback = callback self.name = 'EVENT-QUEUE' self.stop = threading.Event() def put(self, event_type): self.queue.put(event_type) def run(self): """ Actually runs the thread to process events """ try: while not self.stop.is_set(): try: # get event type event_type = self.queue.get(True, 1) # perform callback if we got a event type self.callback(event_type) # event completed self.queue.task_done() except Empty: event_type = None # exiting thread self.stop.clear() except Exception as error: log.error(u'Exception generated in thread %s: %s', self.name, ex(error)) log.debug(repr(traceback.format_exc())) # System Events class SystemEvent(Event): RESTART = 'RESTART' SHUTDOWN = 'SHUTDOWN'
def run_with_timeout(request, timeout): ''' interrupts evaluation after a given time period. ''' if timeout is None: return request() queue = Queue(maxsize=1) # stores the result or exception thread = Thread(target=_thread_target, args=(request, queue)) thread.start() thread.join(timeout) if thread.is_alive(): raise TimeoutInterrupt() success, result = queue.get() if success: return result else: six.reraise(*result)