def crawl(self): """Run the crawler until all finished.""" with (yield From(self.termination)): while self.todo or self.busy: if self.todo: url, max_redirect = self.todo.popitem() fetcher = Fetcher( self.log, url, crawler=self, max_redirect=max_redirect, max_tries=self.max_tries, ) self.busy[url] = fetcher fetcher.task = asyncio.Task(self.fetch(fetcher)) else: yield From(self.termination.wait()) self.t1 = time.time()
def test_readexactly_eof(self): # Read exact number of bytes (eof). stream = asyncio.StreamReader(loop=self.loop) n = 2 * len(self.DATA) read_task = asyncio.Task(stream.readexactly(n), loop=self.loop) def cb(): stream.feed_data(self.DATA) stream.feed_eof() self.loop.call_soon(cb) with self.assertRaises(asyncio.IncompleteReadError) as cm: self.loop.run_until_complete(read_task) self.assertEqual(cm.exception.partial, self.DATA) self.assertEqual(cm.exception.expected, n) self.assertEqual(str(cm.exception), '18 bytes read on a total of 36 expected bytes') self.assertEqual(b'', stream._buffer)
def inferiortty(self, set_inferior_tty_cb): """Spawn the inferior terminal.""" @asyncio.coroutine def _set_inferior_tty(): if self.proc_inftty: if self.proc_inftty.returncode is None: self.proc_inftty.terminate() self.proc_inftty = None try: self.proc_inftty = proc = yield asyncio.From( asyncio.create_subprocess_exec(*args)) info('inferiortty: {}'.format(args)) except OSError as e: self.console_print('Cannot spawn terminal: {}\n'.format(e)) else: start = time.time() while time.time() - start < 2: try: with open(result_file.name) as f: lines = f.readlines() # Commands found in the result file. if len(lines) == 2: set_inferior_tty_cb(lines[0]) set_inferior_tty_cb(lines[1]) break except IOError as e: self.console_print( 'Cannot set the inferior tty: {}\n'.format(e)) proc.terminate() break yield asyncio.From(asyncio.sleep(.100, loop=self.vim.loop)) else: self.console_print('Failed to start inferior_tty.py.\n') proc.terminate() args = self.vim.options.terminal.split(',') result_file = misc.tmpfile('dbg') args.append('%s -m clewn.inferiortty %s' % (sys.executable, result_file.name)) asyncio.Task(_set_inferior_tty(), loop=self.vim.loop)
def run(): connection = yield From( trollius_redis.Pool.create(host=u'localhost', port=6379, poolsize=50)) try: # === Benchmark 1 == print(u'1. How much time does it take to set 10,000 values ' u'in Redis? (without pipelining)') print(u'Starting...') start = time.time() # Do 10,000 set requests for i in range(10 * 1000): # By using yield from here, we wait for the answer. yield From(connection.set(u'key', u'value')) print(u'Done. Duration=', time.time() - start) print() # === Benchmark 2 (should be at least 3x as fast) == print(u'2. How much time does it take if we use asyncio.gather, ' u'and pipeline requests?') print(u'Starting...') start = time.time() # Do 10,000 set requests futures = [ asyncio.Task(connection.set(u'key', u'value')) for x in range(10 * 1000) ] yield From(asyncio.gather(*futures)) print(u'Done. Duration=', time.time() - start) finally: connection.close()
def test_wait_for(self): cond = asyncio.Condition(loop=self.loop) presult = False def predicate(): return presult result = [] @asyncio.coroutine def c1(result): yield From(cond.acquire()) if (yield From(cond.wait_for(predicate))): result.append(1) cond.release() raise Return(True) t = asyncio.Task(c1(result), loop=self.loop) test_utils.run_briefly(self.loop) self.assertEqual([], result) self.loop.run_until_complete(cond.acquire()) cond.notify() cond.release() test_utils.run_briefly(self.loop) self.assertEqual([], result) presult = True self.loop.run_until_complete(cond.acquire()) cond.notify() cond.release() test_utils.run_briefly(self.loop) self.assertEqual([1], result) self.assertTrue(t.done()) self.assertTrue(t.result())
def main(testrun=False): """Main. Return the vim instance to avoid its 'f_script' member to be garbage collected and the corresponding 'TmpFile' to be unlinked before Vim has a chance to start and source the file (only needed for the pdb test suite). """ vim = Vim(testrun, sys.argv[1:]) options = vim.options is_pdb = (vim.module == 'pdb') try: # Vim is running the command ':Pyclewn pdb script'. if is_pdb and not testrun and options.args: embed_pdb(vim) return vim vim.set_event_loop() vim.setup() # When is_pdb is True, this is either: # * Vim is being spawned by the test suite. # * or the user is running the Vim command ':Pyclewn pdb' if is_pdb: if not testrun and not options.cargs: critical('This command is meant to be run from Vim.') return vim_tasks = [] # Use pyclewn terminal as the inferior standard input/output when vim is # run as 'gvim'. if (options.editor and vim.module == 'gdb' and not testrun and os.isatty(sys.stdin.fileno()) and not options.daemon): out = exec_vimcmd(['echo has("gui_running")'], options.editor, vim.stderr_hdlr).strip() if out == '1': tasks, ptyname = tty.inferior_tty(vim.stderr_hdlr, vim.loop) options.tty = ptyname vim_tasks.extend(tasks) vim_tasks.append(asyncio.Task(vim.run(), loop=vim.loop)) misc.cancel_after_first_completed(vim_tasks, lambda: vim.signal(None), loop=vim.loop) except Exception as e: except_str = 'Exception in pyclewn: "%s"\n' \ '%s\n' \ 'pyclewn aborting...\n' \ % (e, traceback.format_tb(sys.exc_info()[2])[-1]) critical('\n' + except_str) if vim.netbeans: vim.netbeans.show_balloon(except_str) if not testrun: sys.exit(1) finally: debug('Vim instance: ' + str(vim)) vim.shutdown() return vim
def getaddrinfo_task(*args, **kwds): return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
""" import trollius from trollius import From @trollius.coroutine def factorial(name, number): """ https://en.wikipedia.org/wiki/Factorial """ f = 1 for i in range(2, number + 1): print("Task %s: Compute factorial(%s)..." % (name, i)) yield trollius.sleep(1) f *= i print("Task %s: factorial(%s) = %s" % (name, number, f)) # Instantiating tasks doesn't cause the coroutine to be run. It merely # schedules the tasks. tasks = [ trollius.Task(factorial("A", 2)), trollius.Task(factorial("B", 3)), trollius.Task(factorial("C", 4)), ] # Get the event loop and cause it to run until all the tasks are done. loop = trollius.get_event_loop() loop.run_until_complete(trollius.wait(tasks)) loop.close()
def test_put_with_waiting_getters(self): q = asyncio.Queue(loop=self.loop) t = asyncio.Task(q.get(), loop=self.loop) test_utils.run_briefly(self.loop) self.loop.run_until_complete(q.put('a')) self.assertEqual(self.loop.run_until_complete(t), 'a')
def test_acquire(self): sem = asyncio.Semaphore(3, loop=self.loop) result = [] self.assertTrue(self.loop.run_until_complete(sem.acquire())) self.assertTrue(self.loop.run_until_complete(sem.acquire())) self.assertFalse(sem.locked()) @asyncio.coroutine def c1(result): yield From(sem.acquire()) result.append(1) raise Return(True) @asyncio.coroutine def c2(result): yield From(sem.acquire()) result.append(2) raise Return(True) @asyncio.coroutine def c3(result): yield From(sem.acquire()) result.append(3) raise Return(True) @asyncio.coroutine def c4(result): yield From(sem.acquire()) result.append(4) raise Return(True) t1 = asyncio.Task(c1(result), loop=self.loop) t2 = asyncio.Task(c2(result), loop=self.loop) t3 = asyncio.Task(c3(result), loop=self.loop) # each coroutine requires 2 runs of the event loop test_utils.run_briefly(self.loop, 2) self.assertEqual([1], result) self.assertTrue(sem.locked()) self.assertEqual(2, len(sem._waiters)) self.assertEqual(0, sem._value) t4 = asyncio.Task(c4(result), loop=self.loop) sem.release() sem.release() self.assertEqual(2, sem._value) test_utils.run_briefly(self.loop) self.assertEqual(0, sem._value) self.assertEqual([1, 2, 3], result) self.assertTrue(sem.locked()) self.assertEqual(1, len(sem._waiters)) self.assertEqual(0, sem._value) self.assertTrue(t1.done()) self.assertTrue(t1.result()) self.assertTrue(t2.done()) self.assertTrue(t2.result()) self.assertTrue(t3.done()) self.assertTrue(t3.result()) self.assertFalse(t4.done()) # cleanup locked semaphore sem.release() self.loop.run_until_complete(t4)
def test_wait(self): cond = asyncio.Condition(loop=self.loop) result = [] @asyncio.coroutine def c1(result): yield From(cond.acquire()) if (yield From(cond.wait())): result.append(1) raise Return(True) @asyncio.coroutine def c2(result): yield From(cond.acquire()) if (yield From(cond.wait())): result.append(2) raise Return(True) @asyncio.coroutine def c3(result): yield From(cond.acquire()) if (yield From(cond.wait())): result.append(3) raise Return(True) t1 = asyncio.Task(c1(result), loop=self.loop) t2 = asyncio.Task(c2(result), loop=self.loop) t3 = asyncio.Task(c3(result), loop=self.loop) test_utils.run_briefly(self.loop, 2) self.assertEqual([], result) self.assertFalse(cond.locked()) self.assertTrue(self.loop.run_until_complete(cond.acquire())) cond.notify() test_utils.run_briefly(self.loop, 2) self.assertEqual([], result) self.assertTrue(cond.locked()) cond.release() test_utils.run_briefly(self.loop, 2) self.assertEqual([1], result) self.assertTrue(cond.locked()) cond.notify(2) test_utils.run_briefly(self.loop, 2) self.assertEqual([1], result) self.assertTrue(cond.locked()) cond.release() test_utils.run_briefly(self.loop, 2) self.assertEqual([1, 2], result) self.assertTrue(cond.locked()) cond.release() test_utils.run_briefly(self.loop, 2) self.assertEqual([1, 2, 3], result) self.assertTrue(cond.locked()) self.assertTrue(t1.done()) self.assertTrue(t1.result()) self.assertTrue(t2.done()) self.assertTrue(t2.result()) self.assertTrue(t3.done()) self.assertTrue(t3.result())
def test_start_server(self): class MyServer: def __init__(self, loop): self.server = None self.loop = loop @asyncio.coroutine def handle_client(self, client_reader, client_writer): data = yield From(client_reader.readline()) client_writer.write(data) yield From(client_writer.drain()) client_writer.close() def start(self): sock = socket.socket() sock.bind(('127.0.0.1', 0)) self.server = self.loop.run_until_complete( asyncio.start_server(self.handle_client, sock=sock, loop=self.loop)) return sock.getsockname() def handle_client_callback(self, client_reader, client_writer): self.loop.create_task( self.handle_client(client_reader, client_writer)) def start_callback(self): sock = socket.socket() sock.bind(('127.0.0.1', 0)) addr = sock.getsockname() sock.close() self.server = self.loop.run_until_complete( asyncio.start_server(self.handle_client_callback, host=addr[0], port=addr[1], loop=self.loop)) return addr def stop(self): if self.server is not None: self.server.close() self.loop.run_until_complete(self.server.wait_closed()) self.server = None @asyncio.coroutine def client(addr): reader, writer = yield From( asyncio.open_connection(*addr, loop=self.loop)) # send a line writer.write(b"hello world!\n") # read it back msgback = yield From(reader.readline()) writer.close() raise Return(msgback) # test the server variant with a coroutine as client handler server = MyServer(self.loop) addr = server.start() msg = self.loop.run_until_complete( asyncio.Task(client(addr), loop=self.loop)) server.stop() self.assertEqual(msg, b"hello world!\n") # test the server variant with a callback as client handler server = MyServer(self.loop) addr = server.start_callback() msg = self.loop.run_until_complete( asyncio.Task(client(addr), loop=self.loop)) server.stop() self.assertEqual(msg, b"hello world!\n")
def start_other(): asyncio.Task(second_coroutine())
def start_client(loop, host, port): t = asyncio.Task(loop.create_connection(EchoClient, host, port)) loop.run_until_complete(t)
def CriticalTask(coro): task = asyncio.Task(coro) task.add_done_callback(_critical_task_done) return task
def send_message(self, msg): self._producers.append( asyncio.Task(self._queue.put(msg), loop=self._loop), )
if sorted(res) != res: print('FAIL', res) print('------------') else: print('.', end='') sys.stdout.flush() loop = asyncio.get_event_loop() print('Pass 1') # All permutations of discrete task running times must be returned # by as_completed in the correct order. task_times = [0, 0.1, 0.2, 0.3, 0.4] # 120 permutations for times in itertools.permutations(task_times): tasks = [asyncio.Task(sleeper(t)) for t in times] loop.run_until_complete(asyncio.Task(watcher(tasks))) print() print('Pass 2') # Longer task times, with randomized duplicates. 100 tasks each time. longer_task_times = [x / 10 for x in range(30)] for i in range(20): task_times = longer_task_times * 10 random.shuffle(task_times) #print('Times', task_times[:500]) tasks = [asyncio.Task(sleeper(t)) for t in task_times[:100]] loop.run_until_complete(asyncio.Task(watcher(tasks))) print() print('Pass 3')
def test_start_unix_server(self): class MyServer: def __init__(self, loop, path): self.server = None self.loop = loop self.path = path @asyncio.coroutine def handle_client(self, client_reader, client_writer): data = yield From(client_reader.readline()) client_writer.write(data) yield From(client_writer.drain()) client_writer.close() def start(self): self.server = self.loop.run_until_complete( asyncio.start_unix_server(self.handle_client, path=self.path, loop=self.loop)) def handle_client_callback(self, client_reader, client_writer): self.loop.create_task( self.handle_client(client_reader, client_writer)) def start_callback(self): start = asyncio.start_unix_server(self.handle_client_callback, path=self.path, loop=self.loop) self.server = self.loop.run_until_complete(start) def stop(self): if self.server is not None: self.server.close() self.loop.run_until_complete(self.server.wait_closed()) self.server = None @asyncio.coroutine def client(path): reader, writer = yield From( asyncio.open_unix_connection(path, loop=self.loop)) # send a line writer.write(b"hello world!\n") # read it back msgback = yield From(reader.readline()) writer.close() raise Return(msgback) # test the server variant with a coroutine as client handler with test_utils.unix_socket_path() as path: server = MyServer(self.loop, path) server.start() msg = self.loop.run_until_complete( asyncio.Task(client(path), loop=self.loop)) server.stop() self.assertEqual(msg, b"hello world!\n") # test the server variant with a callback as client handler with test_utils.unix_socket_path() as path: server = MyServer(self.loop, path) server.start_callback() msg = self.loop.run_until_complete( asyncio.Task(client(path), loop=self.loop)) server.stop() self.assertEqual(msg, b"hello world!\n")
[[raw_name_matrix[index][0], raw_name_matrix[index][1]], [in_cof_matrix[0], in_cof_matrix[1]]]) from multiprocessing.pool import ThreadPool tasks = [] final_f = [] final_combination = [] for index, value in enumerate(score_matrix): new_data = result_dta_numenta.copy() new_data.anomaly_score = value #engine.anomaly_detection(new_data, raw_dta, name_coff_metrix[index]); tasks.append( asyncio.Task( engine.anomaly_detection_asysn(new_data, raw_dta, name_coff_metrix[index]))) #final_f.append(return_val) final_combination.append(name_coff_metrix[index]) loop = asyncio.get_event_loop() loop.run_until_complete(asyncio.wait(tasks)) loop.close() end = time.time() print("Total time: {}".format(end - start)) # print final_f # final_index = np.argsort(final_f)[-1] # filed_name = final_combination[final_index] # # print "%%%%%%%%%%%%%%%%%%%%%%%%______BEST CHOICE______%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%"
raise Return(cropped) if __name__ == '__main__': # Build context CASCADES_DIR = '/usr/share/opencv/haarcascades' context = Context(wname='Video', faceCascade=cv2.CascadeClassifier( os.path.join(CASCADES_DIR, 'haarcascade_frontalface_default.xml')), eyeCascade=cv2.CascadeClassifier( os.path.join(CASCADES_DIR, 'haarcascade_eye.xml'))) # Initialize pipeline tasks = [ asyncio.Task(webcam(context)), asyncio.Task(compose(context)), asyncio.Task(display(context)) ] # This will suspend the main routine until q is pressed loop = asyncio.get_event_loop() loop.run_until_complete(keyboard(context)) for t in tasks: t.cancel() # Give tasks a chance to shut down gracefully loop.run_until_complete(asyncio.wait(tasks))
def start_server(loop, addr): import_sdl2() t = asyncio.Task( loop.create_datagram_endpoint(ServerProtocol, local_addr=addr)) transport, server = loop.run_until_complete(t) return transport
def test(): get_task = asyncio.Task(queue_get(), loop=loop) yield From(asyncio.sleep(0.01, loop=loop)) # let the task start q.put_nowait(1) result = (yield From(get_task)) raise Return(result)
def start_client(loop, addr): t = asyncio.Task( loop.create_datagram_endpoint(ClientProtocol, remote_addr=addr)) loop.run_until_complete(t)
message = pygazebo.msg.pose_pb2.Pose() message.position.x = U.translate_speed message.orientation.z = rpy_to_quaternion( Rpy(0, 0, (degree_to_rad(U.rotate_speed)))).z while True: yield From(publisher.publish(message)) yield From(trollius.sleep(0.1)) @trollius.coroutine def server_loop(U): U.server_run() if __name__ == '__main__': util = PyTango.Util(sys.argv) util.add_class(PyDevice, PyTrevorGazebo) U = PyTango.Util.instance() U.server_init() tasks = [ trollius.Task(publish_loop(U)), trollius.Task(server_loop(U)), ] loop = trollius.get_event_loop() loop.run_until_complete(trollius.wait(tasks)) loop.close()
def main(loop): # program which prints evaluation of each expression from stdin code = r'''if 1: import os def writeall(fd, buf): while buf: n = os.write(fd, buf) buf = buf[n:] while True: s = os.read(0, 1024) if not s: break s = s.decode('ascii') s = repr(eval(s)) + '\n' s = s.encode('ascii') writeall(1, s) ''' # commands to send to input commands = iter([b"1+1\n", b"2**16\n", b"1/3\n", b"'x'*50", b"1/0\n"]) # start subprocess and wrap stdin, stdout, stderr p = Popen([sys.executable, '-c', code], stdin=PIPE, stdout=PIPE, stderr=PIPE) stdin = yield From(connect_write_pipe(p.stdin)) stdout, stdout_transport = yield From(connect_read_pipe(p.stdout)) stderr, stderr_transport = yield From(connect_read_pipe(p.stderr)) # interact with subprocess name = {stdout: 'OUT', stderr: 'ERR'} registered = { asyncio.Task(stderr.readline()): stderr, asyncio.Task(stdout.readline()): stdout } while registered: # write command cmd = next(commands, None) if cmd is None: stdin.close() else: print('>>>', cmd.decode('ascii').rstrip()) stdin.write(cmd) # get and print lines from stdout, stderr timeout = None while registered: done, pending = yield From( asyncio.wait(registered, timeout=timeout, return_when=asyncio.FIRST_COMPLETED)) if not done: break for f in done: stream = registered.pop(f) res = f.result() print(name[stream], res.decode('ascii').rstrip()) if res != b'': registered[asyncio.Task(stream.readline())] = stream timeout = 0.0 stdout_transport.close() stderr_transport.close()