async def tasks(tid): jobs = [] jobs.append(asyncio.Task(process_future(test_func, (tid,), {}))) jobs.append(asyncio.Task(process_future(test_func, (tid,), {}))) jobs.append(asyncio.Task(process_future(test_func, (tid,), {}))) print("### jobs", jobs, time.time()) asyncio.gather(*jobs)
async def __crawl__(self): print('Starting webcrawler on url {}'.format(self.basePath)) t1 = time.time() # make tasks that are processing the queue tasks = [asyncio.ensure_future(self.process(), loop=self.loop) for _ in range(self.max_tasks)] # aggregate tasks and squash exceptions asyncio.gather(*tasks, return_exceptions=True) # all queue items should call task_done for each put await self.queue.join() # cancel tasks for t in tasks: t.cancel() self.close() self.loop.stop() # save JSON file for viewing self._save_file() # print('{} pages processed in {} secs. Data saved in sitemap.json'.format(len(self.processed), time.time() - t1)) # leave exit(1)
def step(self): tasks = [firm.call('set_production_target', self.state) for firm in shuffle(self.firms)] jobs = yield from asyncio.gather(*tasks) jobs = list(zip(jobs, self.firms)) yield from self.labor_market(jobs) yield from asyncio.gather(*[firm.call('produce', self.state) for firm in self.raw_material_firms]) # raw material market sold = yield from self.market( self.raw_material_firms, self.consumer_good_firms + self.capital_equipment_firms, 'purchase_materials') print('materials: ${:.2f}'.format(self.mean_sales_price(sold))) yield from asyncio.gather(*[firm.call('produce', self.state) for firm in self.capital_equipment_firms]) # capital equipment market sold = yield from self.market( self.capital_equipment_firms, self.consumer_good_firms + self.raw_material_firms, 'purchase_equipment') print('equipment: ${:.2f}'.format(self.mean_sales_price(sold))) yield from asyncio.gather(*[firm.call('produce', self.state) for firm in self.consumer_good_firms]) # consumer good market sold = yield from self.market( self.consumer_good_firms, self.people, 'purchase_goods') print('consumer goods: ${:.2f}'.format(self.mean_sales_price(sold))) # people consume (reset) their goods yield from asyncio.gather(*[p.call('consume') for p in self.people])
def _call(self): cuuid = str(uuid.uuid4()) logger.info('Start Job: %s UUID(%s)', str(self), cuuid) asyncio.gather( self.target(cuuid, *self.args), loop=self.loop, return_exceptions=True ).add_done_callback(functools.partial(self._result, cuuid))
def __run_loop(self): # Set event loop if self.background: logging.info("Running in background") asyncio.set_event_loop(self.loop) # Create the loop server self.server = self.loop.run_until_complete(websockets.serve(self.handler, self.address, self.port)) try: logging.info("Starting server") self.loop.run_forever() except KeyboardInterrupt: logging.info("Keyboard Interrupt received.") finally: # Tell any clients that we're closing self.server.close() self.loop.run_until_complete(asyncio.sleep(0.1)) # Gather any remaining tasks so we can cancel them asyncio.gather(*asyncio.Task.all_tasks()).cancel() self.loop.stop() logging.info("Cancelling pending tasks...") self.loop.run_forever() logging.info("Stopping Server...") self.loop.close()
def test_starttls_reject_incorrect_arguments(self): ssl_context = unittest.mock.MagicMock() post_handshake_callback = unittest.mock.MagicMock() self.xmlstream.transport = object() with self.assertRaisesRegex(AssertionError, "mismatched starttls argument"): run_coroutine( asyncio.gather( self.xmlstream.starttls(object(), post_handshake_callback), self.xmlstream.run_test( [ XMLStreamMock.STARTTLS( ssl_context, post_handshake_callback) ], ) ) ) with self.assertRaisesRegex(AssertionError, "mismatched starttls argument"): run_coroutine( asyncio.gather( self.xmlstream.starttls(ssl_context, object()), self.xmlstream.run_test( [ XMLStreamMock.STARTTLS( ssl_context, post_handshake_callback) ], ) ) )
async def main(): url, headers = parse_headers_from_stdin() cookies = parse_cookies(headers.get('cookie', '')) futs = [ try_cookie_combinations(url, headers, cookies), try_cookie_removal(url, headers, cookies), ] all_results = [] for result_co in asyncio.as_completed(futs): results = await result_co all_results += results if not results: break min_len = min(len(res) for res in results) if min_len < 2: break asyncio.gather(*futs).cancel() if not all_results: print("No combinations found") return min_len = min(len(res) for res in all_results) valid_cookie_combos = filter(lambda res: len(res) == min_len, all_results) for combo in valid_cookie_combos: print('Found valid combination with cookie names:', ', '.join([ck.partition('=')[0] for ck in combo]))
def run(self): loop = asyncio.get_event_loop() inittasks = [ self.communicator, self.telemetry, self.detection ] print("starting init tasks") loop.run_until_complete(asyncio.gather( *[x.initialise() for x in inittasks] )) tasks = [ self, self.datastore, self.detection, self.messagedispatcher, self.navigator, self.telemetry, self.mesh_controller, self.engine, self.c2_reactor ] print("starting main tasks") loop.run_until_complete(asyncio.gather( *[x.startup() for x in tasks] )) loop.close()
async def command_loop(self): cancelled_task = asyncio.ensure_future(self._command_loop_cancelled.wait()) while not self._command_loop_cancelled.is_set(): get_task = asyncio.ensure_future(self._command_queue.get()) done, pending = await asyncio.wait( [get_task, cancelled_task], return_when=asyncio.FIRST_COMPLETED) if get_task not in done: get_task.cancel() asyncio.gather(get_task, return_exceptions=True) continue cmd, payload, response_container = get_task.result() if self._transport.is_closing(): response_container.set(self.CLOSE_SENTINEL) continue self._transport.write(b'CALL %s %d\n' % (cmd, len(payload))) if payload: self._transport.write(payload) response = await self._protocol.response_queue.get() response_container.set(response) cancelled_task.cancel() asyncio.gather(cancelled_task, return_exceptions=True)
def __start_background_loop(self): # Set event loop asyncio.set_event_loop(self.loop) # Create the loop server self.server = self.loop.create_server(self.factory, self.address, self.port) self.loop.run_until_complete(self.server) try: logging.info("Starting background server") self.loop.run_forever() except KeyboardInterrupt: logging.info("Keyboard Interrupt received.") finally: # Tell any clients that we're closing for client in self.clients: client.sendClose() pass self.server.close() # Gather any remaining tasks so we can cancel them asyncio.gather(*asyncio.Task.all_tasks()).cancel() self.loop.stop() logging.info("Cancelling pending tasks...") self.loop.run_forever() logging.info("Stopping Server...") self.loop.close()
def test_constructor_heterogenous_futures(self): fut1 = asyncio.Future(loop=self.one_loop) fut2 = asyncio.Future(loop=self.other_loop) with self.assertRaises(ValueError): asyncio.gather(fut1, fut2) with self.assertRaises(ValueError): asyncio.gather(fut1, loop=self.other_loop)
def down(): self.__log.info('shutdown closes all channels') yield from asyncio.gather(*( chan.close() for outs in self.outs.values() for _,chan in outs)) yield from asyncio.gather(*( unit.__teardown__() for unit in self.units.values())) exit(0)
def run(self): args = self.make_arg_parser().parse_args() self.coordinator = coordinator.Coordinator(work_dir=args.work_dir, pseudorange_filename=args.dump_pseudorange, partition=args.partition, tag=args.tag) subtasks = self.make_subtasks(args) # Start everything startup = asyncio.gather(*[x.start() for x in subtasks]) self.loop.run_until_complete(startup) startup.result() # provoke exceptions if something failed self.loop.add_signal_handler(signal.SIGINT, self.stop, "Halting on SIGINT") self.loop.add_signal_handler(signal.SIGTERM, self.stop, "Halting on SIGTERM") self.loop.run_forever() # Well, until stop() is called anyway! logging.info("Server shutting down.") # Stop everything for t in reversed(subtasks): t.close() # Wait for completion shutdown = asyncio.gather(*[t.wait_closed() for t in subtasks], return_exceptions=True) self.loop.run_until_complete(shutdown) for e in shutdown.result(): if isinstance(e, Exception) and not isinstance(e, asyncio.CancelledError): logging.error("Exception thrown during shutdown", exc_info=(type(e), e, e.__traceback__)) self.loop.close() logging.info("Server shutdown done.")
def test_multi_exec(redis, loop): yield from redis.delete('foo', 'bar') tr = redis.multi_exec() f1 = tr.incr('foo') f2 = tr.incr('bar') res = yield from tr.execute() assert res == [1, 1] res2 = yield from asyncio.gather(f1, f2, loop=loop) assert res == res2 tr = redis.multi_exec() f1 = tr.incr('foo') f2 = tr.incr('bar') yield from tr.execute() assert (yield from f1) == 2 assert (yield from f2) == 2 tr = redis.multi_exec() f1 = tr.set('foo', 1.0) f2 = tr.incrbyfloat('foo', 1.2) res = yield from tr.execute() assert res == [True, 2.2] res2 = yield from asyncio.gather(f1, f2, loop=loop) assert res == res2 tr = redis.multi_exec() f1 = tr.incrby('foo', 1.0) with pytest.raises_regex(MultiExecError, "increment must be .* int"): yield from tr.execute() with pytest.raises(TypeError): yield from f1
def _json(self, psem): # Prepare everything in parallel def get_dpi(psem): if self._page["dpi"] == "auto": return (yield from self._input_image.dpi(psem)) return self._page["dpi"], self._page["dpi"] (texts, background, foregrounds_json, (width, height), (dpi_x, dpi_y)) = yield from asyncio.gather( self._ocr.texts(psem), self._background.json(psem), asyncio.gather(*[fg.json(psem) for fg in self._foregrounds]), self._input_image.size(psem), get_dpi(psem)) if texts is not None: for text in texts: text["x"] *= (PDF_DPI / dpi_x) text["y"] = ((height - text["y"] - text["height"]) * (PDF_DPI / dpi_y)) text["width"] *= (PDF_DPI / dpi_x) text["height"] *= (PDF_DPI / dpi_y) # Filter empty foregrounds foregrounds_json = [fg for fg in foregrounds_json if fg is not None] return { "width": width * (PDF_DPI / dpi_x), "height": height * (PDF_DPI / dpi_y), "background": background, "foreground": foregrounds_json, "color": self._page["bg_color"], "text": texts }
def run(self): inittasks = [ self.communicator, self.telemetry, self.detection ] print("starting init tasks") yield from asyncio.gather( *[x.initialise() for x in inittasks] ) tasks = [ self, self.datastore, self.detection, self.messagedispatcher, self.navigator, self.telemetry, self.mesh_controller, self.engine, self.navigator.reactor.c2_reactor, self.navigator.reactor.battery_life_checker, self.navigator.reactor.pit_stop ] print("starting main tasks") yield from asyncio.gather( *[x.startup() for x in tasks] )
def test_multi_exec(self): yield from self.redis.delete("foo", "bar") tr = self.redis.multi_exec() f1 = tr.incr("foo") f2 = tr.incr("bar") res = yield from tr.execute() self.assertEqual(res, [1, 1]) res2 = yield from asyncio.gather(f1, f2, loop=self.loop) self.assertEqual(res, res2) tr = self.redis.multi_exec() f1 = tr.incr("foo") f2 = tr.incr("bar") yield from tr.execute() self.assertEqual((yield from f1), 2) self.assertEqual((yield from f2), 2) tr = self.redis.multi_exec() f1 = tr.set("foo", 1.0) f2 = tr.incrbyfloat("foo", 1.2) res = yield from tr.execute() self.assertEqual(res, [True, 2.2]) res2 = yield from asyncio.gather(f1, f2, loop=self.loop) self.assertEqual(res, res2) tr = self.redis.multi_exec() f1 = tr.incrby("foo", 1.0) with self.assertRaisesRegex(MultiExecError, "increment must be .* int"): yield from tr.execute() with self.assertRaises(TypeError): yield from f1
def run_once(): ''' A helper that takes one trip through the event-loop to process any pending Futures. This is a no-op for Twisted, because you don't need to use the event-loop to get callbacks to happen in Twisted. ''' import txaio if txaio.using_twisted: return try: import asyncio if sys.version_info >= (3, 7): # https://github.com/crossbario/txaio/issues/139 from _asyncio_test_utils import run_once as _run_once else: from asyncio.test_utils import run_once as _run_once return _run_once(txaio.config.loop or asyncio.get_event_loop()) except ImportError: import trollius as asyncio # let any trollius import error out; if we're not using # twisted, and have no asyncio *and* no trollius, that's a # problem. # copied from asyncio.testutils because trollius has no # testutils" # just like modern asyncio.testutils.run_once does it... loop = asyncio.get_event_loop() loop.stop() loop.run_forever() asyncio.gather(*asyncio.Task.all_tasks())
def report(self, problems: List[Problem]) -> None: grouped_problems = Problem.group_by_path_and_line(problems) headers = { 'Authorization': 'token {}'.format(self.auth_token), } with aiohttp.ClientSession(headers=headers) as client_session: (line_map, existing_messages) = yield from asyncio.gather( self.create_line_to_position_map(client_session), self.get_existing_messages(client_session)) lint_errors = 0 review_comment_awaitable = [] pr_url = self._get_pr_url() for location, problems_for_line in grouped_problems: message_for_line = [':sparkles:Linty Fresh Says:sparkles::', '', '```'] reported_problems_for_line = set() for problem in problems_for_line: if problem.message not in reported_problems_for_line: message_for_line.append(problem.message) reported_problems_for_line.add(problem.message) message_for_line.append('```') path = location[0] line_number = location[1] position = line_map.get(path, {}).get(line_number, None) if position is not None: message = '\n'.join(message_for_line) if (path, position, message) not in existing_messages: lint_errors += 1 if lint_errors <= MAX_LINT_ERROR_REPORTS: data = json.dumps({ 'body': message, 'commit_id': self.commit, 'path': path, 'position': position, }, sort_keys=True) review_comment_awaitable.append( client_session.post(pr_url, data=data)) if lint_errors > MAX_LINT_ERROR_REPORTS: message = ''':sparkles:Linty Fresh Says:sparkles:: Too many lint errors to report inline! {0} lines have a problem. Only reporting the first {1}.'''.format( lint_errors, MAX_LINT_ERROR_REPORTS) data = json.dumps({ 'body': message }) review_comment_awaitable.append( asyncio.async(client_session.post(pr_url, data=data))) responses = yield from asyncio.gather( *review_comment_awaitable ) # type: List[aiohttp.ClientResponse] for response in responses: response.close()
def stop(self): LOG.info("Stopping WAMP server...") # Canceling pending tasks and stopping the loop asyncio.gather(*asyncio.Task.all_tasks()).cancel() # Stopping the loop self.loop.stop() LOG.info("WAMP server stopped.")
def run_app(app, *, host=None, port=None, path=None, sock=None, shutdown_timeout=60.0, ssl_context=None, print=print, backlog=128, access_log_format=None, access_log=access_logger, handle_signals=True, loop=None): """Run an app locally""" user_supplied_loop = loop is not None if loop is None: loop = asyncio.get_event_loop() app._set_loop(loop) loop.run_until_complete(app.startup()) try: make_handler_kwargs = dict() if access_log_format is not None: make_handler_kwargs['access_log_format'] = access_log_format handler = app.make_handler(loop=loop, access_log=access_log, **make_handler_kwargs) server_creations, uris = _make_server_creators( handler, loop=loop, ssl_context=ssl_context, host=host, port=port, path=path, sock=sock, backlog=backlog) servers = loop.run_until_complete( asyncio.gather(*server_creations, loop=loop) ) if handle_signals: try: loop.add_signal_handler(signal.SIGINT, raise_graceful_exit) loop.add_signal_handler(signal.SIGTERM, raise_graceful_exit) except NotImplementedError: # pragma: no cover # add_signal_handler is not implemented on Windows pass try: if print: print("======== Running on {} ========\n" "(Press CTRL+C to quit)".format(', '.join(uris))) loop.run_forever() except (GracefulExit, KeyboardInterrupt): # pragma: no cover pass finally: server_closures = [] for srv in servers: srv.close() server_closures.append(srv.wait_closed()) loop.run_until_complete( asyncio.gather(*server_closures, loop=loop)) loop.run_until_complete(app.shutdown()) loop.run_until_complete(handler.shutdown(shutdown_timeout)) finally: loop.run_until_complete(app.cleanup()) if not user_supplied_loop: if hasattr(loop, 'shutdown_asyncgens'): loop.run_until_complete(loop.shutdown_asyncgens()) loop.close()
async def _read(self): while True: msgtype, payload = await self.recvmsg() if msgtype & 0x80000000: msgtype &= 0x7FFFFFFF asyncio.gather(*(f(msgtype, payload) for f in self._eventhandlers)) else: msgtype2, fut = await self._queue.get() assert msgtype2 == msgtype fut.set_result(payload)
def test_constructor_homogenous_futures(self): children = [asyncio.Future(loop=self.other_loop) for i in range(3)] fut = asyncio.gather(*children) self.assertIs(fut._loop, self.other_loop) self._run_loop(self.other_loop) self.assertFalse(fut.done()) fut = asyncio.gather(*children, loop=self.other_loop) self.assertIs(fut._loop, self.other_loop) self._run_loop(self.other_loop) self.assertFalse(fut.done())
def test_connection_pool(self): '''Test the connection pool. A very important test!''' client = Echo(self.server_cfg.addresses[0], pool_size=2) self.assertEqual(client._loop, get_event_loop()) # self.assertEqual(client.pool.pool_size, 2) self.assertEqual(client.pool.in_use, 0) self.assertEqual(client.pool.available, 0) self.assertEqual(client.sessions, 0) self.assertEqual(client._requests_processed, 0) # response = yield from client(b'test connection') self.assertEqual(response, b'test connection') self.assertEqual(client.pool.in_use, 0) self.assertEqual(client.pool.available, 1) self.assertEqual(client.sessions, 1) self.assertEqual(client._requests_processed, 1) # response = yield from client(b'test connection 2') self.assertEqual(response, b'test connection 2') self.assertEqual(client.pool.in_use, 0) self.assertEqual(client.pool.available, 1) self.assertEqual(client.sessions, 1) self.assertEqual(client._requests_processed, 2) # result = yield from gather(client(b'ciao'), client(b'pippo'), client(b'foo')) self.assertEqual(len(result), 3) self.assertTrue(b'ciao' in result) self.assertTrue(b'pippo' in result) self.assertTrue(b'foo' in result) self.assertEqual(client.pool.in_use, 0) self.assertEqual(client.pool.available, 2) self.assertEqual(client.sessions, 2) self.assertEqual(client._requests_processed, 5) # # drop a connection yield from run_in_loop(client._loop, self._drop_conection, client) # result = yield from gather(client(b'ciao'), client(b'pippo'), client(b'foo')) self.assertEqual(len(result), 3) self.assertEqual(client.pool.in_use, 0) self.assertEqual(client.pool.available, 2) self.assertEqual(client.sessions, 3) self.assertEqual(client._requests_processed, 8) # yield from run_in_loop(client._loop, client.pool.close) # self.assertEqual(client.pool.in_use, 0) self.assertEqual(client.pool.available, 0) self.assertEqual(client.sessions, 3) self.assertEqual(client._requests_processed, 8)
def _check_empty_sequence(self, seq_or_iter): asyncio.set_event_loop(self.one_loop) self.addCleanup(asyncio.set_event_loop, None) fut = asyncio.gather(*seq_or_iter) self.assertIsInstance(fut, asyncio.Future) self.assertIs(fut._loop, self.one_loop) self._run_loop(self.one_loop) self.assertTrue(fut.done()) self.assertEqual(fut.result(), []) fut = asyncio.gather(*seq_or_iter, loop=self.other_loop) self.assertIs(fut._loop, self.other_loop)
def shutdown_callback(code, _): logging.info('Shutting down. Code: %s.', str(code)) asyncio.gather(*asyncio.Task.all_tasks()).cancel() context.loop.stop() shutdown_webhook() shutdown_workers() if context and context.watcher: context.watcher.close() context.watcher = None logging.shutdown() logging.info('Shut down complete.')
async def stop(self, torrent: MetaInfoFile = None): """ Stops downloading the specified torrent, or all torrents if none specified. :param torrent: torrent to stop downloading. Default = None = ALL torrents """ if torrent: await self.torrents[torrent].cancel() else: tasks = [] for t, ct in self.torrents.items(): tasks.append(ct.cancel) asyncio.gather(*[x() for x in tasks])
def exit_cleanly(self): loop = asyncio.get_event_loop() self.log.info("Shutting down charlesbot") self.set_running(False) for plug in self.plugin_list: self.log.info("Shutting down plugin: %s" % plug.get_plugin_name()) plug.is_running = False pending = asyncio.Task.all_tasks() asyncio.gather(*pending).cancel() loop.stop()
def labor_market(self, jobs): tasks = [p.get('employer') for p in self.people] applicants = {f: [] for (_, __), f in jobs} employers = yield from asyncio.gather(*tasks) job_seekers = [p for p, e in zip(self.people, employers) if e is None] while job_seekers and jobs: job_dist = self.job_distribution(jobs) for p in shuffle(job_seekers): (n_vacancies, wage), firm = random_choice(job_dist) applicants[firm].append(p) # firms select from their applicants _jobs = [] for job in shuffle(jobs): # filter down to valid applicants (n_vacancies, wage), firm = job apps = [a for a in applicants[firm] if a in job_seekers] hired, n_vacancies, wage = yield from firm.call('hire', apps, wage) # remove hired people from the job seeker pool for p in hired: job_seekers.remove(p) if not job_seekers: break # if vacancies remain, post the new jobs with the new wage if n_vacancies: _jobs.append(((n_vacancies, wage), firm)) jobs = _jobs
def run_app(app, *, host='0.0.0.0', port=None, shutdown_timeout=60.0, ssl_context=None, print=print, backlog=128): """Run an app locally""" if port is None: if not ssl_context: port = 8080 else: port = 8443 loop = app.loop handler = app.make_handler() server = loop.create_server(handler, host, port, ssl=ssl_context, backlog=backlog) srv, startup_res = loop.run_until_complete(asyncio.gather(server, app.startup(), loop=loop)) scheme = 'https' if ssl_context else 'http' print("======== Running on {scheme}://{host}:{port}/ ========\n" "(Press CTRL+C to quit)".format( scheme=scheme, host=host, port=port)) try: loop.run_forever() except KeyboardInterrupt: # pragma: no cover pass finally: srv.close() loop.run_until_complete(srv.wait_closed()) loop.run_until_complete(app.shutdown()) loop.run_until_complete(handler.finish_connections(shutdown_timeout)) loop.run_until_complete(app.cleanup()) loop.close()
await db.execute("DELETE FROM users WHERE proofs = $1", ctx.message_id) @bot.event async def on_raw_message_edit(ctx): if ctx.channel_id == 658331377602920471: proofs = await db.fetchrow("SELECT * FROM users WHERE proofs = $1", ctx.message_id) if proofs: await db.execute("DELETE FROM users WHERE proofs = $1", ctx.message_id) message = (await bot.get_channel(ctx.channel_id ).fetch_message(ctx.message_id)) await message.delete() if __name__ == '__main__': loop = asyncio.get_event_loop() botprocess = loop.create_task(bot.start("NO BOT TOKEN PUBLISHING FOR ME")) postgres = loop.create_task(run()) web = loop.create_task(app.run(port=3000, loop=loop)) gathered = asyncio.gather(botprocess, web) try: loop.run_until_complete(gathered) except KeyboardInterrupt: pass finally: loop.run_until_complete(bot.logout()) loop.close()
#! python3 # coding: utf-8 import time import aiohttp import asyncio NUMBERS = range(240) #URL = 'http://httpbin.org/get?a={}' URL = 'http://192.168.1.22:8000/get?a={}' async def fetch_async(a): async with aiohttp.request('GET', URL.format(a)) as r: data = await r.json() return data['args']['a'] start = time.time() event_loop = asyncio.get_event_loop() tasks = [fetch_async(num) for num in NUMBERS] results = event_loop.run_until_complete(asyncio.gather(*tasks)) #for num,result in zip(NUMBERS, results): # print('fetch({}) = {}'.format(num, result)) print('Use aiohttp+asyncio cost: {}'.format(time.time() - start))
def run_sweeps(self): # Propagate the descriptors through the network self.update_descriptors() # Make sure we are starting from scratch... is this necessary? self.reset() # Update the progress bar if need be if self.progressbar is not None: self.progressbar.reset() #Make sure we have axes. if not any( [oc.descriptor.axes for oc in self.output_connectors.values()]): logger.warning( "There do not appear to be any axes defined for this experiment!" ) # Go find any writers self.writers = [n for n in self.nodes if isinstance(n, WriteToHDF5)] self.buffers = [n for n in self.nodes if isinstance(n, DataBuffer)] if self.name: for w in self.writers: w.filename.value = os.path.join( os.path.dirname(w.filename.value), self.name) self.filenames = [w.filename.value for w in self.writers] self.files = [] # Check for redundancy in filenames, and share plot file objects for filename in set(self.filenames): wrs = [w for w in self.writers if w.filename.value == filename] # Let the first writer with this filename create the file... wrs[0].file = wrs[0].new_file() self.files.append(wrs[0].file) # Make the rest of the writers use this same file object for w in wrs[1:]: w.file = wrs[0].file w.filename.value = wrs[0].filename.value # Remove the nodes with 0 dimension self.nodes = [ n for n in self.nodes if not (hasattr(n, 'input_connectors') and n.input_connectors['sink'].descriptor.num_dims() == 0) ] # Go and find any plotters self.standard_plotters = [ n for n in self.nodes if isinstance(n, (Plotter, MeshPlotter, XYPlotter)) ] self.plotters = copy.copy(self.standard_plotters) # We might have some additional plotters that are separate from # The asyncio filter pipeline self.plotters.extend(self.extra_plotters) # These use neither streams nor the filter pipeline self.plotters.extend(self.manual_plotters) # Call any final initialization on the filter pipeline for n in self.nodes + self.extra_plotters: n.experiment = self n.loop = self.loop # n.executor = self.executor if hasattr(n, 'final_init'): n.final_init() # Launch plot servers. if len(self.plotters) > 0: self.init_plot_servers() time.sleep(1) #connect all instruments self.connect_instruments() #initialize instruments self.init_instruments() def catch_ctrl_c(signum, frame): logger.info("Caught SIGINT. Shutting down.") self.shutdown() raise NameError("Shutting down.") sys.exit(0) signal.signal(signal.SIGINT, catch_ctrl_c) # We want to wait for the sweep method above, # not the experiment's run method, so replace this # in the list of tasks. other_nodes = self.nodes[:] other_nodes.extend(self.extra_plotters) other_nodes.remove(self) tasks = [n.run() for n in other_nodes] tasks.append(self.sweep()) try: self.loop.run_until_complete(asyncio.gather(*tasks)) self.loop.run_until_complete(asyncio.sleep(1)) except Exception as e: logger.exception("message") for plot, callback in zip(self.manual_plotters, self.manual_plotter_callbacks): if callback: callback(plot) self.shutdown()
def _unknown_command(self, message): message.message = "." + message.message[len("borgcmd" + str(self.instance_id)) + 1:] return asyncio.gather(*[uk(message, "") for uk in self._unknowns])
if __name__ == "__main__": local_node = OEFLocalProxy.LocalNode() client = WeatherClient("weather_client", local_node) server = WeatherStation("weather_station", local_node) client.connect() server.connect() server.register_service(0, server.weather_service_description) query = Query([ Constraint(TEMPERATURE_ATTR.name, Eq(True)), Constraint(AIR_PRESSURE_ATTR.name, Eq(True)), Constraint(HUMIDITY_ATTR.name, Eq(True)) ], WEATHER_DATA_MODEL) client.on_search_result(0, ["weather_station"]) try: loop = asyncio.get_event_loop() asyncio.ensure_future(local_node.run()) loop.run_until_complete( asyncio.gather(client.async_run(), server.async_run())) local_node.stop() finally: local_node.stop() client.stop() server.stop()
g.setup(proxy=proxy_for_parser.host + ':' + str(proxy_for_parser.port), proxy_type='http') print('2') try: g.go('http://www.google.com/search?q=Spam') except: pass print('2.1') print(g.doc.url) print('3') proxy_for_parser = await proxies.get() #async def test_my_server(): #proxy_server = Proxy('94.177.175.232', '3128') proxies = asyncio.Queue() broker = Broker(proxies) tasks = asyncio.gather( broker.find(types={'HTTP': 'High'}, limit=50, countries='CA'), prx_srv(proxies)) loop = asyncio.get_event_loop() loop.run_until_complete(tasks) #loop.run_until_complete(test_my_server()) #proxy_server = Proxy('94.177.175.232','3128') #print(proxy_server.geo) #print(proxy_server.is_working) #help(asyncio.gather())
# ), # parser.add_argument("--play-from", help="Read the media from a file and sent it."), # parser.add_argument("--record-to", help="Write received media to a file."), # parser.add_argument("--verbose", "-v", action="count") # args = parser.parse_args() # if args.verbose: # logging.basicConfig(level=logging.DEBUG) # create signaling and peer connection session = JanusSession("http://192.168.3.101:8088/janus") loop = asyncio.get_event_loop() try: loop.run_until_complete(run(session=session)) except KeyboardInterrupt: pass finally: if recorder is not None: loop.run_until_complete(recorder.stop()) loop.run_until_complete(session.destroy()) # close peer connections coros = [pc.close() for pc in pcs] loop.run_until_complete(asyncio.gather(*coros)) while True: asyncio.sleep(1) # DTLS HANDSHAKE DIFFERENT BETWEEN TWO PROBABLY ONLY HAPPENS WHEN YOU DO SOMETHING WITH THE TRACK (Or something that happens before you start using the track)
async with aiohttp.ClientSession() as session: async with session.get(url) as response: data = await response.read() with open(dest, 'wb') as f: f.write(data) async def setup_mask(): await download_file(export_file_url, path / export_file_name) BLACK_IMAGE_PATH = path / export_file_name return BLACK_IMAGE_PATH loop = asyncio.get_event_loop() tasks = [asyncio.ensure_future(setup_mask())] BLACK_IMAGE_PATH = loop.run_until_complete(asyncio.gather(*tasks))[0] loop.close() @app.route('/') async def homepage(request): html_file = path / 'view' / 'index.html' if os.path.exists("app/newmask.png"): os.remove("app/newmask.png") else: print("The file does not exist") return HTMLResponse(html_file.open().read()) @app.route('/analyze', methods=['POST', 'GET']) async def analyze(request):
"""Find and show 10 working HTTP(S) proxies.""" import asyncio from proxybroker import Broker async def show(proxies): while True: proxy = await proxies.get() if proxy is None: break print('Found proxy: %s' % proxy) proxies = asyncio.Queue() broker = Broker(proxies) tasks = asyncio.gather(broker.find(types=['HTTP', 'HTTPS'], limit=10), show(proxies)) loop = asyncio.get_event_loop() loop.run_until_complete(tasks)
print("woshi disange") await asyncio.sleep(1) return "three" def call_back(loop, futu): print("调用回调函数") # 在回调函数中,同样会将future对象传进来,因此获取到一步函数的返回值 print(futu.result(), 111111111) loop.stop() if __name__ == '__main__': start = time.time() # 获得时间循环对象 loop = asyncio.get_event_loop() # 创建异步协成列表 geater = [first(), second(), three()] # 可以创建一个future对象,将多个异步任务合并成一个future futu = asyncio.gather(*geater) # 使用future添加异步任务都完成后的回调函数 # functools.partial会先将函数的部分参数传递进去,等所有参数都传递完了,再开始调用 futu.add_done_callback(functools.partial(call_back, loop)) # 使用ayncio.gather将事件添加进时间循环 # loop.run_until_complete(asyncio.gather(*geater)) loop.run_forever() # 循环完成后,关闭事件循环 # loop.close() ends = time.time() print(ends - start)
async def start(self): """Start telegram queue.""" self._consumer_task = asyncio.gather(self._telegram_consumer(), self._outgoing_rate_limiter())
async def socketconnection(self): # noqa: C901 ''' Main socket connection function May be called repeatedly when trying to open a connection ''' # Make sure we retry tasks on reconnection self.retry_task = True # Setup SSL context self.ctx = ssl.SSLContext(protocol=ssl.PROTOCOL_TLSv1_2) # Handle both IPv4 and IPv6 cases try: logging.debug("Try IPv4 (may autodetect IPv6)") self.reader, self.writer = await asyncio.wait_for( asyncio.open_connection( self.addr, self.port, ssl=self.ctx, ), timeout=1.0, ) except (asyncio.TimeoutError, OSError): logging.debug("Try IPv6") try: self.reader, self.writer = await asyncio.wait_for( asyncio.open_connection(self.addr, self.port, ssl=self.ctx, family=socket.AF_INET6), timeout=1.0, ) except (asyncio.TimeoutError, OSError): logger.debug( "Retrying port connection {}:{} auth level {}".format( self.addr, self.port, self.auth)) return False self.overalltasks = [] # Assemble reader and writer tasks, run in the background logging.debug("Backgrounding socket reader and writer functions") coroutines = [self.socketreader(), self.socketwriter()] self.overalltasks.append( asyncio.gather(*coroutines, return_exceptions=True)) # Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode) logging.debug("Starting TwoPartyClient") self.client = capnp.TwoPartyClient() logging.debug("Starting Bootstrap") self.cap = self.client.bootstrap().cast_as(hidio_capnp.HidIoServer) # Start watcher to restart socket connection if it is lost logging.debug("Backgrounding socketwatcher") watcher = [self.socketwatcher()] self.overalltasks.append( asyncio.gather(*watcher, return_exceptions=True)) # Start hid-io-core log watcher logging.debug("Backgrounding corelogwatcher") watcher = [self.corelogwatcher()] self.overalltasks.append( asyncio.gather(*watcher, return_exceptions=True)) # Lookup version information self.version_info = (await self.cap.version().a_wait()).version logger.info(self.version_info) # Lookup uid self.uid_info = (await self.cap.id().a_wait()).id logger.info("uid: %s", self.uid_info) # Lookup name self.daemon_name = (await self.cap.name().a_wait()).name logger.info("name: %s", self.daemon_name) # Lookup log files self.core_log_files = (await self.cap.logFiles().a_wait()).paths logger.info("hid-io-core log files: %s", self.core_log_files) try: self.core_current_log_file = [ f for f in self.core_log_files if 'rCURRENT' in f ][0] self.core_current_log_file_offset = "{}.offset.{}".format( self.core_current_log_file, self.uid_info) self.reset_corelog_followposition() except IndexError: self.core_current_log_file = None logger.warn("Could not find current hid-io-core log file...") # AUTH_NONE doesn't need to go any further cap_auth = None if self.auth: # Lookup key information self.key_info = (await self.cap.key().a_wait()).key logger.info(self.key_info) # Lookup key for auth level key_lookup = { self.AUTH_BASIC: self.key_info.basicKeyPath, self.AUTH_ADMIN: self.key_info.authKeyPath, } key_location = key_lookup[self.auth] # Fail connection if authentication key cannot be read # This usually means that the client doesn't have permission self.key = None try: with open(key_location, 'r') as myfile: self.key = myfile.read() except OSError as err: # XXX (HaaTa): Commented out as this info is sensitive # logger.error("Could not read '%s'. This usually means insufficient permissions.", key_location) logger.error( "Could not read keyfile. This usually means insufficient permissions." ) logger.error(err) await self.disconnect() return False logger.info("Key: %s", self.key) # Connect to specified auth level cap_auth = await self.capability_authenticate() if not cap_auth: await self.disconnect() return False logger.debug("Authenticated with %s", self.auth) # Add nodes subscription background_tasks = [self.nodeswatcher()] self.overalltasks.append( asyncio.gather(*background_tasks, return_exceptions=True)) # Callback await self.on_connect(self.cap, cap_auth) # Spin here until connection is broken while self.retry_task: await asyncio.sleep(1) logger.debug("socketconnection done.")
def finalizer(): event_loop.run_until_complete( asyncio.gather(client.cancel(), server.cancel())) # Yield control so that client/server.run() returns, otherwise asyncio will complain. event_loop.run_until_complete(asyncio.sleep(0.1))
def main4(): loop = asyncio.get_event_loop() future = asyncio.gather(prime_filter(2, 100), square_mapper(1, 100)) future.add_done_callback(lambda x: print(x.result())) loop.run_until_complete(future) loop.close()
def insert_topics(data): #es = Elasticsearch(['%s:%d'%(es_ip, es_port)]) some_bulks = '' bulk_result = 0 #bulk_result = None try: result = teaclient.request(data) #print(result) root = et.fromstring(result) status = root.findall( "./results/result[@name='status']")[0].text if len( root.findall("./results/result[@name='status']")) > 0 else '' #print(">>> Tea client response : %s" % status) if status == "success" and len( root.findall("./results/result[@name='keywords']")) > 0: result_scd = root.findall( "./results/result[@name='keywords']")[0].text terms = "" verbs = "" for line in result_scd.split("\n"): if line.startswith("<TERMS>"): terms = line.replace("<TERMS>", "") # 하늘:387^테스트:14^도움:11 elif line.startswith("<VERBS>"): verbs = line.replace("<VERBS>", "") # 하늘:387^테스트:14^도움:11 #print("### terms : %s" % terms) # <TERMS> #t = asyncio.ensure_future(time_log()) terms = [('NN', term) for term in terms.split(teaclient.ITEM_DELIMITER)] verbs = [('VV', verb) for verb in verbs.split(teaclient.ITEM_DELIMITER)] # 2018.03.26 terms와 verbs에 모두 등장하면 명사형으로 간주. newDict = {} for cl, topic in terms + verbs: if topic in newDict: newDict[topic]['cnt'] += 1 else: newDict[topic] = {'cnt': 1, 'topic_class': cl} newArr = [] for x in newDict.items(): if x[1]['cnt'] > 1 or x[1]['topic_class'] == 'NN': newArr.append(('NN', x[0])) else: newArr.append(('VV', x[0])) fts = [make_bulk(t, data) for t in (newArr)] #t.cancel() some_bulks = yield from asyncio.gather(*fts) for bulk in some_bulks: print(bulk) except ParseError as xmlerror: logger.error("[insert_topics] TeaClient failed. (%s)" % str(xmlerror)) logger.error("==============> teaclient's xml response : %s" % result)
#await asyncio.sleep(8) await ipc.disconnect() async def call_2(): ipc = AsyncIpcClient(klass=Hello(), resending=False) await ipc.connect() my_class = ipc.server # Get class instance proxy items = 1 while True: try: jobs = [my_class.echo(1, 2, 3, me="hello", you=3) for _ in range(items)] t = time.time() res = await asyncio.gather(*jobs) except Exception as e: print(e) #print(f"it took {(time.time() -t):.2f} seconds to complete {items} requests.\n" # f"One request took {(time.time()-t) / items * 1000:.4f} millisecond") #print(res[0]) await asyncio.sleep(8) await ipc.disconnect() try: jobs = [call_2() for _ in range(100)] asyncio.get_event_loop().run_until_complete(asyncio.gather(*jobs)) except KeyboardInterrupt: print("Keyboard interrupt")
def close_rpc(self): if self.status != Status.closed: rpc.active.discard(self) self.status = Status.closed return asyncio.gather(*self.close_comms())
async with aiohttp.ClientSession() as session: html = await get_content(session, url) parser_content(html) # 全部网页 urls = [ 'http://bang.dangdang.com/books/bestsellers/01.00.00.00.00.00-recent7-0-0-1-%d' % i for i in range(1, 3) ] # 统计该爬虫的消耗时间 print('#' * 50) t1 = time.time() # 开始时间 # 利用asyncio模块进行异步IO处理 ''' 获取Eventloop 执行coroutine ''' loop = asyncio.get_event_loop() # loop.run_until_complete(download(urls)) tasks = [asyncio.ensure_future(download(url)) for url in urls] tasks = asyncio.gather(*tasks) loop.run_until_complete(tasks) loop.close() t2 = time.time() # 结束时间 print('使用一般方法,总共耗时:%s' % (t2 - t1)) print('#' * 50)
def get(self): """ copies file from remote host to local host performs file split/transfer/join/verify functions :returns loop_start: time when transfers started :type: datetime object :returns loop_end: time when transfers ended :type: datetime object """ ssh_kwargs = { "username": self.user, "hostname": self.host, "password": self.passwd, "key_filename": self.ssh_key, "ssh_port": self.ssh_port, } # handle sigint gracefully on *nix, WIN32 is (of course) a basket case signal.signal(signal.SIGINT, self.handlesigint) # connect to host self.sshshell, ssh_kwargs = self.scs.connect(SSHShell, **ssh_kwargs) # determine remote host os junos, evo, bsd_version, sshd_version = self.scs.which_os() # verify which protocol to use self.copy_proto, self.passwd = self.scs.which_proto(self.copy_proto) # ensure dest path is valid self.validate_remote_path_get() # check required binaries exist on remote host self.scs.req_binaries(junos=junos, evo=evo) # cleanup previous remote tmp directory if found self.scs.remote_cleanup( remote_dir=self.remote_dir, remote_file=self.remote_file, silent=True ) # delete target file if it already exists self.delete_target_local() # determine remote file size file_size = self.remote_filesize() # determine optimal size for chunks split_size, executor = self.scs.file_split_size( file_size, sshd_version, bsd_version, evo, self.copy_proto ) # confirm remote storage is sufficient self.scs.storage_check_remote(file_size, split_size) # confirm local storage is sufficient self.scs.storage_check_local(file_size) if not self.noverify: # get/create sha hash for remote file sha_hash = self.remote_sha_get() # create tmp directory on remote host remote_tmpdir = self.scs.mkdir_remote() # split file into chunks self.split_file_remote(SCPClient, file_size, split_size, remote_tmpdir) # add chunk names to a list, pass this info to Progress chunks = self.get_chunk_info(remote_tmpdir) self.progress.add_chunks(file_size, chunks) # begin connection/rate limit check and transfer process command_list = [] if junos or evo: command_list = self.scs.limit_check(self.copy_proto) print("starting transfer...") self.progress.start_progress(self.use_curses) with self.scs.tempdir(): # copy files from remote host self.hard_close = True loop_start = datetime.datetime.now() loop = asyncio.new_event_loop() tasks = [] for chunk in chunks: task = loop.run_in_executor( executor, functools.partial( self.get_files, FTP, SSHShell, SCPClient, chunk, remote_tmpdir, ssh_kwargs, ), ) tasks.append(task) try: loop.run_until_complete(asyncio.gather(*tasks)) except TransferError: self.progress.stop_progress() self.scs.close( err_str="an error occurred while copying the files from the remote host", hard_close=self.hard_close, ) finally: loop.close() self.hard_close = False while self.progress.totals["percent_done"] != 100: time.sleep(0.1) self.progress.stop_progress() print("\ntransfer complete") loop_end = datetime.datetime.now() # combine chunks self.join_files_local() # remove remote tmp dir self.scs.remote_cleanup() # rollback any config changes made if command_list: self.scs.limits_rollback() if self.noverify: print( f"file has been successfully copied to {self.local_dir}/{self.local_file}" ) else: # generate a sha hash for the combined file, compare to hash of src self.local_sha_get(sha_hash) self.sshshell.close() return loop_start, loop_end
if filetuple_or_none is None: break file_and_stats = await get_file_and_stats(*filetuple_or_none) file_matched_size = await match_file_size(file_and_stats, args.smaller, args.bigger) file_matched_date = await match_file_date(file_and_stats, args.days) file_matched_suffix = await match_file_suffix(file_and_stats, args.suffix) if file_matched_size and file_matched_date and file_matched_suffix: txt = '{:<30.30}'.format(os.path.basename(file_matched_size[0])) if args.output: if 'date' in args.output: txt += ' {}'.format( datetime.date.fromtimestamp( file_matched_size[1].st_mtime)) if 'size' in args.output: txt += ' {:<15}'.format(file_matched_size[1].st_size) print(txt) if __name__ == '__main__': args = get_args() loop = asyncio.get_event_loop() queue = asyncio.Queue(loop=loop) producer = produce_files(queue, args.path) consumer = main(args, queue) loop.run_until_complete(asyncio.gather(producer, consumer)) loop.close()
def run(self): self._loop.run_until_complete( asyncio.gather(self._workers_tasks, self.consume()))
root@ubuntu:/home/zhouping/anaconda3/pyLearn/scalary# python37 asyncio_sleep_gather.py hello world! hello Python! [None, None] root@ubuntu:/home/zhouping/anaconda3/pyLearn/scalary# cat asyncio_sleep_gather.py import asyncio async def hello_world(): print("hello world!") async def hello_python(): print("hello Python!") await asyncio.sleep(0.1) event_loop = asyncio.get_event_loop() try: result = event_loop.run_until_complete(asyncio.gather( hello_world(), hello_python(), )) print(result) finally: event_loop.close() root@ubuntu:/home/zhouping/anaconda3/pyLearn/scalary#
def process(self, event): """ :type event: Event """ run_before_tasks = [] tasks = [] command_prefix = event.conn.config.get('command_prefix', '.') # Raw IRC hook for raw_hook in self.plugin_manager.catch_all_triggers: # run catch-all coroutine hooks before all others - TODO: Make this a plugin argument if not raw_hook.threaded: run_before_tasks.append( self.plugin_manager.launch(raw_hook, Event(hook=raw_hook, base_event=event))) else: tasks.append(self.plugin_manager.launch(raw_hook, Event(hook=raw_hook, base_event=event))) if event.irc_command in self.plugin_manager.raw_triggers: for raw_hook in self.plugin_manager.raw_triggers[event.irc_command]: tasks.append(self.plugin_manager.launch(raw_hook, Event(hook=raw_hook, base_event=event))) # Event hooks if event.type in self.plugin_manager.event_type_hooks: for event_hook in self.plugin_manager.event_type_hooks[event.type]: tasks.append(self.plugin_manager.launch(event_hook, Event(hook=event_hook, base_event=event))) if event.type is EventType.message: # Commands if event.chan.lower() == event.nick.lower(): # private message, no command prefix command_re = r'(?i)^(?:[{}]?|{}[,;:]+\s+)(\w+)(?:$|\s+)(.*)'.format(command_prefix, event.conn.nick) else: command_re = r'(?i)^(?:[{}]|{}[,;:]+\s+)(\w+)(?:$|\s+)(.*)'.format(command_prefix, event.conn.nick) cmd_match = re.match(command_re, event.content) if cmd_match: command = cmd_match.group(1).lower() if command in self.plugin_manager.commands: command_hook = self.plugin_manager.commands[command] command_event = CommandEvent(hook=command_hook, text=cmd_match.group(2).strip(), triggered_command=command, base_event=event) tasks.append(self.plugin_manager.launch(command_hook, command_event)) else: potential_matches = [] for potential_match, plugin in self.plugin_manager.commands.items(): if potential_match.startswith(command): potential_matches.append((potential_match, plugin)) if potential_matches: if len(potential_matches) == 1: command_hook = potential_matches[0][1] command_event = CommandEvent(hook=command_hook, text=cmd_match.group(2).strip(), triggered_command=command, base_event=event) tasks.append(self.plugin_manager.launch(command_hook, command_event)) else: event.notice("Possible matches: {}".format( formatting.get_text_list([command for command, plugin in potential_matches]))) # Regex hooks for regex, regex_hook in self.plugin_manager.regex_hooks: if not regex_hook.run_on_cmd and cmd_match: pass else: regex_match = regex.search(event.content) if regex_match: regex_event = RegexEvent(hook=regex_hook, match=regex_match, base_event=event) tasks.append(self.plugin_manager.launch(regex_hook, regex_event)) # Run the tasks yield from asyncio.gather(*run_before_tasks, loop=self.loop) yield from asyncio.gather(*tasks, loop=self.loop)
('waiting', 'yellow', 'default'), ('navigation', 'dark gray,underline', 'default'), ('mainview_title', 'default,bold', 'default'), ('reveal focus', 'default,standout', 'default'), ('dialog background', 'default', 'dark gray'), ('dialog button', 'default', 'dark gray'), ('dialog button focused', 'default', 'light blue'), ('edit', 'default,underline', 'default')] frame = urwid_views.MainFrame() welcome_view = urwid_views.WelcomeView(frame) welcome_view.app_quit = app_quit frame.set_body(welcome_view) evl = urwid.AsyncioEventLoop(loop=asyncio.get_event_loop()) loop = urwid.MainLoop(frame, event_loop=evl, unhandled_input=unhandled_input, palette=palette) urwid_views.set_ui(loop.screen) try: loop.run() except KeyboardInterrupt: pass finally: for task in asyncio.Task.all_tasks(): task.cancel() asyncio.get_event_loop().run_until_complete( asyncio.gather(*asyncio.Task.all_tasks())) asyncio.get_event_loop().close() print("Goodbye!")
def serve( host, port, request_handler, error_handler, before_start=None, after_start=None, before_stop=None, after_stop=None, debug=False, request_timeout=60, response_timeout=60, keep_alive_timeout=5, ssl=None, sock=None, request_max_size=None, reuse_port=False, loop=None, protocol=HttpProtocol, backlog=100, register_sys_signals=True, run_multiple=False, run_async=False, connections=None, signal=Signal(), request_class=None, access_log=True, keep_alive=True, is_request_stream=False, router=None, websocket_max_size=None, websocket_max_queue=None, websocket_read_limit=2 ** 16, websocket_write_limit=2 ** 16, state=None, graceful_shutdown_timeout=15.0, ): """Start asynchronous HTTP Server on an individual process. :param host: Address to host on :param port: Port to host on :param request_handler: Sanic request handler with middleware :param error_handler: Sanic error handler with middleware :param before_start: function to be executed before the server starts listening. Takes arguments `app` instance and `loop` :param after_start: function to be executed after the server starts listening. Takes arguments `app` instance and `loop` :param before_stop: function to be executed when a stop signal is received before it is respected. Takes arguments `app` instance and `loop` :param after_stop: function to be executed when a stop signal is received after it is respected. Takes arguments `app` instance and `loop` :param debug: enables debug output (slows server) :param request_timeout: time in seconds :param response_timeout: time in seconds :param keep_alive_timeout: time in seconds :param ssl: SSLContext :param sock: Socket for the server to accept connections from :param request_max_size: size in bytes, `None` for no limit :param reuse_port: `True` for multiple workers :param loop: asyncio compatible event loop :param protocol: subclass of asyncio protocol class :param request_class: Request class to use :param access_log: disable/enable access log :param websocket_max_size: enforces the maximum size for incoming messages in bytes. :param websocket_max_queue: sets the maximum length of the queue that holds incoming messages. :param websocket_read_limit: sets the high-water limit of the buffer for incoming bytes, the low-water limit is half the high-water limit. :param websocket_write_limit: sets the high-water limit of the buffer for outgoing bytes, the low-water limit is a quarter of the high-water limit. :param is_request_stream: disable/enable Request.stream :param router: Router object :param graceful_shutdown_timeout: How long take to Force close non-idle connection :return: Nothing """ if not run_async: # create new event_loop after fork loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) if debug: loop.set_debug(debug) connections = connections if connections is not None else set() server = partial( protocol, loop=loop, connections=connections, signal=signal, request_handler=request_handler, error_handler=error_handler, request_timeout=request_timeout, response_timeout=response_timeout, keep_alive_timeout=keep_alive_timeout, request_max_size=request_max_size, request_class=request_class, access_log=access_log, keep_alive=keep_alive, is_request_stream=is_request_stream, router=router, websocket_max_size=websocket_max_size, websocket_max_queue=websocket_max_queue, websocket_read_limit=websocket_read_limit, websocket_write_limit=websocket_write_limit, state=state, debug=debug, ) server_coroutine = loop.create_server( server, host, port, ssl=ssl, reuse_port=reuse_port, sock=sock, backlog=backlog, ) # Instead of pulling time at the end of every request, # pull it once per minute loop.call_soon(partial(update_current_time, loop)) if run_async: return server_coroutine trigger_events(before_start, loop) try: http_server = loop.run_until_complete(server_coroutine) except BaseException: logger.exception("Unable to start server") return trigger_events(after_start, loop) # Ignore SIGINT when run_multiple if run_multiple: signal_func(SIGINT, SIG_IGN) # Register signals for graceful termination if register_sys_signals: _singals = (SIGTERM,) if run_multiple else (SIGINT, SIGTERM) for _signal in _singals: try: loop.add_signal_handler(_signal, loop.stop) except NotImplementedError: logger.warning( "Sanic tried to use loop.add_signal_handler " "but it is not implemented on this platform." ) pid = os.getpid() try: logger.info("Starting worker [%s]", pid) loop.run_forever() finally: logger.info("Stopping worker [%s]", pid) # Run the on_stop function if provided trigger_events(before_stop, loop) # Wait for event loop to finish and all connections to drain http_server.close() loop.run_until_complete(http_server.wait_closed()) # Complete all tasks on the loop signal.stopped = True for connection in connections: connection.close_if_idle() # Gracefully shutdown timeout. # We should provide graceful_shutdown_timeout, # instead of letting connection hangs forever. # Let's roughly calcucate time. start_shutdown = 0 while connections and (start_shutdown < graceful_shutdown_timeout): loop.run_until_complete(asyncio.sleep(0.1)) start_shutdown = start_shutdown + 0.1 # Force close non-idle connection after waiting for # graceful_shutdown_timeout coros = [] for conn in connections: if hasattr(conn, "websocket") and conn.websocket: coros.append(conn.websocket.close_connection()) else: conn.close() _shutdown = asyncio.gather(*coros, loop=loop) loop.run_until_complete(_shutdown) trigger_events(after_stop, loop) loop.close()
def main(addresses): return asyncio.gather(*(connect_to_device(address) for address in addresses))
def wait(): pending = asyncio.Task.all_tasks() relevant_tasks = [t for t in pending if ('test_' not in t._coro.__name__)] yield from asyncio.gather(*relevant_tasks)
async def forward_request( self) -> [wresults.AbstractWazuhResult, exception.WazuhException]: """Forward a request to the node who has all available information to answer it. This function is called when a distributed_master function is used. Only the master node calls this function. An API request will only be forwarded to worker nodes. Returns ------- wresults.AbstractWazuhResult or exception.WazuhException """ async def forward( node_name: Tuple ) -> [wresults.AbstractWazuhResult, exception.WazuhException]: """Forward a request to a node. Parameters ---------- node_name : tuple Node to forward a request to. Returns ------- wresults.AbstractWazuhResult or exception.WazuhException """ node_name, agent_list = node_name if agent_list: self.f_kwargs['agent_id' if 'agent_id' in self.f_kwargs else 'agent_list'] = agent_list if node_name == 'unknown' or node_name == '' or node_name == self.node_info[ 'node']: # The request will be executed locally if the the node to forward to is unknown, empty or the master # itself result = await self.distribute_function() else: if 'tmp_file' in self.f_kwargs: await self.send_tmp_file(node_name) client = self.get_client() try: result = json.loads(await client.execute( b'dapi_forward', "{} {}".format( node_name, json.dumps( self.to_dict(), cls=c_common.WazuhJSONEncoder)).encode(), self.wait_for_complete), object_hook=c_common.as_wazuh_object) except WazuhClusterError as e: if e.code == 3022: result = e else: raise e # Convert a non existing node into a WazuhError exception if isinstance(result, WazuhClusterError) and result.code == 3022: common.rbac.set(self.rbac_permissions) try: await get_nodes_info(client, filter_node=[node_name]) except WazuhError as e: if e.code == 4000: result = e dikt = result.to_dict() # Add node ID to error message dikt['ids'] = {node_name} result = WazuhError.from_dict(dikt) return result if isinstance(result, (wresults.AbstractWazuhResult, exception.WazuhException)) \ else wresults.WazuhResult(result) # get the node(s) who has all available information to answer the request. nodes = await self.get_solver_node() self.from_cluster = True common.rbac.set(self.rbac_permissions) common.cluster_nodes.set(self.nodes) common.broadcast.set(self.broadcasting) if 'node_id' in self.f_kwargs or 'node_list' in self.f_kwargs: # Check cluster:read permissions for each node filter_node_kwarg = {'filter_node': list(nodes)} if nodes else {} allowed_nodes = await get_nodes_info(self.get_client(), **filter_node_kwarg) valid_nodes = list() if not nodes: nodes = { node_name['name']: [] for node_name in allowed_nodes.affected_items } for node in nodes.items(): if node[0] in [ node_name['name'] for node_name in allowed_nodes.affected_items ] or node[0] == 'unknown': valid_nodes.append(node) del self.f_kwargs['node_id' if 'node_id' in self.f_kwargs else 'node_list'] else: if nodes: valid_nodes = list(nodes.items()) else: broadcasted_nodes = await get_nodes_info(self.get_client()) valid_nodes = [(n['name'], []) for n in broadcasted_nodes.affected_items] allowed_nodes = wresults.AffectedItemsWazuhResult() allowed_nodes.affected_items = list(nodes) allowed_nodes.total_affected_items = len( allowed_nodes.affected_items) response = await asyncio.shield( asyncio.gather(*[forward(node) for node in valid_nodes])) if allowed_nodes.total_affected_items > 1: response = reduce(or_, response) if isinstance(response, wresults.AbstractWazuhResult): response = response.limit(limit=self.f_kwargs.get('limit', common.database_limit), offset=self.f_kwargs.get('offset', 0)) \ .sort(fields=self.f_kwargs.get('fields', []), order=self.f_kwargs.get('order', 'asc')) elif response: response = response[0] else: response = deepcopy(allowed_nodes) # It might be a WazuhError after reducing if isinstance(response, wresults.AffectedItemsWazuhResult): for failed in copy(allowed_nodes.failed_items): # Avoid errors coming from 'unknown' node (they are included in the forward) if allowed_nodes.failed_items[failed] == {'unknown'}: del allowed_nodes.failed_items[failed] response.add_failed_items_from(allowed_nodes) return response
await download_file(export_file_url, path / export_file_name) try: learn = load_learner(path, export_file_name) return learn except RuntimeError as e: if len(e.args) > 0 and 'CPU-only machine' in e.args[0]: print(e) message = "\n\nThis model was trained with an old version of fastai and will not work in a CPU environment.\n\nPlease update the fastai library in your training environment and export your model again.\n\nSee instructions for 'Returning to work' at https://course.fast.ai." raise RuntimeError(message) else: raise loop = asyncio.get_event_loop() tasks = [asyncio.ensure_future(setup_learner())] learn = loop.run_until_complete(asyncio.gather(*tasks))[0] loop.close() @app.route('/') async def homepage(request): html_file = path / 'view' / 'index.html' return HTMLResponse(html_file.open().read()) @app.route('/analyze', methods=['POST']) async def analyze(request): img_data = await request.form() img_bytes = await (img_data['file'].read()) img = open_image(BytesIO(img_bytes)) prediction = learn.predict(img)[0]
await asyncio.sleep(1.0) # time.sleep(1) return tuple(primes) async def square_mapper(m, n): """ Square mapper co-routine """ squares = [] for i in number_generator(m, n): print('Square=>',i*i) squares.append(i*i) # Uncomment this out and comment next line and see what happens !!! await asyncio.sleep(1.0) # time.sleep(1) return squares def print_result(future): print('Result=>',future.result()) loop = asyncio.get_event_loop() future = asyncio.gather(prime_filter(10, 50), square_mapper(10, 50)) future.add_done_callback(print_result) loop.run_until_complete(future) loop.close()