def with_timeout(timeout, future_or_iterable): """ Yields on a future (or a list of futures) for a maximum amount of time Returns the results of the completed futures completed after timeout (or None if the future is still pending) :param timeout: The number of seconds to wait for the future. 0 or None to disable. :param future_or_iterable: A future to yield on, or a list of futures :return: The results from the future or the list of futures, or None if the future didn't complete in time """ # Waiting for a list if isinstance(future_or_iterable, list): futures = future_or_iterable try: if timeout: yield gen.with_timeout(timedelta(seconds=timeout), gen.multi(futures)) else: yield gen.multi(futures) except util.TimeoutError: pass return [None if not future.done() else future.result() for future in futures] # Waiting for a single future try: if timeout: yield gen.with_timeout(timedelta(seconds=timeout), future_or_iterable) else: yield future_or_iterable except util.TimeoutError: pass return None if not future_or_iterable.done() else future_or_iterable.result()
def close(self, timeout=None): self.state = CLOSING self.logger.debug("Flushing send queue") drain_futures = [drain.close() for drain in self.emitter.values()] if timeout: yield gen.with_timeout(timeout.total_seconds(), gen.multi(drain_futures)) else: yield gen.multi(drain_futures)
def initialize_env(): phases = WorkEnv.Environment.keys() #scheduler = TornadoScheduler() for idx, p in enumerate(phases): defi = WorkEnv.Environment[p] ini = {} ini["phase"] = idx + 1 fp = open(ApplicationProperties.configure( "application.storage.filesystem.uc.rootdir") + "/temp_data/phase" + str(ini["phase"]), 'w+', encoding="utf-8") if "pagination" in defi: bridgeq = queues.Queue() ini["pagination"] = defi["pagination"] ini["selectors"] = defi["selectors"] interim = ini["pagination"]["interim"] print("gen pworkers") pworkers = gen.multi([ pagination( ini["pagination"], defi["srcq"], bridgeq, "phase[" + str(ini["phase"]) + "]-pagination[" + str(_) + "]") for _ in range(p_concurrency) ]) #print(pworkers) print(defi["srcq"]) print(bridgeq) #scheduler.add_job(pagination,'interval', seconds=interim, args=(ini, defi["srcq"], bridgeq)) cworkers = gen.multi([ worker( Crawler(ini), bridgeq, defi["distq"], "phase[" + str(ini["phase"]) + "]-crawler[" + str(_) + "]", fp) for _ in range(c_concurrency) ]) print("gen cworkers") print(cworkers) print(bridgeq) print(defi["distq"]) else: ini["selectors"] = defi["selectors"] dq = None if "distq" in defi: dq = defi["distq"] print(defi["distq"]) cworkers = gen.multi([ worker( Crawler(ini), defi["srcq"], dq, "phase[" + str(ini["phase"]) + "]-crawler[" + str(_) + "]", fp) for _ in range(c_concurrency) ]) print("gen cworkers") print(cworkers) print(defi["srcq"])
def close(self, timeout=None): self.state = CLOSING self.logger.debug("Flushing send queue") drain_futures = [ drain.close() for drain in self.emitter.values() ] if timeout: yield gen.with_timeout(timeout.total_seconds(), gen.multi(drain_futures)) else: yield gen.multi(drain_futures)
def run(self): """Starts the clients serial aggregation workflow.""" # Connect all clients yield gen.multi([client.connect() for client in self.clients.values()]) # Start stdin listener self._listen_stdin() # Start listening all opened client connections yield gen.multi([ client.listen() for client in self.clients.values() if client.websocket is not None ]) tornado.ioloop.IOLoop.instance().stop()
def assemble_fresh_response(self): yield self.__class__._lock.acquire() self.logger.log(logging.INFO, "Fetching latest release from GitHub") data = None failure = True try: data = yield gen.multi({ "stats": assemble_stats(), "flavors": assemble_flavors() }) failure = False except tornado.httpclient.HTTPError as error: response = error.response self.logger.log( logging.ERROR, "API error: {} -> {} ({})".format(response.effective_url, response.error, response.body), ) except Exception as error: self.logger.log(logging.ERROR, error) if failure: self.__class__._last_failed_request = datetime.now() self.__class__._lock.release() self.send_error(500) return False self.update_cache(data) self.__class__._lock.release() self.add_default_headers() self.write(data) self.finish()
async def main(): seen_set = set() q = queues.Queue() async def fetch_url(current_url): # 生产者 if current_url in seen_set: return print(f"获取: {current_url}") seen_set.add(current_url) next_urls = await get_url_links(current_url) for new_url in next_urls: if new_url.startswith(base_url): # 放入队列, 使用await 是为了当这个协程放不进去的时候切换出来,切换到另一个get()协程上 await q.put(new_url) async def worker(): async for url in q: if url is None: return try: await fetch_url(url) except Exception as e: print("Exception") finally: # 将数据减一 消费了一个 q.task_done() await q.put(base_url) workers = gen.multi([worker() for _ in range(concurrency)]) await q.join() for _ in range(concurrency): await q.put(None) await workers
def server_stop(): deinit_futures = [ integration.deinitialize_app(app) for integration in app.available_integrations ] if deinit_futures: def await_deinit(future): if future.exception() is not None: log.error('failed to deinit, deinit returned: %s', future.exception()) io_loop.add_future(gen.multi(deinit_futures), await_deinit) http_server.stop() if ioloop_is_running(): log.info('going down in %s seconds', options.stop_timeout) def ioloop_stop(): if ioloop_is_running(): log.info('stopping IOLoop') io_loop.stop() log.info('stopped') io_loop.add_timeout(time.time() + options.stop_timeout, ioloop_stop)
def assemble_stats(): repo_url = "https://api.github.com/repos/TeamNewPipe/NewPipe" contributors_url = "https://github.com/TeamNewPipe/NewPipe" translations_url = "https://hosted.weblate.org/api/components/newpipe/" \ "strings/translations/" repo_data, contributors_data, translations_data = \ [x.body for x in (yield gen.multi(( fetch(repo_url), fetch(contributors_url), fetch(translations_url), )))] repo = json.loads(repo_data) translations = json.loads(translations_data) document = html.fromstring(contributors_data) tags = document.cssselect(".numbers-summary a[href$=contributors] .num") contributors = int(tags[0].text) return { "stargazers": repo["stargazers_count"], "watchers": repo["subscribers_count"], "forks": repo["forks_count"], "contributors": contributors, "translations": int(translations["count"]), }
def assemble_flavors(): return gen.multi({ "github": get_github_flavor("NewPipe"), "fdroid": get_fdroid_flavor("org.schabi.newpipe"), "github_legacy": get_github_flavor("NewPipe-legacy"), "fdroid_legacy": get_fdroid_flavor("org.schabi.newpipelegacy"), })
def handle_user(user): """Handle one user. Create a list of their servers, and async exec them. Wait for that to be done, and if all servers are stopped, possibly cull the user. """ # shutdown servers first. # Hub doesn't allow deleting users with running servers. # jupyterhub 0.9 always provides a 'servers' model. # 0.8 only does this when named servers are enabled. if 'servers' in user: servers = user['servers'] else: # jupyterhub < 0.9 without named servers enabled. # create servers dict with one entry for the default server # from the user model. # only if the server is running. servers = {} if user['server']: servers[''] = { 'last_activity': user['last_activity'], 'pending': user['pending'], 'url': user['server'], } server_futures = [ handle_server(user, server_name, server) for server_name, server in servers.items() ] results = yield multi(server_futures)
def _test(): url = app.get_url(uri) coros = (app.http_client.fetch(url) for _ in range(2)) responses = app.io_loop.run_sync(lambda: gen.multi(coros)) for response in responses: assert response.code == 200
def Main(p_command=None): print('PyTask loaded!') properties = ApplicationProperties(p_command) #dlist = dashboard.populate() + [ (RequestMapping.submit_fetch_url_task, MainHandler)] application = tornado.web.Application([ (RequestMapping.submit_fetch_url_task, MainHandler), (RequestMapping.get_current_fetch_worker_queue_num, dashboard), ]) application.listen(properties.getProperty(p_key='server.port')) workers = gen.multi([worker(idx) for idx in range(int(properties.getProperty(p_key='application.worker.fetchUrl.num')))]) ioLoop = tornado.ioloop.IOLoop.current() partitiondir = properties.getProperty(p_key='application.persistence.fetchdir') + '/' + properties.getProperty(p_key='application.persistence.defaultPartition') if( os.path.exists( partitiondir ) ): for vol in os.listdir( partitiondir ): if vol.lower().startswith('volumn'): vols.append(partitiondir+"/"+vol) print('%d volumns in %s'%(len(vols), properties.getProperty(p_key='application.persistence.defaultPartition') )) #request = HttpRequest() #await request.do() #ApplicationContext.getContext().putInstance( p_name='var.application.inq', p_obj=inq ) ApplicationContext.getContext().putInstance( p_name='var.application.workers', p_obj=workers ) ApplicationContext.getContext().putInstance( p_name='var.application.configuration', p_obj=properties ) ApplicationContext.getContext().putInstance( p_name='var.application.webApplication', p_obj=application ) #ApplicationContext.getContext().putInstance( p_name='var.application.urlRequest', p_obj=request ) ApplicationContext.getContext().putInstance( p_name='var.ioloop.current', p_obj=ioLoop ) ioLoop.start()
def test_rate_limiting_boto3(self): """ Test that rate limiting with boto3 works. """ # Each one of these will raise a throttling exception, then succeed. # Between calls, each should delay for 1 cycle of `delay_min`. # Delay min is 0.05s, so total runtime should be ~0.15s. api_call_queue_calls = [ self.api_call_queue.call( self._mock_api_function_sync, result=1, exception=[self.boto3_throttle_exception]), self.api_call_queue.call( self._mock_api_function_sync, result=2, exception=[self.boto3_throttle_exception]), self.api_call_queue.call( self._mock_api_function_sync, result=3, exception=[self.boto3_throttle_exception]), ] start = time.time() results = yield gen.multi(api_call_queue_calls) stop = time.time() run_time = stop - start self.assertTrue(0.15 <= run_time < 0.25) self.assertEqual(results, [1, 2, 3])
def handle(self): query = self.get_argument('query', None) if not query: ret = {} ret["status"] = "missing params" serverlg.info( '[chatbot] [ERROR: missing params] [REQUEST] [%s] [%s]' % (time.strftime('%Y-%m-%d %H:%M:%S'), self.request.uri)) self.write(json.dumps(ret, ensure_ascii=False)) self.finish() results = [] debug_infos = [] graph_stubs = [ model_conf["graph_stub"] for model_conf in schedule["servables"] ] # Multi model compatible, but here just one model exists multi_models = [] for graph, stub in graph_stubs: multi_models.append( self.run_model(graph, stub, [query.encode("utf-8")])) outs = yield multi(multi_models) serverlg.info(outs) raise gen.Return(None)
def test_concurrent_calls_with_delay(self): """ Test concurrent calls with some latency run serially and return independent results. The api_call_queue runs calls synchronously and serially. """ api_call_queue_calls = [ self.api_call_queue.call( self._mock_api_function_sync, result=1, delay=0.05), self.api_call_queue.call( self._mock_api_function_sync, result=2, delay=0.05), self.api_call_queue.call( self._mock_api_function_sync, result=3, delay=0.05), self.api_call_queue.call( self._mock_api_function_sync, result=4, delay=0.05), self.api_call_queue.call( self._mock_api_function_sync, result=5, delay=0.05), ] start = time.time() results = yield gen.multi(api_call_queue_calls) stop = time.time() run_time = stop - start self.assertTrue(0.25 <= run_time < 0.35) self.assertEqual(results, [1, 2, 3, 4, 5])
def test_rate_limiting_boto3(self): """ Test that rate limiting with boto3 works. """ # Each one of these will raise a throttling exception, then succeed. # Between calls, each should delay for 1 cycle of `delay_min`. # Delay min is 0.05s, so total runtime should be ~0.15s. api_call_queue_calls = [ self.api_call_queue.call(self._mock_api_function_sync, result=1, exception=[self.boto3_throttle_exception ]), self.api_call_queue.call(self._mock_api_function_sync, result=2, exception=[self.boto3_throttle_exception ]), self.api_call_queue.call(self._mock_api_function_sync, result=3, exception=[self.boto3_throttle_exception ]), ] start = time.time() results = yield gen.multi(api_call_queue_calls) stop = time.time() run_time = stop - start self.assertTrue(0.15 <= run_time < 0.25) self.assertEqual(results, [1, 2, 3])
def test_concurrent_calls_with_delay(self): """ Test concurrent calls with some latency run serially and return independent results. The api_call_queue runs calls synchronously and serially. """ api_call_queue_calls = [ self.api_call_queue.call(self._mock_api_function_sync, result=1, delay=0.05), self.api_call_queue.call(self._mock_api_function_sync, result=2, delay=0.05), self.api_call_queue.call(self._mock_api_function_sync, result=3, delay=0.05), self.api_call_queue.call(self._mock_api_function_sync, result=4, delay=0.05), self.api_call_queue.call(self._mock_api_function_sync, result=5, delay=0.05), ] start = time.time() results = yield gen.multi(api_call_queue_calls) stop = time.time() run_time = stop - start self.assertTrue(0.25 <= run_time < 0.35) self.assertEqual(results, [1, 2, 3, 4, 5])
def get(self, jwt, msg_id): user = yield self.authenticate_user(jwt) if user: setattr(self, 'user', user) if self.user: cursor_check = yield db.execute( QUERIES.CHECK_USER_ACCESS_TO_MSG_FILES, (self.user.id, self.user.id, msg_id)) if cursor_check.fetchone()[0] == 1: cursor_streamed_files = yield db.execute( QUERIES.GET_STREAMED_FILES_INFO, (msg_id, )) files = [] s3_files = [] for file_ in ResultIter(cursor_streamed_files, 500): files.append(file_[1]) real_aws_url = s3_storage._generate_url(file_[0]) s3_files.append( httpclient.AsyncHTTPClient().fetch(real_aws_url)) s3_raw_data = yield gen.multi(s3_files) print s3_raw_data, files in_memory_zip = StringIO() zip_file = ZipFile(in_memory_zip, 'w', ZIP_DEFLATED) for i, file_ in enumerate(files): zip_file.writestr(file_, s3_raw_data[i].buffer.getvalue()) zip_file.close() in_memory_zip.seek(0) zip_content = in_memory_zip.getvalue() self.set_header('Content-Type', 'application/zip') self.set_header('Content-Length', len(zip_content)) self.set_header( 'Content-Disposition', 'attachment; filename=%s' % '{0}.zip'.format(str(uuid.uuid4()))) self.flush() self.write(zip_content) self.finish()
def start(self): global __g_userSpace global __g_memDB __g_memDB = MemDB() __g_userSpace = UserSpace(__g_memDB) application = tornado.web.Application([ (RequestMapping.register_node, RegisterHandler), (RequestMapping.fetch_dashboard, DashBoardHandler), ]) application.listen( ApplicationProperties.configure( p_key='application.controller.register.server.port')) print('Application listeming...[Port:%s]' % (ApplicationProperties.configure( p_key='application.controller.register.server.port'))) workers = gen.multi([worker(idx) for idx in range(1)]) __g_userSpace.init() ioLoop = tornado.ioloop.IOLoop.current() #print(ioLoop) ioLoop.start()
def get(self): client = httpclient.AsyncHTTPClient() futures = [] for provider in HOTEL_PROVIDERS: futures.append( client.fetch( "http://127.0.0.1:9000/scrapers/{}".format(provider))) responses = yield gen.multi(futures) results = [] for response in responses: # Put each hotel into a 2-tuple where the first value is it's ecstasy # and the second is the full hotel data # Sort by smalles ecstasy to greatest to use pythons merge heapq.merge results.append( list( reversed([(hotel['ecstasy'], hotel) for hotel in json.loads(response.body)['results'] ]))) # Merge all the lists of results together and reverse the result so largest is first sorted_hotels = [ hotel[1] for hotel in reversed(list(heapq.merge(*results))) ] self.write({'results': sorted_hotels})
async def main(): testing = False if testing: await tests() else: control_server_ip_address = '127.0.0.1' control_server_port_number = '8080' url_prefix = '/'.join(['http:', '', ':'.join([control_server_ip_address, control_server_port_number])]) user_home_directory = '/home/dannya951' max_subprocesses = 5 network = 'testnet' clients_working = True debug = True request_utility = Aru.AsyncRequestUtility(control_server_ip_address, control_server_port_number) post_clients_code = await request_utility.post_clients() if post_clients_code is None: print('Failed to create new clients.') clients_working = False while clients_working: intervals = await request_utility.get_client() interval_subprocesses = len(intervals) interval_subprocesses = min(interval_subprocesses, max_subprocesses) if interval_subprocesses == 0: # server is done allocating blocks, break from while loop clients_working = False else: gztar_name = 'blockchain' await initialize_all_local_blockchains( request_utility, user_home_directory, gztar_name, interval_subprocesses ) for interval in intervals: await request_utility.post_results({interval: {}}) connections = await Anm.async_identify_connections() await Anm.async_manage_connections(connections, 'disconnect', debug=debug) if debug: print('Starting ' + str(interval_subprocesses) + ' client subprocesses.') interval_names = sorted([ik for ik in intervals.keys()]) clients = gen.multi([run_client(subprocess_index, user_home_directory, intervals, network=network) for subprocess_index in range(interval_subprocesses)]) await clients await Anm.async_manage_connections(connections, 'connect', debug=debug) for subprocess_index in range(interval_subprocesses): interval_name = interval_names[subprocess_index] await request_utility.put_client({interval_name: intervals[interval_name]}) await request_utility.put_result({interval_name: intervals[interval_name]})
async def async_call(self, args=None, timeout=None): """ Calls system command and return parsed output or standard error output """ if args is None: args = [] # Executing command with Tornado subprocess is possible only in main thread if threading.main_thread().ident != threading.get_ident(): return self.call(args=args, timeout=timeout) all_args = [ self.CMD if self.CMD is not None else cfg['tools.%s.cmd' % self.NAME] ] all_args.extend(self.COMMON_ARGS) all_args.extend(args) cmd = ' '.join(all_args), log.debug('Executing: %s', cmd) if self._cancelled: raise Exception('Task was cancelled') task = process.Subprocess(all_args, stderr=process.Subprocess.STREAM, stdout=process.Subprocess.STREAM) self.proc = task.proc coroutine = gen.multi([ task.wait_for_exit(raise_error=False), task.stdout.read_until_close(), task.stderr.read_until_close() ]) if not timeout: return_code, stdout, stderr = await coroutine else: try: return_code, stdout, stderr = await gen.with_timeout( timedelta(seconds=timeout), coroutine) except gen.TimeoutError as exception: log.exception( "Command %s timed out after %s while executing %s", self.NAME, timeout, cmd) task.proc.kill() raise exception self.proc = None if return_code != 0: log.warning("Command '%s' failed wit exit code: %s", cmd, return_code) log.debug("Command '%s':\nSTDOUT:\n%s\nSTDERR:\n%s", cmd, stdout, stderr) if self.RAISE_ERROR: raise subprocess.CalledProcessError(return_code, cmd) return self.parser.parse(stdout.decode('utf-8'), stderr.decode('utf-8'))
def wait(futures): """Gather multiply futures into one future Returns: future: New future wait all child futures the result is a list contains the results of all child futures """ return multi(set(futures))
def _loop_read(self): for line in chain.from_iterable(self.config.inputs): if self._stop: break line = line.strip() yield self._queue.put(line) yield gen.multi( [self._queue.put(None) for _ in range(0, self.config.concurrency)])
def test_init_close_race(self): # Regression test for #2367 def f(): for i in range(10): loop = IOLoop() loop.close() yield gen.multi([self.io_loop.run_in_executor(None, f) for i in range(2)])
def _get_collection_png_tile(self, fc, x, y, z): rasters = yield gen.multi([ self._get_raster_png_tile(f.raster(), x, y, z) for f in fc ]) # type: ignore if len(rasters) < 1: return None tile = yield self._merge_rasters(rasters, z) return tile
def main(): args = parser.parse_args() r_host, r_port = args.redis.split(':') storage = RedisStorage(r_host, int(r_port), args.queue_timeout) yield storage.url_discovered(args.url) yield gen.multi([Crawler(storage).run() for i in range(args.workers)]) print 'All website crawled.' yield print_sitemap(storage)
def _stop_watchers(self, close_output_streams=False, watcher_iter_func=None, for_shutdown=False): if watcher_iter_func is None: watchers = self.iter_watchers(reverse=False) else: watchers = watcher_iter_func(reverse=False) futures = [w._stop(close_output_streams, for_shutdown) for w in watchers] yield gen.multi(futures)
def wait(tasks): """ Simple helper for gathering all passed :class:`Task`s. :param tasks: list of the :class:`asyncio.Task`s :param _loop: Event loop (:func:`tornado.ioloop.IOLoop.current()` when :class:`None`) :return: :class:`tuple` of results """ raise gen.Return((yield gen.multi(tasks)))
def run_tasks(): tasks = [ fetch1('http://example.com'), fetch2('http://example.com'), fetch3('http://example.com'), fetch4('http://example.com'), ] for r in (yield gen.multi(tasks)): print(r) print('done')
def invoke(): # important to start the sleep before starting callback so any initial # time spent in callback "counts against" the period. sleep_future = self.sleep() result = self._func() if result is None: return sleep_future callback_future = gen.convert_yielded(result) return gen.multi([sleep_future, callback_future])
def _run(self, args): try: IOLoop.current().spawn_callback(self._loop_read) yield gen.multi([ self._loop_process(args) for _ in range(0, self.config.concurrency) ]) except Exception as e: self._logger.error('Error {}'.format(e)) finally: yield self._close()
def invoke(): # important to start the sleep before starting callback # so any initial time spent in callback "counts against" # the period. sleep_future = self.sleep() result = self._func() try: callback_future = gen.convert_yielded(result) except gen.BadYieldError: # result is not a yieldable thing return sleep_future else: return gen.multi([sleep_future, callback_future])
def _run_preprocessors(self, preprocessors): for p in preprocessors: yield gen.coroutine(p)(self) if self._finished: self.log.info('page was already finished, breaking preprocessors chain') return False yield gen.multi(self._preprocessor_futures) self._preprocessor_futures = None if self._finished: self.log.info('page was already finished, breaking preprocessors chain') return False return True
def initialize_app(self, app) -> Optional[Future]: def get_kafka_producer(producer_name: str) -> AIOKafkaProducer: return self.kafka_producers.get(producer_name) app.get_kafka_producer = get_kafka_producer if options.kafka_clusters: init_futures = [] for cluster_name, cluster_settings in options.kafka_clusters.items(): if cluster_settings: producer = AIOKafkaProducer(loop=asyncio.get_event_loop(), **cluster_settings) self.kafka_producers[cluster_name] = producer init_futures.append(asyncio.ensure_future(producer.start())) return gen.multi(init_futures) return None
async def main(): q = queues.Queue() start = time.time() fetching, fetched = set(), set() async def fetch_url(current_url): if current_url in fetching: return print('fetching %s' % current_url) fetching.add(current_url) urls = await get_links_from_url(current_url) fetched.add(current_url) for new_url in urls: # Only follow links beneath the base URL if new_url.startswith(base_url): await q.put(new_url) async def worker(): async for url in q: if url is None: return try: await fetch_url(url) except Exception as e: print('Exception: %s %s' % (e, url)) finally: q.task_done() await q.put(base_url) # Start workers, then wait for the work queue to be empty. workers = gen.multi([worker() for _ in range(concurrency)]) await q.join(timeout=timedelta(seconds=300)) assert fetching == fetched print('Done in %d seconds, fetched %s URLs.' % ( time.time() - start, len(fetched))) # Signal all the workers to exit. for _ in range(concurrency): await q.put(None) await workers
def invoke(): # important to start the sleep before starting callback # so any initial time spent in callback "counts against" # the period. sleep_future = self.sleep() result = self._func() # This is needed for Tornado >= 4.5 where convert_yielded will no # longer raise BadYieldError on None if result is None: return sleep_future try: callback_future = gen.convert_yielded(result) except gen.BadYieldError: # result is not a yieldable thing return sleep_future else: return gen.multi([sleep_future, callback_future])
def _async_init_cb(): try: init_futures = app.default_init_futures + list(app.init_async()) if init_futures: def await_init(future): if future.exception() is not None: log.error('failed to initialize application, init_async returned: %s', future.exception()) sys.exit(1) run_server(app) ioloop.add_future(gen.multi(init_futures), await_init) else: run_server(app) except Exception: log.exception('failed to initialize application') sys.exit(1)
def test_api_call_queue_future_is_nonblocking(self): """ Test that the api call queue future is nonblocking for other futures. It needs to execute in a different thread because of the synchronous nature of AWS APIs. """ futures = [ self.api_call_queue.call( self._mock_api_function_sync, result=1, delay=0.05), self._mock_api_function_async(result=2, delay=0.05), self._mock_api_function_async(result=3, delay=0.05), self._mock_api_function_async(result=4, delay=0.05), self._mock_api_function_async(result=5, delay=0.05), ] start = time.time() results = yield gen.multi(futures) stop = time.time() run_time = stop - start self.assertTrue(0.05 <= run_time < 0.15) self.assertEqual(results, [1, 2, 3, 4, 5])
def test_api_call_queue_raises_exceptions(self): """ Test that the api call queue raises exceptions and proceeds to execute other queued api calls. """ @gen.coroutine def _call_without_exception(): result = yield self.api_call_queue.call( self._mock_api_function_sync, delay=0.05) self.assertEqual(result, 'OK') raise gen.Return('no exception') @gen.coroutine def _call_with_exception(): with self.assertRaises(ValueError): yield self.api_call_queue.call( self._mock_api_function_sync, exception=ValueError('test exception'), delay=0.05) raise gen.Return('exception') @gen.coroutine def _call_with_exception_after_boto2_rate_limit(): """ First rate limit, then raise an exception. This should take: call delay * 2 + min rate limiting delay * 1 """ with self.assertRaises(boto_exception.BotoServerError): yield self.api_call_queue.call( self._mock_api_function_sync, exception=[ self.boto2_throttle_exception_1, self.boto2_exception], delay=0.05) raise gen.Return('exception') @gen.coroutine def _call_with_exception_after_boto3_rate_limit(): """ First rate limit, then raise an exception. This should take: call delay * 2 + min rate limiting delay * 1 """ with self.assertRaises(botocore_exceptions.ClientError): yield self.api_call_queue.call( self._mock_api_function_sync, exception=[ self.boto3_throttle_exception, self.boto3_exception], delay=0.05) raise gen.Return('exception') call_wrappers = [ # Should take 0.05s. _call_without_exception(), # Should take 0.05s. _call_with_exception(), # Should take 0.05s. _call_without_exception(), # Should take 0.05s + 0.05s + 0.05s. _call_with_exception_after_boto2_rate_limit(), # Should take 0.05s + 0.05s + 0.05s. _call_with_exception_after_boto3_rate_limit(), ] start = time.time() results = yield gen.multi(call_wrappers) stop = time.time() run_time = stop - start self.assertTrue(0.45 <= run_time < 0.55) self.assertEqual( results, ['no exception', 'exception', 'no exception', 'exception', 'exception'])
def start(self): '''Start the challenge. Returns: dict: Challenge result. ''' print('StdChal %d started'%self.chal_id) self.chal_path = 'container/standard/home/%d'%self.uniqid with StackContext(Privilege.fileaccess): os.mkdir(self.chal_path, mode=0o771) yield self.prefetch() print('StdChal %d prefetched'%self.chal_id) if self.comp_typ in ['g++', 'clang++']: ret = yield self.comp_cxx() elif self.comp_typ == 'makefile': ret = yield self.comp_make() elif self.comp_typ == 'python3': ret = yield self.comp_python() if ret != PyExt.DETECT_NONE: ret_result = [(0, 0, STATUS_CE)] * len(self.test_list) else: print('StdChal %d compiled'%self.chal_id) if self.comp_typ == 'python3': exefile_path = self.chal_path \ + '/compile/__pycache__/test.cpython-34.pyc' exe_path = '/usr/bin/python3.4' argv = ['./a.out'] envp = ['HOME=/', 'LANG=en_US.UTF-8'] else: exefile_path = self.chal_path + '/compile/a.out' exe_path = './a.out' argv = [] envp = [] test_future = [] for test in self.test_list: test_future.append(self.judge_diff( exefile_path, exe_path, argv, envp, test['in'], test['ans'], test['timelimit'], test['memlimit'])) test_result = yield gen.multi(test_future) ret_result = list() for result in test_result: test_pass, data = result runtime, peakmem, error = data status = STATUS_ERR if error == PyExt.DETECT_NONE: if test_pass is True: status = STATUS_AC else: status = STATUS_WA elif error == PyExt.DETECT_OOM: status = STATUS_MLE elif error == PyExt.DETECT_TIMEOUT \ or error == PyExt.DETECT_FORCETIMEOUT: status = STATUS_TLE elif error == PyExt.DETECT_EXITERR: status = STATUS_RE else: status = STATUS_ERR ret_result.append((runtime, peakmem, status)) with StackContext(Privilege.fileaccess): shutil.rmtree(self.chal_path) print('StdChal %d done'%self.chal_id) return ret_result
def start(self): '''Start the challenge. Returns: dict: Challenge result. ''' cache_hash = None cache_gid = None # Check if special judge needs to rebuild. if self.judge_typ in ['ioredir']: hashproc = process.Subprocess( \ ['./HashDir.py', self.res_path + '/check'], \ stdout=process.Subprocess.STREAM) dirhash = yield hashproc.stdout.read_until(b'\n') dirhash = int(dirhash.decode('utf-8').rstrip('\n'), 16) ret = StdChal.build_cache_find(self.res_path) if ret is not None and ret[0] == dirhash: cache_hash, cache_gid = ret judge_ioredir = IORedirJudge('container/standard', \ '/cache/%x'%cache_hash) else: cache_hash = dirhash _, cache_gid = StdChal.get_standard_ugid() build_ugid = StdChal.get_standard_ugid() build_relpath = '/cache/%x'%cache_hash build_path = 'container/standard' + build_relpath judge_ioredir = IORedirJudge('container/standard', \ build_relpath) if not (yield judge_ioredir.build(build_ugid, self.res_path)): return [(0, 0, STATUS_ERR)] * len(self.test_list) FileUtils.setperm(build_path, \ Privilege.JUDGE_UID, cache_gid, umask=0o750) with StackContext(Privilege.fullaccess): os.chmod(build_path, 0o750) StdChal.build_cache_update(self.res_path, cache_hash, cache_gid) print('StdChal %d built checker %x'%(self.chal_id, cache_hash)) StdChal.build_cache_incref(cache_hash) print('StdChal %d started'%self.chal_id) # Create challenge environment. self.chal_path = 'container/standard/home/%d'%self.uniqid with StackContext(Privilege.fileaccess): os.mkdir(self.chal_path, mode=0o771) try: yield self.prefetch() print('StdChal %d prefetched'%self.chal_id) if self.comp_typ in ['g++', 'clang++']: ret, verdict = yield self.comp_cxx() elif self.comp_typ == 'makefile': ret, verdict = yield self.comp_make() elif self.comp_typ == 'python3': ret, verdict = yield self.comp_python() if ret != PyExt.DETECT_NONE: return [(0, 0, STATUS_CE)] * len(self.test_list), verdict print('StdChal %d compiled'%self.chal_id) # Prepare test arguments if self.comp_typ == 'python3': exefile_path = self.chal_path \ + '/compile/__pycache__/test.cpython-34.pyc' exe_path = '/usr/bin/python3.4' argv = ['./a.out'] envp = ['HOME=/', 'LANG=en_US.UTF-8'] else: exefile_path = self.chal_path + '/compile/a.out' exe_path = './a.out' argv = [] envp = [] # Prepare judge test_future = [] if self.judge_typ == 'diff': for test in self.test_list: test_future.append(self.judge_diff( exefile_path, exe_path, argv, envp, test['in'], test['ans'], test['timelimit'], test['memlimit'])) elif self.judge_typ == 'ioredir': for test in self.test_list: check_uid, _ = StdChal.get_standard_ugid() test_uid, test_gid = StdChal.get_restrict_ugid() test_future.append(judge_ioredir.judge( \ exefile_path, exe_path, argv, envp, \ (check_uid, cache_gid), \ (test_uid, test_gid), \ '/home/%d/run_%d'%(self.uniqid, test_uid), \ test, self.metadata)) # Emit tests test_result = yield gen.multi(test_future) ret_result = list() for result in test_result: test_pass, data = result runtime, peakmem, error = data status = STATUS_ERR if error == PyExt.DETECT_NONE: if test_pass is True: status = STATUS_AC else: status = STATUS_WA elif error == PyExt.DETECT_OOM: status = STATUS_MLE elif error == PyExt.DETECT_TIMEOUT \ or error == PyExt.DETECT_FORCETIMEOUT: status = STATUS_TLE elif error == PyExt.DETECT_EXITERR: status = STATUS_RE else: status = STATUS_ERR ret_result.append((runtime, peakmem, status)) return ret_result, '' finally: if cache_hash is not None: StdChal.build_cache_decref(cache_hash) with StackContext(Privilege.fileaccess): shutil.rmtree(self.chal_path) print('StdChal %d done'%self.chal_id)
def test_rate_limit_stepping(self): """ Test that rate limiting steps delay up and down. """ # The delay will step from delay_min to delay_max by doubling. # 0s -> 0.05s -> 0.10s -> 0.20s. # When a call succeeds, the delay goes back down a step. def _throttle_twice(result): # This will raise two throttling exception, then succeed. # Because of that, this should end with the delay going up # one step. return self.api_call_queue.call( self._mock_api_function_sync, result=result, exception=[ self.boto2_throttle_exception_1, self.boto3_throttle_exception]) api_call_queue_calls = [ _throttle_twice(result=1), ] start = time.time() results = yield gen.multi(api_call_queue_calls) stop = time.time() run_time = stop - start # Delay should be up one step total: delay_min. self.assertEqual( self.api_call_queue.delay, self.api_call_queue.delay_min) self.assertTrue(0.15 <= run_time < 0.25) self.assertEqual(results, [1]) # Do it again, to go up one more step. api_call_queue_calls = [ _throttle_twice(result=2), ] start = time.time() results = yield gen.multi(api_call_queue_calls) stop = time.time() run_time = stop - start # Delay should be up two steps total: delay_min * 2. self.assertEqual( self.api_call_queue.delay, self.api_call_queue.delay_min * 2) self.assertTrue(0.35 <= run_time < 0.45) self.assertEqual(results, [2]) # Do it again, to go up one more step. api_call_queue_calls = [ _throttle_twice(result=3), ] start = time.time() results = yield gen.multi(api_call_queue_calls) stop = time.time() run_time = stop - start # Delay should be up to delay_min * 2 again, # because it always goes down once it succeeds, # but total runtime should be ~0.5s: # (0delay_min * 2 + delay_max + delay_max). self.assertEqual( self.api_call_queue.delay, self.api_call_queue.delay_min * 2) self.assertTrue(0.5 <= run_time < 0.6) self.assertEqual(results, [3]) # Now do it with successes to step back down. api_call_queue_calls = [ self.api_call_queue.call( self._mock_api_function_sync, result=4), self.api_call_queue.call( self._mock_api_function_sync, result=5), self.api_call_queue.call( self._mock_api_function_sync, result=6), ] start = time.time() results = yield gen.multi(api_call_queue_calls) stop = time.time() run_time = stop - start # Delay should be up to 0 again. # Total runtime should be ~0.15s: # (delay_min * 2 + delay_min). self.assertEqual(self.api_call_queue.delay, 0) self.assertTrue(0.15 <= run_time < 0.25) self.assertEqual(results, [4, 5, 6])
def wait_a_moment(): result = yield gen.multi([gen.moment, gen.moment]) raise gen.Return(result)
def handle_user(user): """Handle one user. Create a list of their servers, and async exec them. Wait for that to be done, and if all servers are stopped, possibly cull the user. """ # shutdown servers first. # Hub doesn't allow deleting users with running servers. # jupyterhub 0.9 always provides a 'servers' model. # 0.8 only does this when named servers are enabled. if 'servers' in user: servers = user['servers'] else: # jupyterhub < 0.9 without named servers enabled. # create servers dict with one entry for the default server # from the user model. # only if the server is running. servers = {} if user['server']: servers[''] = { 'last_activity': user['last_activity'], 'pending': user['pending'], 'url': user['server'], } server_futures = [ handle_server(user, server_name, server) for server_name, server in servers.items() ] results = yield multi(server_futures) if not cull_users: return # some servers are still running, cannot cull users still_alive = len(results) - sum(results) if still_alive: app_log.debug( "Not culling user %s with %i servers still alive", user['name'], still_alive) return False should_cull = False if user.get('created'): age = now - parse_date(user['created']) else: # created may be undefined on jupyterhub < 0.9 age = None # check last activity # last_activity can be None in 0.9 if user['last_activity']: inactive = now - parse_date(user['last_activity']) else: # no activity yet, use start date # last_activity may be None with jupyterhub 0.9, # which introduces the 'created' field which is never None inactive = age should_cull = (inactive is not None and inactive.total_seconds() >= inactive_limit) if should_cull: app_log.info( "Culling user %s (inactive for %s)", user['name'], inactive) if max_age and not should_cull: # only check created if max_age is specified # so that we can still be compatible with jupyterhub 0.8 # which doesn't define the 'started' field if age is not None and age.total_seconds() >= max_age: app_log.info( "Culling user %s (age: %s, inactive for %s)", user['name'], format_td(age), format_td(inactive)) should_cull = True if not should_cull: app_log.debug( "Not culling user %s (created: %s, last active: %s)", user['name'], format_td(age), format_td(inactive)) return False req = HTTPRequest( url=url + '/users/%s' % user['name'], method='DELETE', headers=auth_header, ) yield fetch(req) return True