def with_timeout(timeout, future_or_iterable):
""" Yields on a future (or a list of futures) for a maximum amount of time
Returns the results of the completed futures completed after timeout (or None if the future is still pending)
:param timeout: The number of seconds to wait for the future. 0 or None to disable.
:param future_or_iterable: A future to yield on, or a list of futures
:return: The results from the future or the list of futures, or None if the future didn't complete in time
"""
# Waiting for a list
if isinstance(future_or_iterable, list):
futures = future_or_iterable
try:
if timeout:
yield gen.with_timeout(timedelta(seconds=timeout), gen.multi(futures))
else:
yield gen.multi(futures)
except util.TimeoutError:
pass
return [None if not future.done() else future.result() for future in futures]
# Waiting for a single future
try:
if timeout:
yield gen.with_timeout(timedelta(seconds=timeout), future_or_iterable)
else:
yield future_or_iterable
except util.TimeoutError:
pass
return None if not future_or_iterable.done() else future_or_iterable.result()
def close(self, timeout=None):
self.state = CLOSING
self.logger.debug("Flushing send queue")
drain_futures = [drain.close() for drain in self.emitter.values()]
if timeout:
yield gen.with_timeout(timeout.total_seconds(),
gen.multi(drain_futures))
else:
yield gen.multi(drain_futures)
def initialize_env():
phases = WorkEnv.Environment.keys()
#scheduler = TornadoScheduler()
for idx, p in enumerate(phases):
defi = WorkEnv.Environment[p]
ini = {}
ini["phase"] = idx + 1
fp = open(ApplicationProperties.configure(
"application.storage.filesystem.uc.rootdir") +
"/temp_data/phase" + str(ini["phase"]),
'w+',
encoding="utf-8")
if "pagination" in defi:
bridgeq = queues.Queue()
ini["pagination"] = defi["pagination"]
ini["selectors"] = defi["selectors"]
interim = ini["pagination"]["interim"]
print("gen pworkers")
pworkers = gen.multi([
pagination(
ini["pagination"], defi["srcq"], bridgeq, "phase[" +
str(ini["phase"]) + "]-pagination[" + str(_) + "]")
for _ in range(p_concurrency)
])
#print(pworkers)
print(defi["srcq"])
print(bridgeq)
#scheduler.add_job(pagination,'interval', seconds=interim, args=(ini, defi["srcq"], bridgeq))
cworkers = gen.multi([
worker(
Crawler(ini), bridgeq, defi["distq"], "phase[" +
str(ini["phase"]) + "]-crawler[" + str(_) + "]", fp)
for _ in range(c_concurrency)
])
print("gen cworkers")
print(cworkers)
print(bridgeq)
print(defi["distq"])
else:
ini["selectors"] = defi["selectors"]
dq = None
if "distq" in defi:
dq = defi["distq"]
print(defi["distq"])
cworkers = gen.multi([
worker(
Crawler(ini), defi["srcq"], dq, "phase[" +
str(ini["phase"]) + "]-crawler[" + str(_) + "]", fp)
for _ in range(c_concurrency)
])
print("gen cworkers")
print(cworkers)
print(defi["srcq"])
def close(self, timeout=None):
self.state = CLOSING
self.logger.debug("Flushing send queue")
drain_futures = [
drain.close()
for drain in self.emitter.values()
]
if timeout:
yield gen.with_timeout(timeout.total_seconds(),
gen.multi(drain_futures))
else:
yield gen.multi(drain_futures)
def run(self):
"""Starts the clients serial aggregation workflow."""
# Connect all clients
yield gen.multi([client.connect() for client in self.clients.values()])
# Start stdin listener
self._listen_stdin()
# Start listening all opened client connections
yield gen.multi([
client.listen() for client in self.clients.values()
if client.websocket is not None
])
tornado.ioloop.IOLoop.instance().stop()
def assemble_fresh_response(self):
yield self.__class__._lock.acquire()
self.logger.log(logging.INFO, "Fetching latest release from GitHub")
data = None
failure = True
try:
data = yield gen.multi({
"stats": assemble_stats(),
"flavors": assemble_flavors()
})
failure = False
except tornado.httpclient.HTTPError as error:
response = error.response
self.logger.log(
logging.ERROR,
"API error: {} -> {} ({})".format(response.effective_url, response.error, response.body),
)
except Exception as error:
self.logger.log(logging.ERROR, error)
if failure:
self.__class__._last_failed_request = datetime.now()
self.__class__._lock.release()
self.send_error(500)
return False
self.update_cache(data)
self.__class__._lock.release()
self.add_default_headers()
self.write(data)
self.finish()
async def main():
seen_set = set()
q = queues.Queue()
async def fetch_url(current_url):
# 生产者
if current_url in seen_set:
return
print(f"获取: {current_url}")
seen_set.add(current_url)
next_urls = await get_url_links(current_url)
for new_url in next_urls:
if new_url.startswith(base_url):
# 放入队列, 使用await 是为了当这个协程放不进去的时候切换出来，切换到另一个get()协程上
await q.put(new_url)
async def worker():
async for url in q:
if url is None:
return
try:
await fetch_url(url)
except Exception as e:
print("Exception")
finally:
# 将数据减一 消费了一个
q.task_done()
await q.put(base_url)
workers = gen.multi([worker() for _ in range(concurrency)])
await q.join()
for _ in range(concurrency):
await q.put(None)
await workers
def server_stop():
deinit_futures = [
integration.deinitialize_app(app)
for integration in app.available_integrations
]
if deinit_futures:
def await_deinit(future):
if future.exception() is not None:
log.error('failed to deinit, deinit returned: %s',
future.exception())
io_loop.add_future(gen.multi(deinit_futures), await_deinit)
http_server.stop()
if ioloop_is_running():
log.info('going down in %s seconds', options.stop_timeout)
def ioloop_stop():
if ioloop_is_running():
log.info('stopping IOLoop')
io_loop.stop()
log.info('stopped')
io_loop.add_timeout(time.time() + options.stop_timeout,
ioloop_stop)
def assemble_stats():
repo_url = "https://api.github.com/repos/TeamNewPipe/NewPipe"
contributors_url = "https://github.com/TeamNewPipe/NewPipe"
translations_url = "https://hosted.weblate.org/api/components/newpipe/" \
"strings/translations/"
repo_data, contributors_data, translations_data = \
[x.body for x in (yield gen.multi((
fetch(repo_url),
fetch(contributors_url),
fetch(translations_url),
)))]
repo = json.loads(repo_data)
translations = json.loads(translations_data)
document = html.fromstring(contributors_data)
tags = document.cssselect(".numbers-summary a[href$=contributors] .num")
contributors = int(tags[0].text)
return {
"stargazers": repo["stargazers_count"],
"watchers": repo["subscribers_count"],
"forks": repo["forks_count"],
"contributors": contributors,
"translations": int(translations["count"]),
}
def assemble_flavors():
return gen.multi({
"github": get_github_flavor("NewPipe"),
"fdroid": get_fdroid_flavor("org.schabi.newpipe"),
"github_legacy": get_github_flavor("NewPipe-legacy"),
"fdroid_legacy": get_fdroid_flavor("org.schabi.newpipelegacy"),
})
def handle_user(user):
"""Handle one user.
Create a list of their servers, and async exec them. Wait for
that to be done, and if all servers are stopped, possibly cull
the user.
"""
# shutdown servers first.
# Hub doesn't allow deleting users with running servers.
# jupyterhub 0.9 always provides a 'servers' model.
# 0.8 only does this when named servers are enabled.
if 'servers' in user:
servers = user['servers']
else:
# jupyterhub < 0.9 without named servers enabled.
# create servers dict with one entry for the default server
# from the user model.
# only if the server is running.
servers = {}
if user['server']:
servers[''] = {
'last_activity': user['last_activity'],
'pending': user['pending'],
'url': user['server'],
}
server_futures = [
handle_server(user, server_name, server)
for server_name, server in servers.items()
]
results = yield multi(server_futures)
def _test():
url = app.get_url(uri)
coros = (app.http_client.fetch(url) for _ in range(2))
responses = app.io_loop.run_sync(lambda: gen.multi(coros))
for response in responses:
assert response.code == 200
def Main(p_command=None):
print('PyTask loaded!')
properties = ApplicationProperties(p_command)
#dlist = dashboard.populate() + [ (RequestMapping.submit_fetch_url_task, MainHandler)]
application = tornado.web.Application([ (RequestMapping.submit_fetch_url_task, MainHandler),
(RequestMapping.get_current_fetch_worker_queue_num, dashboard), ])
application.listen(properties.getProperty(p_key='server.port'))
workers = gen.multi([worker(idx) for idx in range(int(properties.getProperty(p_key='application.worker.fetchUrl.num')))])
ioLoop = tornado.ioloop.IOLoop.current()
partitiondir = properties.getProperty(p_key='application.persistence.fetchdir') + '/' + properties.getProperty(p_key='application.persistence.defaultPartition')
if( os.path.exists( partitiondir ) ):
for vol in os.listdir( partitiondir ):
if vol.lower().startswith('volumn'):
vols.append(partitiondir+"/"+vol)
print('%d volumns in %s'%(len(vols), properties.getProperty(p_key='application.persistence.defaultPartition') ))
#request = HttpRequest()
#await request.do()
#ApplicationContext.getContext().putInstance( p_name='var.application.inq', p_obj=inq )
ApplicationContext.getContext().putInstance( p_name='var.application.workers', p_obj=workers )
ApplicationContext.getContext().putInstance( p_name='var.application.configuration', p_obj=properties )
ApplicationContext.getContext().putInstance( p_name='var.application.webApplication', p_obj=application )
#ApplicationContext.getContext().putInstance( p_name='var.application.urlRequest', p_obj=request )
ApplicationContext.getContext().putInstance( p_name='var.ioloop.current', p_obj=ioLoop )
ioLoop.start()
def test_rate_limiting_boto3(self):
"""
Test that rate limiting with boto3 works.
"""
# Each one of these will raise a throttling exception, then succeed.
# Between calls, each should delay for 1 cycle of `delay_min`.
# Delay min is 0.05s, so total runtime should be ~0.15s.
api_call_queue_calls = [
self.api_call_queue.call(
self._mock_api_function_sync,
result=1,
exception=[self.boto3_throttle_exception]),
self.api_call_queue.call(
self._mock_api_function_sync,
result=2,
exception=[self.boto3_throttle_exception]),
self.api_call_queue.call(
self._mock_api_function_sync,
result=3,
exception=[self.boto3_throttle_exception]),
]
start = time.time()
results = yield gen.multi(api_call_queue_calls)
stop = time.time()
run_time = stop - start
self.assertTrue(0.15 <= run_time < 0.25)
self.assertEqual(results, [1, 2, 3])
def handle(self):
query = self.get_argument('query', None)
if not query:
ret = {}
ret["status"] = "missing params"
serverlg.info(
'[chatbot] [ERROR: missing params] [REQUEST] [%s] [%s]' %
(time.strftime('%Y-%m-%d %H:%M:%S'), self.request.uri))
self.write(json.dumps(ret, ensure_ascii=False))
self.finish()
results = []
debug_infos = []
graph_stubs = [
model_conf["graph_stub"] for model_conf in schedule["servables"]
]
# Multi model compatible, but here just one model exists
multi_models = []
for graph, stub in graph_stubs:
multi_models.append(
self.run_model(graph, stub, [query.encode("utf-8")]))
outs = yield multi(multi_models)
serverlg.info(outs)
raise gen.Return(None)
def test_concurrent_calls_with_delay(self):
"""
Test concurrent calls with some latency run serially
and return independent results.
The api_call_queue runs calls synchronously and serially.
"""
api_call_queue_calls = [
self.api_call_queue.call(
self._mock_api_function_sync, result=1, delay=0.05),
self.api_call_queue.call(
self._mock_api_function_sync, result=2, delay=0.05),
self.api_call_queue.call(
self._mock_api_function_sync, result=3, delay=0.05),
self.api_call_queue.call(
self._mock_api_function_sync, result=4, delay=0.05),
self.api_call_queue.call(
self._mock_api_function_sync, result=5, delay=0.05),
]
start = time.time()
results = yield gen.multi(api_call_queue_calls)
stop = time.time()
run_time = stop - start
self.assertTrue(0.25 <= run_time < 0.35)
self.assertEqual(results, [1, 2, 3, 4, 5])
def test_rate_limiting_boto3(self):
"""
Test that rate limiting with boto3 works.
"""
# Each one of these will raise a throttling exception, then succeed.
# Between calls, each should delay for 1 cycle of `delay_min`.
# Delay min is 0.05s, so total runtime should be ~0.15s.
api_call_queue_calls = [
self.api_call_queue.call(self._mock_api_function_sync,
result=1,
exception=[self.boto3_throttle_exception
]),
self.api_call_queue.call(self._mock_api_function_sync,
result=2,
exception=[self.boto3_throttle_exception
]),
self.api_call_queue.call(self._mock_api_function_sync,
result=3,
exception=[self.boto3_throttle_exception
]),
]
start = time.time()
results = yield gen.multi(api_call_queue_calls)
stop = time.time()
run_time = stop - start
self.assertTrue(0.15 <= run_time < 0.25)
self.assertEqual(results, [1, 2, 3])
def test_concurrent_calls_with_delay(self):
"""
Test concurrent calls with some latency run serially
and return independent results.
The api_call_queue runs calls synchronously and serially.
"""
api_call_queue_calls = [
self.api_call_queue.call(self._mock_api_function_sync,
result=1,
delay=0.05),
self.api_call_queue.call(self._mock_api_function_sync,
result=2,
delay=0.05),
self.api_call_queue.call(self._mock_api_function_sync,
result=3,
delay=0.05),
self.api_call_queue.call(self._mock_api_function_sync,
result=4,
delay=0.05),
self.api_call_queue.call(self._mock_api_function_sync,
result=5,
delay=0.05),
]
start = time.time()
results = yield gen.multi(api_call_queue_calls)
stop = time.time()
run_time = stop - start
self.assertTrue(0.25 <= run_time < 0.35)
self.assertEqual(results, [1, 2, 3, 4, 5])
def get(self, jwt, msg_id):
user = yield self.authenticate_user(jwt)
if user:
setattr(self, 'user', user)
if self.user:
cursor_check = yield db.execute(
QUERIES.CHECK_USER_ACCESS_TO_MSG_FILES,
(self.user.id, self.user.id, msg_id))
if cursor_check.fetchone()[0] == 1:
cursor_streamed_files = yield db.execute(
QUERIES.GET_STREAMED_FILES_INFO, (msg_id, ))
files = []
s3_files = []
for file_ in ResultIter(cursor_streamed_files, 500):
files.append(file_[1])
real_aws_url = s3_storage._generate_url(file_[0])
s3_files.append(
httpclient.AsyncHTTPClient().fetch(real_aws_url))
s3_raw_data = yield gen.multi(s3_files)
print s3_raw_data, files
in_memory_zip = StringIO()
zip_file = ZipFile(in_memory_zip, 'w', ZIP_DEFLATED)
for i, file_ in enumerate(files):
zip_file.writestr(file_, s3_raw_data[i].buffer.getvalue())
zip_file.close()
in_memory_zip.seek(0)
zip_content = in_memory_zip.getvalue()
self.set_header('Content-Type', 'application/zip')
self.set_header('Content-Length', len(zip_content))
self.set_header(
'Content-Disposition', 'attachment; filename=%s' %
'{0}.zip'.format(str(uuid.uuid4())))
self.flush()
self.write(zip_content)
self.finish()
def start(self):
global __g_userSpace
global __g_memDB
__g_memDB = MemDB()
__g_userSpace = UserSpace(__g_memDB)
application = tornado.web.Application([
(RequestMapping.register_node, RegisterHandler),
(RequestMapping.fetch_dashboard, DashBoardHandler),
])
application.listen(
ApplicationProperties.configure(
p_key='application.controller.register.server.port'))
print('Application listeming...[Port:%s]' %
(ApplicationProperties.configure(
p_key='application.controller.register.server.port')))
workers = gen.multi([worker(idx) for idx in range(1)])
__g_userSpace.init()
ioLoop = tornado.ioloop.IOLoop.current()
#print(ioLoop)
ioLoop.start()
def get(self):
client = httpclient.AsyncHTTPClient()
futures = []
for provider in HOTEL_PROVIDERS:
futures.append(
client.fetch(
"http://127.0.0.1:9000/scrapers/{}".format(provider)))
responses = yield gen.multi(futures)
results = []
for response in responses:
# Put each hotel into a 2-tuple where the first value is it's ecstasy
# and the second is the full hotel data
# Sort by smalles ecstasy to greatest to use pythons merge heapq.merge
results.append(
list(
reversed([(hotel['ecstasy'], hotel)
for hotel in json.loads(response.body)['results']
])))
# Merge all the lists of results together and reverse the result so largest is first
sorted_hotels = [
hotel[1] for hotel in reversed(list(heapq.merge(*results)))
]
self.write({'results': sorted_hotels})
async def main():
testing = False
if testing:
await tests()
else:
control_server_ip_address = '127.0.0.1'
control_server_port_number = '8080'
url_prefix = '/'.join(['http:', '', ':'.join([control_server_ip_address, control_server_port_number])])
user_home_directory = '/home/dannya951'
max_subprocesses = 5
network = 'testnet'
clients_working = True
debug = True
request_utility = Aru.AsyncRequestUtility(control_server_ip_address, control_server_port_number)
post_clients_code = await request_utility.post_clients()
if post_clients_code is None:
print('Failed to create new clients.')
clients_working = False
while clients_working:
intervals = await request_utility.get_client()
interval_subprocesses = len(intervals)
interval_subprocesses = min(interval_subprocesses, max_subprocesses)
if interval_subprocesses == 0:
# server is done allocating blocks, break from while loop
clients_working = False
else:
gztar_name = 'blockchain'
await initialize_all_local_blockchains(
request_utility,
user_home_directory,
gztar_name,
interval_subprocesses
)
for interval in intervals:
await request_utility.post_results({interval: {}})
connections = await Anm.async_identify_connections()
await Anm.async_manage_connections(connections, 'disconnect', debug=debug)
if debug:
print('Starting ' + str(interval_subprocesses) + ' client subprocesses.')
interval_names = sorted([ik for ik in intervals.keys()])
clients = gen.multi([run_client(subprocess_index,
user_home_directory,
intervals,
network=network)
for subprocess_index in range(interval_subprocesses)])
await clients
await Anm.async_manage_connections(connections, 'connect', debug=debug)
for subprocess_index in range(interval_subprocesses):
interval_name = interval_names[subprocess_index]
await request_utility.put_client({interval_name: intervals[interval_name]})
await request_utility.put_result({interval_name: intervals[interval_name]})
async def async_call(self, args=None, timeout=None):
"""
Calls system command and return parsed output or standard error output
"""
if args is None:
args = []
# Executing command with Tornado subprocess is possible only in main thread
if threading.main_thread().ident != threading.get_ident():
return self.call(args=args, timeout=timeout)
all_args = [
self.CMD if self.CMD is not None else cfg['tools.%s.cmd' %
self.NAME]
]
all_args.extend(self.COMMON_ARGS)
all_args.extend(args)
cmd = ' '.join(all_args),
log.debug('Executing: %s', cmd)
if self._cancelled:
raise Exception('Task was cancelled')
task = process.Subprocess(all_args,
stderr=process.Subprocess.STREAM,
stdout=process.Subprocess.STREAM)
self.proc = task.proc
coroutine = gen.multi([
task.wait_for_exit(raise_error=False),
task.stdout.read_until_close(),
task.stderr.read_until_close()
])
if not timeout:
return_code, stdout, stderr = await coroutine
else:
try:
return_code, stdout, stderr = await gen.with_timeout(
timedelta(seconds=timeout), coroutine)
except gen.TimeoutError as exception:
log.exception(
"Command %s timed out after %s while executing %s",
self.NAME, timeout, cmd)
task.proc.kill()
raise exception
self.proc = None
if return_code != 0:
log.warning("Command '%s' failed wit exit code: %s", cmd,
return_code)
log.debug("Command '%s':\nSTDOUT:\n%s\nSTDERR:\n%s", cmd, stdout,
stderr)
if self.RAISE_ERROR:
raise subprocess.CalledProcessError(return_code, cmd)
return self.parser.parse(stdout.decode('utf-8'),
stderr.decode('utf-8'))
def wait(futures):
"""Gather multiply futures into one future
Returns:
future: New future wait all child futures
the result is a list contains the results of all child futures
"""
return multi(set(futures))
def _loop_read(self):
for line in chain.from_iterable(self.config.inputs):
if self._stop:
break
line = line.strip()
yield self._queue.put(line)
yield gen.multi(
[self._queue.put(None) for _ in range(0, self.config.concurrency)])
def test_init_close_race(self):
# Regression test for #2367
def f():
for i in range(10):
loop = IOLoop()
loop.close()
yield gen.multi([self.io_loop.run_in_executor(None, f) for i in range(2)])
def test_init_close_race(self):
# Regression test for #2367
def f():
for i in range(10):
loop = IOLoop()
loop.close()
yield gen.multi([self.io_loop.run_in_executor(None, f) for i in range(2)])
def _get_collection_png_tile(self, fc, x, y, z):
rasters = yield gen.multi([
self._get_raster_png_tile(f.raster(), x, y, z) for f in fc
]) # type: ignore
if len(rasters) < 1:
return None
tile = yield self._merge_rasters(rasters, z)
return tile
def main():
args = parser.parse_args()
r_host, r_port = args.redis.split(':')
storage = RedisStorage(r_host, int(r_port), args.queue_timeout)
yield storage.url_discovered(args.url)
yield gen.multi([Crawler(storage).run() for i in range(args.workers)])
print 'All website crawled.'
yield print_sitemap(storage)
def _stop_watchers(self, close_output_streams=False,
watcher_iter_func=None, for_shutdown=False):
if watcher_iter_func is None:
watchers = self.iter_watchers(reverse=False)
else:
watchers = watcher_iter_func(reverse=False)
futures = [w._stop(close_output_streams, for_shutdown)
for w in watchers]
yield gen.multi(futures)
def wait(tasks):
"""
Simple helper for gathering all passed :class:`Task`s.
:param tasks: list of the :class:`asyncio.Task`s
:param _loop: Event loop (:func:`tornado.ioloop.IOLoop.current()` when :class:`None`)
:return: :class:`tuple` of results
"""
raise gen.Return((yield gen.multi(tasks)))
def run_tasks():
tasks = [
fetch1('http://example.com'),
fetch2('http://example.com'),
fetch3('http://example.com'),
fetch4('http://example.com'),
]
for r in (yield gen.multi(tasks)):
print(r)
print('done')
def invoke():
# important to start the sleep before starting callback so any initial
# time spent in callback "counts against" the period.
sleep_future = self.sleep()
result = self._func()
if result is None:
return sleep_future
callback_future = gen.convert_yielded(result)
return gen.multi([sleep_future, callback_future])
def _run(self, args):
try:
IOLoop.current().spawn_callback(self._loop_read)
yield gen.multi([
self._loop_process(args)
for _ in range(0, self.config.concurrency)
])
except Exception as e:
self._logger.error('Error {}'.format(e))
finally:
yield self._close()
def invoke():
# important to start the sleep before starting callback
# so any initial time spent in callback "counts against"
# the period.
sleep_future = self.sleep()
result = self._func()
try:
callback_future = gen.convert_yielded(result)
except gen.BadYieldError:
# result is not a yieldable thing
return sleep_future
else:
return gen.multi([sleep_future, callback_future])
def _run_preprocessors(self, preprocessors):
for p in preprocessors:
yield gen.coroutine(p)(self)
if self._finished:
self.log.info('page was already finished, breaking preprocessors chain')
return False
yield gen.multi(self._preprocessor_futures)
self._preprocessor_futures = None
if self._finished:
self.log.info('page was already finished, breaking preprocessors chain')
return False
return True
def initialize_app(self, app) -> Optional[Future]:
def get_kafka_producer(producer_name: str) -> AIOKafkaProducer:
return self.kafka_producers.get(producer_name)
app.get_kafka_producer = get_kafka_producer
if options.kafka_clusters:
init_futures = []
for cluster_name, cluster_settings in options.kafka_clusters.items():
if cluster_settings:
producer = AIOKafkaProducer(loop=asyncio.get_event_loop(), **cluster_settings)
self.kafka_producers[cluster_name] = producer
init_futures.append(asyncio.ensure_future(producer.start()))
return gen.multi(init_futures)
return None
async def main():
q = queues.Queue()
start = time.time()
fetching, fetched = set(), set()
async def fetch_url(current_url):
if current_url in fetching:
return
print('fetching %s' % current_url)
fetching.add(current_url)
urls = await get_links_from_url(current_url)
fetched.add(current_url)
for new_url in urls:
# Only follow links beneath the base URL
if new_url.startswith(base_url):
await q.put(new_url)
async def worker():
async for url in q:
if url is None:
return
try:
await fetch_url(url)
except Exception as e:
print('Exception: %s %s' % (e, url))
finally:
q.task_done()
await q.put(base_url)
# Start workers, then wait for the work queue to be empty.
workers = gen.multi([worker() for _ in range(concurrency)])
await q.join(timeout=timedelta(seconds=300))
assert fetching == fetched
print('Done in %d seconds, fetched %s URLs.' % (
time.time() - start, len(fetched)))
# Signal all the workers to exit.
for _ in range(concurrency):
await q.put(None)
await workers
def invoke():
# important to start the sleep before starting callback
# so any initial time spent in callback "counts against"
# the period.
sleep_future = self.sleep()
result = self._func()
# This is needed for Tornado >= 4.5 where convert_yielded will no
# longer raise BadYieldError on None
if result is None:
return sleep_future
try:
callback_future = gen.convert_yielded(result)
except gen.BadYieldError:
# result is not a yieldable thing
return sleep_future
else:
return gen.multi([sleep_future, callback_future])
def _async_init_cb():
try:
init_futures = app.default_init_futures + list(app.init_async())
if init_futures:
def await_init(future):
if future.exception() is not None:
log.error('failed to initialize application, init_async returned: %s', future.exception())
sys.exit(1)
run_server(app)
ioloop.add_future(gen.multi(init_futures), await_init)
else:
run_server(app)
except Exception:
log.exception('failed to initialize application')
sys.exit(1)
def test_api_call_queue_future_is_nonblocking(self):
"""
Test that the api call queue future is nonblocking for other futures.
It needs to execute in a different thread because of the synchronous
nature of AWS APIs.
"""
futures = [
self.api_call_queue.call(
self._mock_api_function_sync, result=1, delay=0.05),
self._mock_api_function_async(result=2, delay=0.05),
self._mock_api_function_async(result=3, delay=0.05),
self._mock_api_function_async(result=4, delay=0.05),
self._mock_api_function_async(result=5, delay=0.05),
]
start = time.time()
results = yield gen.multi(futures)
stop = time.time()
run_time = stop - start
self.assertTrue(0.05 <= run_time < 0.15)
self.assertEqual(results, [1, 2, 3, 4, 5])
def test_api_call_queue_raises_exceptions(self):
"""
Test that the api call queue raises exceptions and proceeds to execute
other queued api calls.
"""
@gen.coroutine
def _call_without_exception():
result = yield self.api_call_queue.call(
self._mock_api_function_sync, delay=0.05)
self.assertEqual(result, 'OK')
raise gen.Return('no exception')
@gen.coroutine
def _call_with_exception():
with self.assertRaises(ValueError):
yield self.api_call_queue.call(
self._mock_api_function_sync,
exception=ValueError('test exception'),
delay=0.05)
raise gen.Return('exception')
@gen.coroutine
def _call_with_exception_after_boto2_rate_limit():
"""
First rate limit, then raise an exception.
This should take:
call delay * 2 + min rate limiting delay * 1
"""
with self.assertRaises(boto_exception.BotoServerError):
yield self.api_call_queue.call(
self._mock_api_function_sync,
exception=[
self.boto2_throttle_exception_1,
self.boto2_exception],
delay=0.05)
raise gen.Return('exception')
@gen.coroutine
def _call_with_exception_after_boto3_rate_limit():
"""
First rate limit, then raise an exception.
This should take:
call delay * 2 + min rate limiting delay * 1
"""
with self.assertRaises(botocore_exceptions.ClientError):
yield self.api_call_queue.call(
self._mock_api_function_sync,
exception=[
self.boto3_throttle_exception,
self.boto3_exception],
delay=0.05)
raise gen.Return('exception')
call_wrappers = [
# Should take 0.05s.
_call_without_exception(),
# Should take 0.05s.
_call_with_exception(),
# Should take 0.05s.
_call_without_exception(),
# Should take 0.05s + 0.05s + 0.05s.
_call_with_exception_after_boto2_rate_limit(),
# Should take 0.05s + 0.05s + 0.05s.
_call_with_exception_after_boto3_rate_limit(),
]
start = time.time()
results = yield gen.multi(call_wrappers)
stop = time.time()
run_time = stop - start
self.assertTrue(0.45 <= run_time < 0.55)
self.assertEqual(
results,
['no exception', 'exception', 'no exception',
'exception', 'exception'])
def start(self):
'''Start the challenge.
Returns:
dict: Challenge result.
'''
print('StdChal %d started'%self.chal_id)
self.chal_path = 'container/standard/home/%d'%self.uniqid
with StackContext(Privilege.fileaccess):
os.mkdir(self.chal_path, mode=0o771)
yield self.prefetch()
print('StdChal %d prefetched'%self.chal_id)
if self.comp_typ in ['g++', 'clang++']:
ret = yield self.comp_cxx()
elif self.comp_typ == 'makefile':
ret = yield self.comp_make()
elif self.comp_typ == 'python3':
ret = yield self.comp_python()
if ret != PyExt.DETECT_NONE:
ret_result = [(0, 0, STATUS_CE)] * len(self.test_list)
else:
print('StdChal %d compiled'%self.chal_id)
if self.comp_typ == 'python3':
exefile_path = self.chal_path \
+ '/compile/__pycache__/test.cpython-34.pyc'
exe_path = '/usr/bin/python3.4'
argv = ['./a.out']
envp = ['HOME=/', 'LANG=en_US.UTF-8']
else:
exefile_path = self.chal_path + '/compile/a.out'
exe_path = './a.out'
argv = []
envp = []
test_future = []
for test in self.test_list:
test_future.append(self.judge_diff(
exefile_path,
exe_path, argv, envp,
test['in'], test['ans'],
test['timelimit'], test['memlimit']))
test_result = yield gen.multi(test_future)
ret_result = list()
for result in test_result:
test_pass, data = result
runtime, peakmem, error = data
status = STATUS_ERR
if error == PyExt.DETECT_NONE:
if test_pass is True:
status = STATUS_AC
else:
status = STATUS_WA
elif error == PyExt.DETECT_OOM:
status = STATUS_MLE
elif error == PyExt.DETECT_TIMEOUT \
or error == PyExt.DETECT_FORCETIMEOUT:
status = STATUS_TLE
elif error == PyExt.DETECT_EXITERR:
status = STATUS_RE
else:
status = STATUS_ERR
ret_result.append((runtime, peakmem, status))
with StackContext(Privilege.fileaccess):
shutil.rmtree(self.chal_path)
print('StdChal %d done'%self.chal_id)
return ret_result
def start(self):
'''Start the challenge.
Returns:
dict: Challenge result.
'''
cache_hash = None
cache_gid = None
# Check if special judge needs to rebuild.
if self.judge_typ in ['ioredir']:
hashproc = process.Subprocess( \
['./HashDir.py', self.res_path + '/check'], \
stdout=process.Subprocess.STREAM)
dirhash = yield hashproc.stdout.read_until(b'\n')
dirhash = int(dirhash.decode('utf-8').rstrip('\n'), 16)
ret = StdChal.build_cache_find(self.res_path)
if ret is not None and ret[0] == dirhash:
cache_hash, cache_gid = ret
judge_ioredir = IORedirJudge('container/standard', \
'/cache/%x'%cache_hash)
else:
cache_hash = dirhash
_, cache_gid = StdChal.get_standard_ugid()
build_ugid = StdChal.get_standard_ugid()
build_relpath = '/cache/%x'%cache_hash
build_path = 'container/standard' + build_relpath
judge_ioredir = IORedirJudge('container/standard', \
build_relpath)
if not (yield judge_ioredir.build(build_ugid, self.res_path)):
return [(0, 0, STATUS_ERR)] * len(self.test_list)
FileUtils.setperm(build_path, \
Privilege.JUDGE_UID, cache_gid, umask=0o750)
with StackContext(Privilege.fullaccess):
os.chmod(build_path, 0o750)
StdChal.build_cache_update(self.res_path, cache_hash, cache_gid)
print('StdChal %d built checker %x'%(self.chal_id, cache_hash))
StdChal.build_cache_incref(cache_hash)
print('StdChal %d started'%self.chal_id)
# Create challenge environment.
self.chal_path = 'container/standard/home/%d'%self.uniqid
with StackContext(Privilege.fileaccess):
os.mkdir(self.chal_path, mode=0o771)
try:
yield self.prefetch()
print('StdChal %d prefetched'%self.chal_id)
if self.comp_typ in ['g++', 'clang++']:
ret, verdict = yield self.comp_cxx()
elif self.comp_typ == 'makefile':
ret, verdict = yield self.comp_make()
elif self.comp_typ == 'python3':
ret, verdict = yield self.comp_python()
if ret != PyExt.DETECT_NONE:
return [(0, 0, STATUS_CE)] * len(self.test_list), verdict
print('StdChal %d compiled'%self.chal_id)
# Prepare test arguments
if self.comp_typ == 'python3':
exefile_path = self.chal_path \
+ '/compile/__pycache__/test.cpython-34.pyc'
exe_path = '/usr/bin/python3.4'
argv = ['./a.out']
envp = ['HOME=/', 'LANG=en_US.UTF-8']
else:
exefile_path = self.chal_path + '/compile/a.out'
exe_path = './a.out'
argv = []
envp = []
# Prepare judge
test_future = []
if self.judge_typ == 'diff':
for test in self.test_list:
test_future.append(self.judge_diff(
exefile_path,
exe_path, argv, envp,
test['in'], test['ans'],
test['timelimit'], test['memlimit']))
elif self.judge_typ == 'ioredir':
for test in self.test_list:
check_uid, _ = StdChal.get_standard_ugid()
test_uid, test_gid = StdChal.get_restrict_ugid()
test_future.append(judge_ioredir.judge( \
exefile_path, exe_path, argv, envp, \
(check_uid, cache_gid), \
(test_uid, test_gid), \
'/home/%d/run_%d'%(self.uniqid, test_uid), \
test, self.metadata))
# Emit tests
test_result = yield gen.multi(test_future)
ret_result = list()
for result in test_result:
test_pass, data = result
runtime, peakmem, error = data
status = STATUS_ERR
if error == PyExt.DETECT_NONE:
if test_pass is True:
status = STATUS_AC
else:
status = STATUS_WA
elif error == PyExt.DETECT_OOM:
status = STATUS_MLE
elif error == PyExt.DETECT_TIMEOUT \
or error == PyExt.DETECT_FORCETIMEOUT:
status = STATUS_TLE
elif error == PyExt.DETECT_EXITERR:
status = STATUS_RE
else:
status = STATUS_ERR
ret_result.append((runtime, peakmem, status))
return ret_result, ''
finally:
if cache_hash is not None:
StdChal.build_cache_decref(cache_hash)
with StackContext(Privilege.fileaccess):
shutil.rmtree(self.chal_path)
print('StdChal %d done'%self.chal_id)
def test_rate_limit_stepping(self):
"""
Test that rate limiting steps delay up and down.
"""
# The delay will step from delay_min to delay_max by doubling.
# 0s -> 0.05s -> 0.10s -> 0.20s.
# When a call succeeds, the delay goes back down a step.
def _throttle_twice(result):
# This will raise two throttling exception, then succeed.
# Because of that, this should end with the delay going up
# one step.
return self.api_call_queue.call(
self._mock_api_function_sync,
result=result,
exception=[
self.boto2_throttle_exception_1,
self.boto3_throttle_exception])
api_call_queue_calls = [
_throttle_twice(result=1),
]
start = time.time()
results = yield gen.multi(api_call_queue_calls)
stop = time.time()
run_time = stop - start
# Delay should be up one step total: delay_min.
self.assertEqual(
self.api_call_queue.delay,
self.api_call_queue.delay_min)
self.assertTrue(0.15 <= run_time < 0.25)
self.assertEqual(results, [1])
# Do it again, to go up one more step.
api_call_queue_calls = [
_throttle_twice(result=2),
]
start = time.time()
results = yield gen.multi(api_call_queue_calls)
stop = time.time()
run_time = stop - start
# Delay should be up two steps total: delay_min * 2.
self.assertEqual(
self.api_call_queue.delay,
self.api_call_queue.delay_min * 2)
self.assertTrue(0.35 <= run_time < 0.45)
self.assertEqual(results, [2])
# Do it again, to go up one more step.
api_call_queue_calls = [
_throttle_twice(result=3),
]
start = time.time()
results = yield gen.multi(api_call_queue_calls)
stop = time.time()
run_time = stop - start
# Delay should be up to delay_min * 2 again,
# because it always goes down once it succeeds,
# but total runtime should be ~0.5s:
# (0delay_min * 2 + delay_max + delay_max).
self.assertEqual(
self.api_call_queue.delay,
self.api_call_queue.delay_min * 2)
self.assertTrue(0.5 <= run_time < 0.6)
self.assertEqual(results, [3])
# Now do it with successes to step back down.
api_call_queue_calls = [
self.api_call_queue.call(
self._mock_api_function_sync, result=4),
self.api_call_queue.call(
self._mock_api_function_sync, result=5),
self.api_call_queue.call(
self._mock_api_function_sync, result=6),
]
start = time.time()
results = yield gen.multi(api_call_queue_calls)
stop = time.time()
run_time = stop - start
# Delay should be up to 0 again.
# Total runtime should be ~0.15s:
# (delay_min * 2 + delay_min).
self.assertEqual(self.api_call_queue.delay, 0)
self.assertTrue(0.15 <= run_time < 0.25)
self.assertEqual(results, [4, 5, 6])
def wait_a_moment():
result = yield gen.multi([gen.moment, gen.moment])
raise gen.Return(result)
def handle_user(user):
"""Handle one user.
Create a list of their servers, and async exec them. Wait for
that to be done, and if all servers are stopped, possibly cull
the user.
"""
# shutdown servers first.
# Hub doesn't allow deleting users with running servers.
# jupyterhub 0.9 always provides a 'servers' model.
# 0.8 only does this when named servers are enabled.
if 'servers' in user:
servers = user['servers']
else:
# jupyterhub < 0.9 without named servers enabled.
# create servers dict with one entry for the default server
# from the user model.
# only if the server is running.
servers = {}
if user['server']:
servers[''] = {
'last_activity': user['last_activity'],
'pending': user['pending'],
'url': user['server'],
}
server_futures = [
handle_server(user, server_name, server)
for server_name, server in servers.items()
]
results = yield multi(server_futures)
if not cull_users:
return
# some servers are still running, cannot cull users
still_alive = len(results) - sum(results)
if still_alive:
app_log.debug(
"Not culling user %s with %i servers still alive",
user['name'], still_alive)
return False
should_cull = False
if user.get('created'):
age = now - parse_date(user['created'])
else:
# created may be undefined on jupyterhub < 0.9
age = None
# check last activity
# last_activity can be None in 0.9
if user['last_activity']:
inactive = now - parse_date(user['last_activity'])
else:
# no activity yet, use start date
# last_activity may be None with jupyterhub 0.9,
# which introduces the 'created' field which is never None
inactive = age
should_cull = (inactive is not None and
inactive.total_seconds() >= inactive_limit)
if should_cull:
app_log.info(
"Culling user %s (inactive for %s)",
user['name'], inactive)
if max_age and not should_cull:
# only check created if max_age is specified
# so that we can still be compatible with jupyterhub 0.8
# which doesn't define the 'started' field
if age is not None and age.total_seconds() >= max_age:
app_log.info(
"Culling user %s (age: %s, inactive for %s)",
user['name'], format_td(age), format_td(inactive))
should_cull = True
if not should_cull:
app_log.debug(
"Not culling user %s (created: %s, last active: %s)",
user['name'], format_td(age), format_td(inactive))
return False
req = HTTPRequest(
url=url + '/users/%s' % user['name'],
method='DELETE',
headers=auth_header,
)
yield fetch(req)
return True
