def main():
try:
tornado.options.parse_command_line()
application = tornado.web.Application([
(r'/', OverviewHandler),
(r'/mogrify', MogrifyHandler),
(r'/query', SingleQueryHandler),
(r'/hstore', HstoreQueryHandler),
(r'/json', JsonQueryHandler),
(r'/transaction', TransactionHandler),
(r'/multi_query', MultiQueryHandler),
(r'/connection', ConnectionQueryHandler),
], debug=True)
ioloop = tornado.ioloop.IOLoop.instance()
application.db = asyncpg.Pool(
dsn=dsn,
size=1,
max_size=3,
ioloop=ioloop,
setsession=("SET TIME ZONE UTC",),
raise_connect_errors=False,
)
# this is a one way to run ioloop in sync
future = application.db.connect()
ioloop.add_future(future, lambda f: ioloop.stop())
ioloop.start()
if enable_hstore:
future = application.db.register_hstore()
# This is the other way to run ioloop in sync
ioloop.run_sync(lambda: future)
if application.db.server_version >= 90200:
future = application.db.register_json()
# This is the other way to run ioloop in sync
ioloop.run_sync(lambda: future)
http_server = tornado.httpserver.HTTPServer(application)
http_server.listen(8888, 'localhost')
ioloop.start()
except KeyboardInterrupt:
print('Exit')
def setup_app(app, extra_configs=None, ioloop=None, listen=None):
if ioloop is None:
ioloop = tornado.ioloop.IOLoop.current()
if extra_configs is None:
extra_configs = []
if isinstance(app, tornado.util.basestring_type):
module_path = app
module_name = 'root'
if ':' in module_path:
module_path, module_name = module_path.split(':', 1)
module_path = module_path.replace('/', '.').strip('.')
module = __import__(module_path, fromlist=[module_name])
module = getattr(module, module_name)
app = rw.httpbase.Application(root=module, extra_configs=extra_configs)
http_server_settings = app.scope['settings'].get('httpserver', {})
server = tornado.httpserver.HTTPServer(app, **http_server_settings)
if listen:
server.listen(*listen)
scope = rw.scope.Scope()
with scope():
ioloop.run_sync(rw.server.start)
return app.scope
def setup_app(app, extra_configs=None, ioloop=None, listen=None):
if ioloop is None:
ioloop = tornado.ioloop.IOLoop.current()
if extra_configs is None:
extra_configs = []
if isinstance(app, tornado.util.basestring_type):
module_path = app
module_name = 'root'
if ':' in module_path:
module_path, module_name = module_path.split(':', 1)
module_path = module_path.replace('/', '.').strip('.')
module = __import__(module_path, fromlist=[module_name])
module = getattr(module, module_name)
app = rw.httpbase.Application(root=module, extra_configs=extra_configs)
http_server_settings = app.scope['settings'].get('httpserver', {})
server = tornado.httpserver.HTTPServer(app, **http_server_settings)
if listen:
server.listen(*listen)
scope = rw.scope.Scope()
with scope():
ioloop.run_sync(rw.server.start)
return app.scope
def exit_gracefully(self):
ioloop = tornado.ioloop.IOLoop.current()
stop_all_modules()
stop_all_schema()
if self.database:
ioloop.run_sync(self.database.shutdown)
ioloop.close()
application_log.info("Maverick API server has stopped")
def start_ioloop(): # pragma: no cover
args = parse_args()
ioloop = tornado.ioloop.IOLoop.current()
if not args.in_port:
ioloop.run_sync(main)
else:
main()
ioloop.start()
def start(self, task):
if isinstance(task, Master):
# 主端启动
task.run()
elif isinstance(task, Slave):
# 从端启动
ioloop = tornado.ioloop.IOLoop.current()
ioloop.run_sync(task.run)
else:
raise Exception('must task class is Master or Slave')
def whois(query, fields=None):
@tornado.gen.coroutine
def _whois():
result = yield whois_async(query, fields)
r.append(result)
ioloop = tornado.ioloop.IOLoop.instance()
r = []
ioloop.run_sync(_whois)
return r[0]
def connect(dsn, *args, **kwds):
pool = Pool(dsn, *args, **kwds)
ioloop = tornado.ioloop.IOLoop.current()
try:
return ioloop.run_sync(pool.connect)
except sanepg.SaneError as e:
raise
def fetch_sync(
url,
method="GET",
headers=None,
body=None,
connect_timeout=DEFAULT_CONNECT_TIMEOUT,
request_timeout=DEFAULT_REQUEST_TIMEOUT,
resolver=resolve,
max_buffer_size=DEFAULT_BUFFER_SIZE,
follow_redirects=False,
max_redirects=DEFAULT_MAX_REDIRECTS,
validate_cert=config.http_client.validate_certs,
allow_proxy=False,
proxies=None,
user=None,
password=None,
content_encoding=None,
eof_mark=None,
):
@tornado.gen.coroutine
def _fetch():
result = yield fetch(
url,
method=method,
headers=headers,
body=body,
connect_timeout=connect_timeout,
request_timeout=request_timeout,
resolver=resolver,
max_buffer_size=max_buffer_size,
follow_redirects=follow_redirects,
max_redirects=max_redirects,
validate_cert=validate_cert,
allow_proxy=allow_proxy,
proxies=proxies,
user=user,
password=password,
content_encoding=content_encoding,
eof_mark=eof_mark,
)
r.append(result)
r = []
ioloop = tornado.ioloop.IOLoop()
ioloop.run_sync(_fetch)
return r[0]
def main():
init_log()
try:
tornado.options.parse_command_line()
application = tornado.web.Application(
[(r'/', OverviewHander), (r'/get-tick-data', TickDataHandler),
(r'/get-new-stock', NewStockHandler),
(r'/get-new-high', NewHighHandler), (r'/query', QueryHandler),
(r'/connection', ConnectionHandler)],
debug=True)
ioloop = tornado.ioloop.IOLoop.instance()
application.db = momoko.Pool(
dsn=dsn,
size=1,
max_size=3,
ioloop=ioloop,
setsession=("SET TIME ZONE UTC", ),
raise_connect_errors=False,
)
# this is a one way to run ioloop in sync
future = application.db.connect()
ioloop.add_future(future, lambda f: ioloop.stop())
ioloop.start()
if enable_hstore:
future = application.db.register_hstore()
# This is the other way to run ioloop in sync
ioloop.run_sync(lambda: future)
if application.db.server_version >= 90200:
future = application.db.register_json()
# This is the other way to run ioloop in sync
ioloop.run_sync(lambda: future)
# create http server
http_server = tornado.httpserver.HTTPServer(application)
http_server.listen(6888, 'localhost')
# create json rpc server
rpc_server = RPCServer()
rpc_server.listen(8001)
ioloop.start()
except KeyboardInterrupt:
print('Exit')
def main():
try:
tornado.options.parse_command_line()
application = tornado.web.Application([
(r'/', OverviewHandler),
(r'/mogrify', MogrifyHandler),
(r'/query', SingleQueryHandler),
(r'/hstore', HstoreQueryHandler),
(r'/json', JsonQueryHandler),
(r'/transaction', TransactionHandler),
(r'/multi_query', MultiQueryHandler),
(r'/connection', ConnectionQueryHandler),
],
debug=True)
ioloop = tornado.ioloop.IOLoop.instance()
application.db = momoko.Pool(
dsn=dsn,
size=1,
max_size=3,
ioloop=ioloop,
setsession=("SET TIME ZONE UTC", ),
raise_connect_errors=False,
)
# this is a one way to run ioloop in sync
future = application.db.connect()
ioloop.add_future(future, lambda f: ioloop.stop())
ioloop.start()
if enable_hstore:
future = application.db.register_hstore()
# This is the other way to run ioloop in sync
ioloop.run_sync(lambda: future)
if application.db.server_version >= 90200:
future = application.db.register_json()
# This is the other way to run ioloop in sync
ioloop.run_sync(lambda: future)
http_server = tornado.httpserver.HTTPServer(application)
http_server.listen(8888, 'localhost')
ioloop.start()
except KeyboardInterrupt:
print('Exit')
def main():
args = get_args()
arg1 = InMemStream("echo")
arg2 = InMemStream()
arg3 = InMemStream()
ioloop = tornado.ioloop.IOLoop.current()
if args.filename == "stdin":
arg3 = PipeStream(sys.stdin.fileno())
send_stream(arg1, arg2, arg3, args.host)
ioloop.start()
elif args.filename:
f = os.open(args.filename, os.O_RDONLY)
arg3 = PipeStream(f)
ioloop.run_sync(lambda: send_stream(arg1, arg2, arg3, args.host))
else:
raise NotImplementedError()
def main():
try:
if len(sys.argv) > 1:
if sys.argv[1] == "config":
ioloop.run_sync(config)
elif sys.argv[1].endswith("help") or sys.argv[1] == "-h":
print(helptext)
else:
print("I don't understand that. Here's the help text:")
print(helptext)
else:
run()
except KeyboardInterrupt:
ioloop.stop()
print("Stopping...")
except Closed:
ioloop.stop()
print("Connection was closed")
def main():
try:
if len(sys.argv) > 1:
if sys.argv[1] == "config":
ioloop.run_sync(config)
elif sys.argv[1].endswith("help") or sys.argv[1] == "-h":
print(helptext)
else:
print("I don't understand that. Here's the help text:")
print(helptext)
else:
run()
except KeyboardInterrupt:
ioloop.stop()
print("Stopping...")
except Closed:
ioloop.stop()
print("Connection was closed")
def get_res_from_sever(dicts, url):
@gen.coroutine
def main(_dic):
http_client = tornado.httpclient.AsyncHTTPClient()
responses = yield [
http_client.fetch(url, method='POST', request_timeout=60.0, body=json.dumps(_dic[i]))
for i in range(len(_dic)) # 请求URL
]
return responses
res = []
asyncio.set_event_loop(asyncio.new_event_loop())
ioloop = tornado.ioloop.IOLoop.current()
# main = partial(main, _dic=dicts)
# res=ioloop.run_sync(main)
for dict in dicts:
if len(dict['sample'])!=0:
main = partial(main, _dic=[dict])
rd=ioloop.run_sync(main)[0]
res.append(eval(rd.body)['res'])
else:
res.append([])
return res
def json_info(self):
self.model.json_info()
def unknown_action(_logger):
def f(logger=_logger):
logger.info("I don't know that action.")
return f
# Main
# ====
if __name__ == "__main__":
# For some reason this gives me a colored log, WTF
# Not actually used though
parse_command_line()
config = get_config()
ow = OverWatch(config)
action = "run" if len(sys.argv) == 1 else sys.argv[1]
try:
f = getattr(ow, action, unknown_action(ow.logger))()
if isinstance(f, types.CoroutineType):
async def await_f(): await f
ioloop.run_sync(await_f)
except KeyboardInterrupt:
ioloop.stop()
ow.logger.info("Stopping because of KeyboardInterrupt")
application.db = momoko.Pool(
dsn='dbname=%s user=%s password=%s host=%s port=%s' % (
options.db_database,
options.db_user,
options.db_password,
options.db_host,
options.db_port),
size=1,
max_size=3,
ioloop=ioloop,
setsession=('SET TIME ZONE UTC',),
raise_connect_errors=False,
)
logging.info('Connecting to PostgreSQL on [%s]' % options.db_host)
ioloop.run_sync(lambda: application.db.connect())
logging.info('Listening on [8888] port')
application.listen(8888)
# prepares the web server
# srv = tornado.httpserver.HTTPServer(application, xheaders=True)
# listens incoming request on port 8000
# srv.bind(8888, '')
# starts the server using 1 process
# srv.start(1)
logging.info('Starting IO loop')
ioloop.start()
def user(args):
ioloop = tornado.ioloop.IOLoop.instance()
scope = rw.cli.setup_app(args.module, ioloop=ioloop)
with scope():
ioloop.run_sync(_user)
sleep_future.add_done_callback(callback)
return task_future
def synchronous():
for i in range(1, 10):
task(i)
@gen.coroutine
def asynchronous():
yield [executor.submit(task, i) for i in range(10)]
@gen.coroutine
def asynchronous_python2():
yield [task_async(i) for i in range(10)]
print('Synchronous:')
synchronous()
print('Asynchronous:')
ioloop = ioloop.IOLoop.current()
ioloop.run_sync(asynchronous)
print('Asynchronous Python2-compatible:')
ioloop.run_sync(asynchronous_python2)
]
rows = [cursor.fetchone() for cursor in cursors]
worlds = [{"id": row[0], "randomNumber": row[1]} for row in rows]
response = json.dumps(worlds)
self.set_header("Content-Type", "application/json; charset=UTF-8")
self.write(response)
application = tornado.web.Application([
(r"/json", JsonSerializeTestHandler), (r"/plaintext", PlaintextHandler),
(r"/db", DBTestHandler), (r"/queries", QueryTestHandler),
(r"/dbraw", QueryPostgresRawTestHandler),
(r"/queriesraw", MultipleQueriesPostgresRawTestHandler)
])
if __name__ == "__main__":
tornado.options.parse_command_line()
server = tornado.httpserver.HTTPServer(application)
server.bind(options.port)
server.start(0)
ioloop = tornado.ioloop.IOLoop.instance()
if options.postgres:
dsn = "user=benchmarkdbuser password=benchmarkdbpass dbname=hello_world host=%s" % options.postgres
application.db = momoko.Pool(dsn, size=1, max_size=100)
ioloop.run_sync(application.db.connect)
else:
db = motor.MotorClient(options.mongo).hello_world
ioloop.start()
@tornado.gen.coroutine
def print_response(url):
response = yield fetch_coroutine(url)
print(response)
def synchronous_fetch(url):
http_client = tornado.httpclient.HTTPClient()
response = http_client.fetch(url)
return response.body
def asynchronous_fetch(url, callback):
http_client = tornado.httpclient.AsyncHTTPClient()
def handle_response(response):
callback(response.body)
http_client.fetch(url, callback=handle_response)
if __name__ == '__main__':
ioloop = tornado.ioloop.IOLoop.current()
print(synchronous_fetch('http://www.baidu.com/'))
# if the `.IOLoop` is not yet running, you can start the IOLoop, run the coroutine,
# and then stop the IOLoop with the IOLoop.run_sync method.
ioloop.run_sync(lambda: asynchronous_fetch('http://www.baidu.com', lambda x: print(x)))
# call asynchronous func
ioloop.spawn_callback(asynchronous_fetch, 'http://www.baidu.com', lambda x: print(x))
ioloop.spawn_callback(print_response, 'http://www.baidu.com')
ioloop.start()
def main():
tornado.options.define('cull_period', default=600,
help="Interval (s) for culling idle containers."
)
tornado.options.define('cull_timeout', default=3600,
help="Timeout (s) for culling idle containers."
)
tornado.options.define('container_ip', default='127.0.0.1',
help="IP address for containers to bind to"
)
tornado.options.define('container_port', default='8888',
help="Port for containers to bind to"
)
command_default = (
'ipython3 notebook --no-browser'
' --port {port} --ip=0.0.0.0'
' --NotebookApp.base_url=/{base_path}'
' --NotebookApp.tornado_settings=\'{{ \"template_path\": [ \"/srv/ga\", \"/srv/ipython/IPython/html\", \"/srv/ipython/IPython/html/templates\" ] }}\''
)
tornado.options.define('command', default=command_default,
help="command to run when booting the image. A placeholder for base_path should be provided."
)
tornado.options.define('port', default=9999,
help="port for the main server to listen on"
)
tornado.options.define('max_dock_workers', default=2,
help="Maximum number of docker workers"
)
tornado.options.define('mem_limit', default="512m",
help="Limit on Memory, per container"
)
tornado.options.define('cpu_shares', default=None,
help="Limit CPU shares, per container"
)
tornado.options.define('image', default="jupyter/demo",
help="Docker container to spawn for new users. Must be on the system already"
)
tornado.options.define('docker_version', default="1.13",
help="Version of the Docker API to use"
)
tornado.options.define('redirect_uri', default="/tree",
help="URI to redirect users to upon initial notebook launch"
)
tornado.options.define('pool_size', default=128,
help="Capacity for containers on this system. Will be prelaunched at startup."
)
tornado.options.define('pool_name', default=None,
help="Container name fragment used to identity containers that belong to this instance."
)
tornado.options.define('static_files', default=None,
help="Static files to extract from the initial container launch"
)
tornado.options.parse_command_line()
opts = tornado.options.options
handlers = [
(r"/", LoadingHandler),
(r"/spawn/?(/user-\w+(?:/.*)?)?", SpawnHandler),
(r"/(user-\w+)(?:/.*)?", LoadingHandler),
(r"/stats", StatsHandler),
]
proxy_token = os.environ['CONFIGPROXY_AUTH_TOKEN']
proxy_endpoint = os.environ.get('CONFIGPROXY_ENDPOINT', "http://127.0.0.1:8001")
docker_host = os.environ.get('DOCKER_HOST', 'unix://var/run/docker.sock')
max_age = datetime.timedelta(seconds=opts.cull_timeout)
pool_name = opts.pool_name
if pool_name is None:
# Derive a valid container name from the image name by default.
pool_name = re.sub('[^a-zA-Z0_.-]+', '', opts.image.split(':')[0])
container_config = dockworker.ContainerConfig(
image=opts.image,
command=opts.command,
mem_limit=opts.mem_limit,
cpu_shares=opts.cpu_shares,
container_ip=opts.container_ip,
container_port=opts.container_port,
)
spawner = dockworker.DockerSpawner(docker_host,
version=opts.docker_version,
timeout=30,
max_workers=opts.max_dock_workers,
)
static_path = os.path.join(os.path.dirname(__file__), "static")
pool = spawnpool.SpawnPool(proxy_endpoint=proxy_endpoint,
proxy_token=proxy_token,
spawner=spawner,
container_config=container_config,
capacity=opts.pool_size,
max_age=max_age,
static_files=opts.static_files,
static_dump_path=static_path,
pool_name=pool_name,
)
ioloop = tornado.ioloop.IOLoop().instance()
settings = dict(
static_path=static_path,
cookie_secret=uuid.uuid4(),
xsrf_cookies=True,
debug=True,
cull_period=opts.cull_period,
spawner=spawner,
pool=pool,
autoescape=None,
proxy_token=proxy_token,
template_path=os.path.join(os.path.dirname(__file__), 'templates'),
proxy_endpoint=proxy_endpoint,
redirect_uri=opts.redirect_uri.lstrip('/'),
)
# Synchronously cull any existing, inactive containers, and pre-launch a set number of
# containers, ready to serve.
ioloop.run_sync(pool.heartbeat)
if(opts.static_files):
ioloop.run_sync(pool.copy_static)
# Periodically execute a heartbeat function to cull used containers and regenerated failed
# ones, self-healing the cluster.
cull_ms = opts.cull_period * 1e3
app_log.info("Culling containers unused for %i seconds every %i seconds.",
opts.cull_timeout,
opts.cull_period)
culler = tornado.ioloop.PeriodicCallback(pool.heartbeat, cull_ms)
culler.start()
app_log.info("Listening on {}".format(opts.port))
application = tornado.web.Application(handlers, **settings)
application.listen(opts.port)
ioloop.start()
if not args.no_ssl and (args.ssl_cert is None or args.ssl_key is None):
sys.stderr.write(
'--no-ssl argument was not specified but no SSL cert or key '
'provided (--ssl-cert / --ssl-key\n')
sys.exit(1)
# Connect to database
lg.info('Connecting to Redis database at {0}:{1!s}'.format(args.redis_host, args.redis_port))
database = cloaca.db.connect(
host=args.redis_host,
port=args.redis_port,
prefix='') # prefix doesn't work with Lua scripts yet.
ioloop = tornado.ioloop.IOLoop.current()
ioloop.run_sync(lambda: database.select(args.redis_db))
# Start server
lg.info('Starting Cloaca server on port {0}'.format(args.port))
app = make_app(database)
settings = {}
if not args.no_ssl:
settings['ssl_options'] = {
"certfile": os.path.abspath(args.ssl_cert),
"keyfile": os.path.abspath(args.ssl_key),
}
httpserver = tornado.httpserver.HTTPServer(app, **settings)
print("endpoint: %s" % endpoint)
print("project_name: %s" % project_name)
print("topic_name: %s" % topic_name)
print("=======================================\n\n")
if not access_id or not access_key or not endpoint:
print("access_id and access_key and endpoint must be set!")
sys.exit(-1)
dh = DataHub(access_id, access_key, endpoint)
@tornado.gen.coroutine
def run():
try:
topics = yield dh.list_topics(project_name)
for topic in topics:
print(topic)
# dh.split_shard(project_name, topic, '1', '88888888888888888888888888888888')
shards = yield dh.list_shards(project_name, topic)
for shard in shards:
print(shard)
print("=======================================\n\n")
except Exception as e:
print(traceback.format_exc())
sys.exit(-1)
ioloop = tornado.ioloop.IOLoop.current()
ioloop.run_sync(run)
# ioloop.start()
# result is already a python object (a string in this simple example)
print "Result: %s" % result
while True:
result = yield client.pubsub_pop_message()
if isinstance(result, tornadis.TornadisException):
# For specific reasons, tornadis nearly never raises any exception
# they are returned as result
print "got exception: %s" % result
else:
# result is already a python object (a string in this simple example)
print "Result: %s" % result
# Build a tornadis.Client object with some options as kwargs
# host: redis host to connect
# port: redis port to connect
# autoconnect=True: put the Client object in auto(re)connect mode
# Start a tornado IOLoop, execute the coroutine and end the program
ioloop = tornado.ioloop.IOLoop.instance()
#ioloop.run_sync(talk_to_redis)
client = tornadis.PubSubClient(host="localhost", port=6379, autoconnect=True)
ioloop.run_sync(talk_to_redis)
try:
ioloop.start()
except KeyboardInterrupt:
pass
finally:
print "Closing server...\n"
tornado.ioloop.IOLoop.instance().stop()
def async_download(lcs, clobber=False):
f = partial(_async_download, lcs, clobber=clobber)
ioloop = tornado.ioloop.IOLoop.instance()
ioloop.run_sync(f)
@gen.coroutine
def add_member_to_card(self, card_id, member_id):
body = {'value': member_id}
encoded = json.dumps(body)
response = yield self.make_request(
ADD_MEMBER_TO_CARD_URL.format(card_id=card_id),
body=encoded
)
return response
if __name__ == "__main__":
@gen.coroutine
def test_all():
t = Trello()
boards = yield t.list_boards()
pprint(boards)
a_board = list(boards.values())[0]
lists = yield t.list_lists(a_board)
pprint(lists)
a_list = list(lists.values())[0]
card = yield t.add_card("Just a test",
'530259811a17caee605998d1',
'doot doot doo',
members=['51d3212603fc83a55d000cee'])
print(card)
ioloop = tornado.ioloop.IOLoop.instance()
ioloop.run_sync(test_all)
def main():
tornado.options.define('cull_period',
default=600,
help="Interval (s) for culling idle containers.")
tornado.options.define('cull_timeout',
default=3600,
help="Timeout (s) for culling idle containers.")
tornado.options.define('container_ip',
default='127.0.0.1',
help="IP address for containers to bind to")
tornado.options.define('container_port',
default='8888',
help="Port for containers to bind to")
tornado.options.define(
'ipython_executable',
default='ipython3',
help="IPython Notebook startup (e.g. ipython, ipython2, ipython3)")
tornado.options.define('port',
default=9999,
help="port for the main server to listen on")
tornado.options.define('max_dock_workers',
default=24,
help="Maximum number of docker workers")
tornado.options.define('mem_limit',
default="512m",
help="Limit on Memory, per container")
tornado.options.define('cpu_shares',
default=None,
help="Limit CPU shares, per container")
tornado.options.define(
'image',
default="jupyter/demo",
help=
"Docker container to spawn for new users. Must be on the system already"
)
tornado.options.define('docker_version',
default="1.13",
help="Version of the Docker API to use")
tornado.options.define(
'redirect_uri',
default="/tree",
help="URI to redirect users to upon initial notebook launch")
tornado.options.define(
'pool_size',
default=128,
help=
"Capacity for containers on this system. Will be prelaunched at startup."
)
tornado.options.parse_command_line()
opts = tornado.options.options
handlers = [
(r"/", LoadingHandler),
(r"/spawn/?(/user-\w+/.+)?", SpawnHandler),
(r"/(user-\w+)/.*", LoadingHandler),
]
proxy_token = os.environ['CONFIGPROXY_AUTH_TOKEN']
proxy_endpoint = os.environ.get('CONFIGPROXY_ENDPOINT',
"http://127.0.0.1:8001")
docker_host = os.environ.get('DOCKER_HOST', 'unix://var/run/docker.sock')
max_age = datetime.timedelta(seconds=opts.cull_timeout)
container_config = dockworker.ContainerConfig(
image=opts.image,
ipython_executable=opts.ipython_executable,
mem_limit=opts.mem_limit,
cpu_shares=opts.cpu_shares,
container_ip=opts.container_ip,
container_port=opts.container_port)
spawner = dockworker.DockerSpawner(docker_host,
version=opts.docker_version,
timeout=20,
max_workers=opts.max_dock_workers)
pool = spawnpool.SpawnPool(proxy_endpoint=proxy_endpoint,
proxy_token=proxy_token,
spawner=spawner,
container_config=container_config,
capacity=opts.pool_size,
max_age=max_age)
ioloop = tornado.ioloop.IOLoop().instance()
settings = dict(
static_path=os.path.join(os.path.dirname(__file__), "static"),
cookie_secret=uuid.uuid4(),
xsrf_cookies=True,
debug=True,
cull_period=opts.cull_period,
spawner=spawner,
pool=pool,
autoescape=None,
proxy_token=proxy_token,
template_path=os.path.join(os.path.dirname(__file__), 'templates'),
proxy_endpoint=proxy_endpoint,
redirect_uri=opts.redirect_uri.lstrip('/'),
)
# Synchronously cull any existing, inactive containers, and pre-launch a set number of
# containers, ready to serve.
ioloop.run_sync(pool.heartbeat)
# Periodically execute a heartbeat function to cull used containers and regenerated failed
# ones, self-healing the cluster.
cull_ms = opts.cull_period * 1e3
app_log.info("Culling containers unused for %i seconds every %i seconds.",
opts.cull_timeout, opts.cull_period)
culler = tornado.ioloop.PeriodicCallback(pool.heartbeat, cull_ms)
culler.start()
app_log.info("Listening on {}".format(opts.port))
application = tornado.web.Application(handlers, **settings)
application.listen(opts.port)
ioloop.start()
start = time.time()
http_client = httpclient.AsyncHTTPClient()
response = yield http_client.fetch(URL)
datetime = response.headers.get('Date')
print('Process {}: {}, took: {:.2f} seconds'.format(
pid, datetime, time.time() - start))
return datetime
def synchronous():
start = time.time()
for i in range(1, MAX_CLIENTS + 1):
fetch_sync(i)
print("Process took: {:.2f} seconds".format(time.time() - start))
@gen.coroutine
def asynchronous():
start = time.time()
yield [fetch_async(i) for i in range(1, MAX_CLIENTS + 1)]
print("Process took: {:.2f} seconds".format(time.time() - start))
print('Synchronous:')
synchronous()
print('Asynchronous:')
ioloop = ioloop.IOLoop.current()
ioloop.run_sync(asynchronous)
def load(self):
ioloop = tornado.ioloop.IOLoop.current()
ioloop.run_sync(self.load_async)
http_client = httpclient.AsyncHTTPClient()
response = yield http_client.fetch(URL)
datetime = response.headers.get('Date')
print('Process {}: {}, took: {:.2f} seconds'.format(
pid, datetime,
time.time() - start))
return datetime
def synchronous():
start = time.time()
for i in range(1, MAX_CLIENTS + 1):
fetch_sync(i)
print("Process took: {:.2f} seconds".format(time.time() - start))
@gen.coroutine
def asynchronous():
start = time.time()
yield [fetch_async(i) for i in range(1, MAX_CLIENTS + 1)]
print("Process took: {:.2f} seconds".format(time.time() - start))
print('Synchronous:')
synchronous()
print('Asynchronous:')
ioloop = ioloop.IOLoop.current()
ioloop.run_sync(asynchronous)
def start_ioloop(): # pragma: no cover
ioloop = tornado.ioloop.IOLoop.current()
ioloop.run_sync(main)
def make_app(database):
app_path = cloaca.handlers.APPDIR
site_path = os.path.join(app_path, 'site')
js_path = os.path.join(site_path, 'js')
ioloop = tornado.ioloop.IOLoop.current()
ioloop.run_sync(database.load_scripts)
server = GTRServer(database)
def send_command(user_id, command):
try:
cxn = GameWSHandler.client_cxn_by_user_id[user_id]
except KeyError:
lg.debug('User ID {0!s} is not connected.'.format(user_id))
return
cxn.send_command(command)
server.send_command = send_command
settings = dict(
cookie_secret='__TODO:_GENERATE_COOKIE_SECRET__',
login_url='/login',
xsrf_cookies=True,
)
# WEBSOCKET_URI = '/ws/'
WEBSOCKET_URI = '/ws'
JS_MIN = 'cloaca.min.js'
CSS_MIN = 'style.min.css'
# Check if minified files have been built and use them in GameHandler.
js_main_path = '/js/main'
if os.path.exists(os.path.join(site_path, JS_MIN)):
js_main_path = os.path.join('/', JS_MIN.rstrip('.js'))
css_path = '/style.css'
if os.path.exists(os.path.join(site_path, CSS_MIN)):
css_path = os.path.join('/', CSS_MIN)
return tornado.web.Application([
(r'/(favicon.ico)', tornado.web.StaticFileHandler, {'path':site_path}),
(r'/newgame', CreateGameHandler, {'database':database, 'server':server}),
(r'/joingame/([0-9]+)', JoinGameHandler, {'database':database, 'server':server}),
(r'/startgame/([0-9]+)', StartGameHandler, {'database':database, 'server':server}),
(r'/game/([0-9]+)', GameHandler, {'database':database, 'server':server,
'websocket_uri':WEBSOCKET_URI,
'js_main_path': js_main_path,
'stylesheet_path': css_path,
}),
(r'/(style.css)', tornado.web.StaticFileHandler, {'path':site_path}),
(r'/(style.min.css)', tornado.web.StaticFileHandler, {'path':site_path}),
(r'/js/(.*)', tornado.web.StaticFileHandler, {'path':js_path}),
(r'/(cloaca.min.js)', tornado.web.StaticFileHandler, {'path':site_path}),
(r'/', GameListHandler, {'database':database}),
(WEBSOCKET_URI, GameWSHandler, {'database':database, 'server':server}),
#(r'/register', RegisterHandler, {'database': database}),
(r'/login', LoginPageHandler, {'database': database}),
(r'/auth', AuthenticateHandler, {'database': database}),
(r'/logout', LogoutHandler, {'database': database}),
],
**settings)
(r"/db", SingleQueryHandler),
(r"/queries", MultipleQueriesHandler),
(r"/updates", UpdateHandler),
(r"/fortunes", FortuneHandler),
],
template_path="templates")
async def setup_database():
return await asyncpg.create_pool(user=getenv('PGUSER', 'benchmarkdbuser'),
password=getenv('PGPASS',
'benchmarkdbpass'),
database='hello_world',
host='tfb-database',
port=5432)
if __name__ == "__main__":
options.parse_command_line()
uvloop.install()
app = make_app()
server = tornado.httpserver.HTTPServer(app)
server.bind(options.port)
server.start(0)
ioloop = tornado.ioloop.IOLoop.current()
app.pool = ioloop.run_sync(partial(setup_database))
ioloop.start()
cursors = yield [
db.execute(self.SQL, (randint(1, 10000), ))
for _ in xrange(queries)
]
for cursor in cursors:
row = cursor.fetchone()
worlds.append({self.ID: row[0], self.RANDOM_NUMBER: row[1]})
response = json.dumps(worlds)
self.finish(response)
application = tornado.web.Application(
[(r"/json", JsonHelloWorldHandler),
(r"/plaintext", PlaintextHelloWorldHandler), (r"/db", SingleQueryHandler),
(r"/queries", MultipleQueriesHandler)],
template_path="templates")
application.ui_modules = {}
if __name__ == "__main__":
tornado.options.parse_command_line()
server = tornado.httpserver.HTTPServer(application)
server.bind(options.port, backlog=options.backlog)
server.start(0)
ioloop = tornado.ioloop.IOLoop.instance()
dsn = "user=benchmarkdbuser password=benchmarkdbpass dbname=hello_world host=%s" % options.postgres
db = momoko.Pool(dsn, size=100, max_size=200)
ioloop.run_sync(db.connect)
ioloop.start()
return task_future
sleep_future.add_done_callback(callback)
return task_future
def synchronous():
for i in range(1, 10):
task(i)
@gen.coroutine
def asynchronous():
yield [executor.submit(task, i) for i in range(10)]
@gen.coroutine
def asynchronous_python2():
yield [task_async(i) for i in range(10)]
print('Synchronous:')
synchronous()
print('Asynchronous:')
ioloop = ioloop.IOLoop.current()
print(ioloop.run_sync(asynchronous))
print('Asynchronous Python2-compatible:')
print(ioloop.run_sync(asynchronous_python2))
def wrapper(*args, **kwargs):
ioloop = self.get_new_ioloop()
return ioloop.run_sync(lambda: function(*args, **kwargs))
def on_pong(self, data):
print('PONG', data)
# Uncomment this to allow cross-domain access
#def check_origin(self, origin):
# return True
application = tornado.web.Application([
(r'/ws', WSHandler),
(r"/(.*)", MainHandler),
])
if __name__ == "__main__":
#http_server = tornado.httpserver.HTTPServer(application)
#http_server.listen(8888)
application.listen(8888)
webbrowser.open('http://localhost:8888')
# Start the main loop (http://www.tornadoweb.org/en/stable/ioloop.html)
ioloop = tornado.ioloop.IOLoop.instance()
ioloop.instance().start()
ioloop.run_sync(lambda x=None: None) # == process_events
# Also see ioloop.call_later() and ioloop.add_callback()
def async_download(lcs, clobber=False):
f = partial(_async_download, lcs, clobber=clobber)
ioloop = tornado.ioloop.IOLoop.instance()
ioloop.run_sync(f)
ioloop = tornado.ioloop.IOLoop.instance()
application.db = momoko.Pool(
dsn=dsn,
size=1,
max_size=3,
ioloop=ioloop,
setsession=("SET TIME ZONE UTC",),
raise_connect_errors=False,
)
# this is a one way to run ioloop in sync
future = application.db.connect()
ioloop.add_future(future, lambda f: ioloop.stop())
ioloop.start()
if enable_hstore:
future = application.db.register_hstore()
# This is the other way to run ioloop in sync
ioloop.run_sync(lambda: future)
if application.db.server_version >= 90200:
future = application.db.register_json()
# This is the other way to run ioloop in sync
ioloop.run_sync(lambda: future)
http_server = tornado.httpserver.HTTPServer(application)
http_server.listen(8000, 'localhost')
print ('tornado server started on %s port' % 8000)
ioloop.start()
def start_ioloop(): # pragma: no cover
ioloop = tornado.ioloop.IOLoop.current()
ioloop.run_sync(main)
(r'/query/http', QueryHttpHandler),
])
application.db = momoko.Pool(
dsn='dbname=%s user=%s password=%s host=%s port=%s' %
(options.db_database, options.db_user, options.db_password,
options.db_host, options.db_port),
size=1,
max_size=3,
ioloop=ioloop,
setsession=('SET TIME ZONE UTC', ),
raise_connect_errors=False,
)
logging.info('Connecting to PostgreSQL on [%s]' % options.db_host)
ioloop.run_sync(lambda: application.db.connect())
logging.info('Listening on [8888] port')
application.listen(8888)
# prepares the web server
# srv = tornado.httpserver.HTTPServer(application, xheaders=True)
# listens incoming request on port 8000
# srv.bind(8888, '')
# starts the server using 1 process
# srv.start(1)
logging.info('Starting IO loop')
ioloop.start()
ioloop = tornado.ioloop.IOLoop.instance()
application.db = momoko.Pool(
dsn=dsn,
size=1,
max_size=3,
ioloop=ioloop,
setsession=("SET TIME ZONE UTC", ),
raise_connect_errors=False,
)
# this is a one way to run ioloop in sync
future = application.db.connect()
ioloop.add_future(future, lambda f: ioloop.stop())
ioloop.start()
if enable_hstore:
future = application.db.register_hstore()
# This is the other way to run ioloop in sync
ioloop.run_sync(lambda: future)
if application.db.server_version >= 90200:
future = application.db.register_json()
# This is the other way to run ioloop in sync
ioloop.run_sync(lambda: future)
http_server = tornado.httpserver.HTTPServer(application)
http_server.listen(8000, 'localhost')
print('tornado server started on %s port' % 8000)
ioloop.start()
def close_this(self):
self.close(1000, 'closed by server')
def on_pong(self, data):
print('PONG', data)
# Uncomment this to allow cross-domain access
#def check_origin(self, origin):
# return True
application = tornado.web.Application([
(r'/ws', WSHandler),
(r"/(.*)", MainHandler),
])
if __name__ == "__main__":
#http_server = tornado.httpserver.HTTPServer(application)
#http_server.listen(8888)
application.listen(8888)
webbrowser.open('http://localhost:8888')
# Start the main loop (http://www.tornadoweb.org/en/stable/ioloop.html)
ioloop = tornado.ioloop.IOLoop.instance()
ioloop.instance().start()
ioloop.run_sync(lambda x=None: None) # == process_events
# Also see ioloop.call_later() and ioloop.add_callback()
def user(args):
ioloop = tornado.ioloop.IOLoop.instance()
scope = rw.cli.setup_app(args.module, ioloop=ioloop)
with scope():
ioloop.run_sync(_user)
(r"/queries", MultipleQueriesHandler),
(r"/updates", UpdateHandler),
(r"/fortunes", FortuneHandler),
],
template_path="templates"
)
async def setup_database():
return await asyncpg.create_pool(
user=getenv('PGUSER', 'benchmarkdbuser'),
password=getenv('PGPASS', 'benchmarkdbpass'),
database='hello_world',
host='tfb-database',
port=5432
)
if __name__ == "__main__":
options.parse_command_line()
uvloop.install()
app = make_app()
server = tornado.httpserver.HTTPServer(app)
server.bind(options.port)
server.start(0)
ioloop = tornado.ioloop.IOLoop.current()
app.pool = ioloop.run_sync(partial(setup_database))
ioloop.start()
# Main
# ====
def unknown_action(_logger):
def f(logger=_logger):
logger.info("I don't know that action.")
return f
if __name__ == "__main__":
tornado.log.enable_pretty_logging()
config = get_config()
srv = EestecPlatform(config)
action = "run" if len(sys.argv) == 1 else sys.argv[1]
try:
# Some magic to allow both synchronous and asynchronous usage
f = getattr(srv, action, unknown_action(srv.logger))()
if isinstance(f, types.CoroutineType):
async def await_f():
await f
ioloop.run_sync(await_f)
except KeyboardInterrupt:
ioloop.stop()
srv.logger.info("Stopping because of KeyboardInterrupt")
def wrapper(*args, **kwargs):
ioloop = self.get_new_ioloop()
return ioloop.run_sync(lambda: function(*args, **kwargs))
from tornado import gen
from tornado import ioloop
@gen.coroutine
def foo():
print('Running in foo')
yield gen.sleep(0)
print('Explicit context switch to foo again')
@gen.coroutine
def bar():
print('Explicit context to bar')
yield gen.sleep(0)
print('Implicit context switch back to bar')
@gen.coroutine
def main():
yield [foo(), bar()]
ioloop = ioloop.IOLoop.current()
ioloop.run_sync(main)
def main():
tornado.options.define('cull_period',
default=600,
help="Interval (s) for culling idle containers.")
tornado.options.define('cull_timeout',
default=3600,
help="Timeout (s) for culling idle containers.")
tornado.options.define('container_ip',
default='127.0.0.1',
help="IP address for containers to bind to")
tornado.options.define('container_port',
default='8888',
help="Port for containers to bind to")
command_default = ('ipython3 notebook --no-browser'
' --port {port} --ip=0.0.0.0'
' --NotebookApp.base_url=/{base_path}')
tornado.options.define(
'command',
default=command_default,
help=
"command to run when booting the image. A placeholder for base_path should be provided."
)
tornado.options.define('port',
default=9999,
help="port for the main server to listen on")
tornado.options.define(
'ip',
default=None,
help="ip for the main server to listen on [default: all interfaces]")
tornado.options.define('max_dock_workers',
default=2,
help="Maximum number of docker workers")
tornado.options.define('mem_limit',
default="512m",
help="Limit on Memory, per container")
tornado.options.define('cpu_shares',
default=None,
help="Limit CPU shares, per container")
tornado.options.define(
'image',
default="jupyter/minimal",
help=
"Docker container to spawn for new users. Must be on the system already"
)
tornado.options.define('docker_version',
default="1.13",
help="Version of the Docker API to use")
tornado.options.define(
'redirect_uri',
default="/tree",
help="URI to redirect users to upon initial notebook launch")
tornado.options.define(
'pool_size',
default=10,
help=
"Capacity for containers on this system. Will be prelaunched at startup."
)
tornado.options.define(
'pool_name',
default=None,
help=
"Container name fragment used to identity containers that belong to this instance."
)
tornado.options.define(
'static_files',
default=None,
help="Static files to extract from the initial container launch")
tornado.options.parse_command_line()
opts = tornado.options.options
handlers = [
(r"/", LoadingHandler),
(r"/spawn/?(/user/\w+(?:/.*)?)?", SpawnHandler),
(r"/(user/\w+)(?:/.*)?", LoadingHandler),
(r"/stats", StatsHandler),
]
proxy_token = os.environ['CONFIGPROXY_AUTH_TOKEN']
proxy_endpoint = os.environ.get('CONFIGPROXY_ENDPOINT',
"http://127.0.0.1:8001")
docker_host = os.environ.get('DOCKER_HOST', 'unix://var/run/docker.sock')
max_age = datetime.timedelta(seconds=opts.cull_timeout)
pool_name = opts.pool_name
if pool_name is None:
# Derive a valid container name from the image name by default.
pool_name = re.sub('[^a-zA-Z0_.-]+', '', opts.image.split(':')[0])
container_config = dockworker.ContainerConfig(
image=opts.image,
command=opts.command,
mem_limit=opts.mem_limit,
cpu_shares=opts.cpu_shares,
container_ip=opts.container_ip,
container_port=opts.container_port,
)
spawner = dockworker.DockerSpawner(
docker_host,
version=opts.docker_version,
timeout=30,
max_workers=opts.max_dock_workers,
)
static_path = os.path.join(os.path.dirname(__file__), "static")
pool = spawnpool.SpawnPool(
proxy_endpoint=proxy_endpoint,
proxy_token=proxy_token,
spawner=spawner,
container_config=container_config,
capacity=opts.pool_size,
max_age=max_age,
static_files=opts.static_files,
static_dump_path=static_path,
pool_name=pool_name,
)
ioloop = tornado.ioloop.IOLoop().instance()
settings = dict(
static_path=static_path,
cookie_secret=uuid.uuid4(),
xsrf_cookies=True,
debug=True,
cull_period=opts.cull_period,
spawner=spawner,
pool=pool,
autoescape=None,
proxy_token=proxy_token,
template_path=os.path.join(os.path.dirname(__file__), 'templates'),
proxy_endpoint=proxy_endpoint,
redirect_uri=opts.redirect_uri.lstrip('/'),
)
# Synchronously cull any existing, inactive containers, and pre-launch a set number of
# containers, ready to serve.
ioloop.run_sync(pool.heartbeat)
if (opts.static_files):
ioloop.run_sync(pool.copy_static)
# Periodically execute a heartbeat function to cull used containers and regenerated failed
# ones, self-healing the cluster.
cull_ms = opts.cull_period * 1e3
app_log.info("Culling containers unused for %i seconds every %i seconds.",
opts.cull_timeout, opts.cull_period)
culler = tornado.ioloop.PeriodicCallback(pool.heartbeat, cull_ms)
culler.start()
app_log.info("Listening on {}:{}".format(opts.ip or '*', opts.port))
application = tornado.web.Application(handlers, **settings)
application.listen(opts.port, opts.ip)
ioloop.start()
def main():
tornado.options.define('cull_period', default=600,
help="Interval (s) for culling idle containers."
)
tornado.options.define('cull_timeout', default=3600,
help="Timeout (s) for culling idle containers."
)
tornado.options.define('container_ip', default='127.0.0.1',
help="IP address for containers to bind to"
)
tornado.options.define('container_port', default='8888',
help="Port for containers to bind to"
)
tornado.options.define('ipython_executable', default='ipython3',
help="IPython Notebook startup (e.g. ipython, ipython2, ipython3)"
)
tornado.options.define('port', default=9999,
help="port for the main server to listen on"
)
tornado.options.define('max_dock_workers', default=24,
help="Maximum number of docker workers"
)
tornado.options.define('mem_limit', default="512m",
help="Limit on Memory, per container"
)
tornado.options.define('cpu_shares', default=None,
help="Limit CPU shares, per container"
)
tornado.options.define('image', default="jupyter/demo",
help="Docker container to spawn for new users. Must be on the system already"
)
tornado.options.define('docker_version', default="1.13",
help="Version of the Docker API to use"
)
tornado.options.define('redirect_uri', default="/tree",
help="URI to redirect users to upon initial notebook launch"
)
tornado.options.define('pool_size', default=128,
help="Capacity for containers on this system. Will be prelaunched at startup."
)
tornado.options.parse_command_line()
opts = tornado.options.options
handlers = [
(r"/", LoadingHandler),
(r"/spawn/?(/user-\w+(?:/.*)?)?", SpawnHandler),
(r"/(user-\w+)(?:/.*)?", LoadingHandler),
]
proxy_token = os.environ['CONFIGPROXY_AUTH_TOKEN']
proxy_endpoint = os.environ.get('CONFIGPROXY_ENDPOINT', "http://127.0.0.1:8001")
docker_host = os.environ.get('DOCKER_HOST', 'unix://var/run/docker.sock')
max_age = datetime.timedelta(seconds=opts.cull_timeout)
container_config = dockworker.ContainerConfig(
image=opts.image,
ipython_executable=opts.ipython_executable,
mem_limit=opts.mem_limit,
cpu_shares=opts.cpu_shares,
container_ip=opts.container_ip,
container_port=opts.container_port
)
spawner = dockworker.DockerSpawner(docker_host,
version=opts.docker_version,
timeout=20,
max_workers=opts.max_dock_workers)
pool = spawnpool.SpawnPool(proxy_endpoint=proxy_endpoint,
proxy_token=proxy_token,
spawner=spawner,
container_config=container_config,
capacity=opts.pool_size,
max_age=max_age)
ioloop = tornado.ioloop.IOLoop().instance()
settings = dict(
static_path=os.path.join(os.path.dirname(__file__), "static"),
cookie_secret=uuid.uuid4(),
xsrf_cookies=True,
debug=True,
cull_period=opts.cull_period,
spawner=spawner,
pool=pool,
autoescape=None,
proxy_token=proxy_token,
template_path=os.path.join(os.path.dirname(__file__), 'templates'),
proxy_endpoint=proxy_endpoint,
redirect_uri=opts.redirect_uri.lstrip('/'),
)
# Synchronously cull any existing, inactive containers, and pre-launch a set number of
# containers, ready to serve.
ioloop.run_sync(pool.heartbeat)
# Periodically execute a heartbeat function to cull used containers and regenerated failed
# ones, self-healing the cluster.
cull_ms = opts.cull_period * 1e3
app_log.info("Culling containers unused for %i seconds every %i seconds.",
opts.cull_timeout,
opts.cull_period)
culler = tornado.ioloop.PeriodicCallback(pool.heartbeat, cull_ms)
culler.start()
app_log.info("Listening on {}".format(opts.port))
application = tornado.web.Application(handlers, **settings)
application.listen(opts.port)
ioloop.start()
cursors = yield [db.execute(self.SQL, (randint(1, 10000),)) for _ in xrange(queries)]
for cursor in cursors:
row = cursor.fetchone()
worlds.append({self.ID: row[0], self.RANDOM_NUMBER: row[1]})
response = json.dumps(worlds)
self.finish(response)
application = tornado.web.Application([
(r"/json", JsonHelloWorldHandler),
(r"/plaintext", PlaintextHelloWorldHandler),
(r"/db", SingleQueryHandler),
(r"/queries", MultipleQueriesHandler)
],
template_path="templates"
)
application.ui_modules = {}
if __name__ == "__main__":
tornado.options.parse_command_line()
server = tornado.httpserver.HTTPServer(application)
server.bind(options.port, backlog=options.backlog)
server.start(0)
ioloop = tornado.ioloop.IOLoop.instance()
dsn = "user=benchmarkdbuser password=benchmarkdbpass dbname=hello_world host=%s" % options.postgres
db = momoko.Pool(dsn, size=100, max_size=200)
ioloop.run_sync(db.connect)
ioloop.start()
