def main():
tornado.options.define('base_path', default='/',
help="Base path for the server (e.g. /singlecell)"
)
tornado.options.parse_command_line()
opts = tornado.options.options
kernel_manager = MultiKernelManager()
# we are only using one kernel:
kernel_id = '1'
kernel_manager.start_kernel(kernel_id=kernel_id)
logging.basicConfig(level=logging.INFO)
app = WebApp(kernel_manager, opts.base_path)
server = httpserver.HTTPServer(app)
server.listen(8888, '0.0.0.0')
try:
ioloop.IOLoop.instance().start()
except KeyboardInterrupt:
app_log.info("Interrupted...")
finally:
kernel_manager.shutdown_all()
def on_close(self):
app_log.info(f'close socket {self}')
if self in self.sockets:
self.sockets.remove(self)
for i in self.subscriptions:
self.subscriptions[i].dispose()
self.subscriptions = {}
def get(self):
"""
用户使用QQ第三方登录，获取登录地址并跳转
"""
request_url = self.__build_auth_url()
app_log.info("tencent auth_code: {0}".format(request_url))
self.redirect(request_url)
def main():
handlers = [
(r"/", RootHandler),
(r"/login", GitHubLoginHandler),
(r"/oauth", GitHubOAuthHandler)
]
settings = dict(
cookie_secret=str(base64.b64encode(uuid.uuid4().bytes + uuid.uuid4().bytes), 'utf-8'),
login_url="/login",
xsrf_cookies=True,
github_client_id=os.environ["GITHUB_CLIENT_ID"],
github_client_secret=os.environ["GITHUB_CLIENT_SECRET"],
github_redirect_uri=os.environ["GITHUB_REDIRECT_URI"],
github_scope="",
debug=True,
autoescape=None
)
port = 8088
app_log.info("Listening on {}".format(port))
application = tornado.web.Application(handlers, **settings)
application.listen(port)
tornado.ioloop.IOLoop().instance().start()
def get_data(cls, account, source_filter, limit=100, skip=0):
"""
Gathers card information from Google Sheets
GET https://spreadsheets.google.com/feeds/list/[spreadsheet]/[worksheet]/private/full
"""
if not account or not account.enabled:
raise ValueError('cannot gather information without an account')
client = AsyncHTTPClient()
if source_filter.spreadsheet is None:
raise ValueError('required parameter spreadsheet missing')
if source_filter.worksheet is None:
raise ValueError('required parameter worksheet missing')
uri = "https://docs.google.com/spreadsheets/d/{}/export?format=csv&gid={}".format(
source_filter.spreadsheet, source_filter.worksheet
)
app_log.info(
"Start retrieval of worksheet {}/{} for {}".format(source_filter.spreadsheet, source_filter.worksheet,
account._id))
lock = Condition()
oauth_client = account.get_client()
uri, headers, body = oauth_client.add_token(uri)
req = HTTPRequest(uri, headers=headers, body=body, streaming_callback=lambda c: cls.write(c))
client.fetch(req, callback=lambda r: lock.notify())
yield lock.wait(timeout=timedelta(seconds=MAXIMUM_REQ_TIME))
app_log.info(
"Finished retrieving worksheet for {}".format(account._id))
def get_places(self, ll, q):
url = FacebookComm.BASE_URL.format(endpoint=FacebookComm.SEARCH_ENDPOINT)
place = None
try:
url += '&type=place¢er={ll}&distance=100&q={q}'.format(ll=ll, q=q)
log.info('Fetching Facebook places from [{0}]'.format(url))
request = HTTPRequest(url=url, connect_timeout=options.http_request_timeout, request_timeout=options.http_request_timeout)
response = yield self.client.fetch(request)
if response.code != 200:
raise FacebookError(response.code)
body = json.loads(response.body)
places = body['data']
if len(places) > 0:
place = places[0]
except HTTPError as e:
log.error('Facebook error [{0}] while calling [{1}]!'.format(e, url))
raise Return(None)
raise Return(place)
def get_unknown_venues_events(self, venues):
''' Fetch all events for a list of foursquare venue names '''
search_url = FacebookComm.BASE_URL.format(endpoint=FacebookComm.SEARCH_ENDPOINT)
#try matching a foursquare venue to a facebook place
fb_venues = {}
tasks = {}
for vname, venue in venues.iteritems():
ll = venue['ll']
tasks[vname] = Task(self.get_places, ll=ll, q=friendly_str(vname))
log.info('Fetching {0} places from Facebook...'.format(len(tasks.keys())))
places = yield tasks
for vname, place in places.iteritems():
if place is None:
log.info('Venue {0} not found on Facebook!'.format(friendly_str(vname)))
continue
venue = venues[vname]
fb_venues[place['id']] = venue
if len(fb_venues.keys()) == 0:
raise Return({})
# we have the facebook id, fetch the events
fb_venues_events = yield self.get_venues_events(fb_venues)
raise Return(fb_venues_events)
def _start_control(self):
app_log.info("Starting EE Control on port {port}".format(port=config.Control.Rest.PORT))
self._control_process = _start_remote_rest_server(config.Control.Rest.BIN, config.Control.Rest.PORT,
config.Control.Rest.DEBUG)
if self._control_process.is_running() and self._rest_server_listening(config.Control.Rest.BASE_URI):
app_log.info("EEControl REST Server running")
else:
app_log.error("EEControl REST Server not running")
exit(1)
if self._is_engine_supported(
_get_full_uri(config.Control.Rest.BASE_URI, config.Control.Rest.Endpoints.ENGINES)):
app_log.info("{engine} supported by EE Control".format(engine=config.Engine.NAME))
else:
app_log.error("{engine} is not supported by EE Control".format(engine=config.Engine.NAME))
exit(1)
if self._set_engine(_get_full_uri(config.Control.Rest.BASE_URI, config.Control.Rest.Endpoints.ENGINES)):
app_log.info("{engine} set by EE Control".format(engine=config.Engine.NAME))
else:
app_log.error("{engine} not set by EE Control".format(engine=config.Engine.NAME))
exit(1)
if self._connect_control():
app_log.info("EE Control client connected to engine")
else:
app_log.error("EE Control client couldn't connect to engine")
exit(1)
def get(self, file_name, *args, **kwargs):
if not file_name:
return self.send_error(status_code=404)
root_dir = os.path.dirname(os.path.dirname(__file__))
file_path = os.path.join(root_dir, "../static/download/" + file_name)
buf_size = 4096
self.set_header("Content-Type", "application/octet-stream")
self.set_header("Content-Disposition", "attachment; filename=" + file_name)
try:
download_file = open(file_path, "r")
while True:
data = download_file.read(buf_size)
if not data:
break
self.write(data)
download_file.close()
app_log.info("[{ip}] download file success: {name}".format(
ip=self.request.remote_ip, name=file_path
))
except IOError:
app_log.info("[{ip}] download file fail: {name}".format(
ip=self.request.remote_ip, name=file_path
))
return self.send_error(status_code=404)
self.finish()
def start(self):
# reset all connections
try:
app_log.info("Starting Connection Manager!")
if self.idl is not None:
self.idl.close()
# set up the schema and register all tables
self.schema_helper = SchemaHelper(self.schema)
self.schema_helper.register_all()
self.idl = Idl(self.remote, self.schema_helper)
self.curr_seqno = self.idl.change_seqno
# we do not reset transactions when the DB connection goes down
if self.transactions is None:
self.transactions = OvsdbTransactionList()
self.idl_run()
self.monitor_connection()
except Exception as e:
# TODO: log this exception
# attempt again in the next IOLoop iteration
app_log.info("Connection Manager failed! Reason: %s" % e)
IOLoop.current().add_timeout(time.time() + self.timeout,
self.start)
def prepare(self):
request = self.request
if request.headers and 'Content-Encoding' in request.headers:
del request.headers['Content-Encoding']
request.body_arguments = {}
request.files = {}
tornado.httputil.parse_body_arguments(
request.headers.get("Content-Type", ""), request.body, request.body_arguments, request.files)
for k, v in request.body_arguments.items():
request.arguments.setdefault(k, []).extend(v)
# cookie
cookieId = self.get_cookie('cookie_id')
if not cookieId or not Session.getSession(cookieId):
# create cookie id
ua = self.request.headers.get("User-Agent", '')
ip = self.request.remote_ip
time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
md5 = hashlib.md5()
md5.update((ua + ip + time).encode())
cookieId = md5.hexdigest()
Session.createSession(cookieId)
app_log.info('set cookie id ' + cookieId)
self.set_cookie('cookie_id', cookieId, path="/", expires_days=30)
self.cookieId = cookieId
self.db = self.settings['db']
def render_notebook(exporter, json_as_string, url=None, forced_theme=None):
app_log.info("rendering %d B notebook from %s", len(json_as_string), url)
try:
share_as_object = json.loads(json_as_string)
except ValueError:
raise SyntaxError("Notebook does not appear to be JSON: %r" % json_as_string[:16])
template = None
if share_as_object.get('notebookModel') and share_as_object.get('cellModel'):
template = 'old/beaker_section.html'
elif share_as_object.get('notebookModel'):
template = 'old/beaker_notebook.html'
elif share_as_object.get('cellModel'):
template = 'old/beaker_cell.html'
elif share_as_object.get('beaker'):
template = 'beaker_notebook.html'
name = 'Beaker Notebook'
config = {
'download_name': name,
'css_theme': None,
'template': template
}
return json_as_string, config
def test_put_user(self):
client = yield self.auth_client()
user = yield self.create_user(client)
cases = [
("login", "foofoofoo"),
("email", "*****@*****.**"),
("is_admin", False),
("disabled", False),
("password", str(uuid.uuid4()))
]
for i in range(1, len(cases)):
for case in itertools.combinations(cases, i):
body = dict(case)
if 'disabled' in body:
log.info("Deleting user: %r", user['id'])
yield client.fetch(
self.get_url("/api/v1/user/{0}".format(user['id'])),
"DELETE"
)
log.info("Send body: %r", body)
response = yield client.fetch(
self.get_url("/api/v1/user/{0}".format(user['id'])),
"PUT", body
)
for k, v in body.items():
if k == 'password':
continue
self.assertIn(k, response.body)
self.assertEqual(v, response.body[k])
def test_put_errors(self):
client = yield self.auth_client()
user = yield self.create_user(client)
cases = [
("login", False),
("login", [2, 3]),
("email", "@bbb.com"),
("email", {1: 2}),
("password", "123"),
("password", "1"),
("password", [1, '2']),
("password", {1: '2'}),
]
for i in range(1, len(cases)):
for case in itertools.combinations(cases, i):
body = dict(case)
with self.assertRaises(HTTPError) as err:
log.info("Body: %s", body)
yield client.fetch(
self.get_url("/api/v1/user/{0}".format(user['id'])),
"PUT", body
)
self.assertEqual(err.exception.code, 400)
def render_notebook(exporter, nbfile, nb, url=None, config=None):
if not isinstance(exporter, Exporter):
exporter_cls = exporter
if exporter_cls not in exporters:
app_log.info("instantiating %s" % exporter_cls.__name__)
exporters[exporter_cls] = exporter_cls(config=config, log=app_log)
exporter = exporters[exporter_cls]
css_theme = nb.get('metadata', {}).get('_nbviewer', {}).get('css', None)
if not css_theme or not css_theme.strip():
# whitespace
css_theme = None
# get the notebook title, if any
try:
name = nb.metadata.name
except AttributeError:
name = ''
if not name and url is not None:
name = url.rsplit('/')[-1]
if not name.endswith(".ipynb"):
name = name + ".ipynb"
html, resources = exporter.from_filename(nbfile)
config = {
'download_name': name,
'css_theme': css_theme,
}
return html, config
def get(self, gist_id, file=''):
new_url = '/gist/%s' % gist_id
if file:
new_url = "%s/%s" % (new_url, file)
app_log.info("Redirecting %s to %s", self.request.uri, new_url)
self.redirect(new_url)
def dealer_process():
app_log.info('start dealer process')
log_fds('start')
log_mem('start')
s = Storage()
q = Q()
while True:
log_fds('start loop')
log_mem('start loop')
domains = yield s.fetch_domains_for_update(options.dealer_domains_per_task)
if domains and len(domains) < options.dealer_domains_per_task:
time.sleep(options.dealer_fetch_task_sleep_period_sec)
domains = yield s.fetch_domains_for_update(options.dealer_domains_per_task)
if not domains:
app_log.info("not found domains")
time.sleep(options.dealer_sleep_period_sec)
continue
app_log.info("fetch %d domains for new task" % len(domains))
res = q.add_crawler_task(domains)
yield s.update_domains_after_fetch(domains)
app_log.info("add task %s" % res)
del domains
app_log.info('end dealer process')
def _on_pika_message(self, channel, method, props, body):
log.debug('PikaCient: Message received, delivery tag #%i : %r' % (method.delivery_tag, len(body)))
correlation_id = getattr(props, 'correlation_id', None)
if not self.callbacks_hash.has_key(correlation_id) and method.exchange != 'DLX':
log.info('Got result for task "{0}", but no has callback'.format(correlation_id))
return
cb = self.callbacks_hash.pop(correlation_id)
content_type = getattr(props, 'content_type', 'text/plain')
if method.exchange == 'DLX':
dl = props.headers['x-death'][0]
body = ExpirationError("Dead letter received. Reason: {0}".format(dl.get('reason')))
body.reason = dl.get('reason')
body.time = dl.get('time')
body.expiration = int(dl.get('original-expiration')) / 1000
else:
if props.content_encoding == 'gzip':
body = zlib.decompress(body)
if 'application/json' in content_type:
body = json.loads(body)
elif 'application/python-pickle' in content_type:
body = pickle.loads(body)
if isinstance(cb, Future):
if isinstance(body, Exception):
cb.set_exception(body)
else:
cb.set_result(body)
else:
out = cb(body, headers=props.headers)
return out
def get(self, gist_id, file=''):
new_url = '%s/gist/%s' % (self.format_prefix, gist_id)
if file:
new_url = "%s/%s" % (new_url, file)
app_log.info("Redirecting %s to %s", self.request.uri, new_url)
self.redirect(new_url)
def get(self, path=None):
'''Spawns a brand new server'''
try:
if path is None:
# No path. Assign a prelaunched container from the pool and redirect to it.
# Append self.redirect_uri to the redirect target.
container_path = self.pool.acquire().path
app_log.info("Allocated [%s] from the pool.", container_path)
url = "/{}/{}".format(container_path, self.redirect_uri)
else:
path_parts = path.lstrip('/').split('/', 1)
container_path = path_parts[0]
# Scrap a container from the pool and replace it with an ad-hoc replacement.
# This takes longer, but is necessary to support ad-hoc containers
yield self.pool.adhoc(container_path)
app_log.info("Allocated ad-hoc container at [%s].", container_path)
url = path
app_log.debug("Redirecting [%s] -> [%s].", self.request.path, url)
self.redirect(url, permanent=False)
except spawnpool.EmptyPoolError:
app_log.warning("The container pool is empty!")
self.render("full.html", cull_period=self.cull_period)
def release(self, container, replace_if_room=True):
'''Shut down a container and delete its proxy entry.
Destroy the container in an orderly fashion. If requested and capacity is remaining, create
a new one to take its place.'''
try:
app_log.info("Releasing container [%s].", container)
yield [
self.spawner.shutdown_notebook_server(container.id),
self._proxy_remove(container.path)
]
app_log.debug("Container [%s] has been released.", container)
except Exception as e:
app_log.error("Unable to release container [%s]: %s", container, e)
return
if replace_if_room:
running = yield self.spawner.list_notebook_servers(self.container_config,
all=False)
if len(running) + 1 <= self.capacity:
app_log.debug("Launching a replacement container.")
yield self._launch_container()
else:
app_log.info("Declining to launch a new container because [%i] containers are" +
" already running, and the capacity is [%i].",
len(running), self.capacity)
def get(self):
code = self.get_argument("code", False)
if not code:
raise HTTPError(400, "oauth callback made without a token")
# validate OAuth state
arg_state = self.get_argument("state", None)
cookie_state = self.get_secure_cookie(self.hub_auth.state_cookie_name)
next_url = None
if arg_state or cookie_state:
# clear cookie state now that we've consumed it
self.clear_cookie(self.hub_auth.state_cookie_name)
if isinstance(cookie_state, bytes):
cookie_state = cookie_state.decode('ascii', 'replace')
# check that state matches
if arg_state != cookie_state:
app_log.debug("oauth state %r != %r", arg_state, cookie_state)
raise HTTPError(403, "oauth state does not match")
next_url = self.hub_auth.get_next_url(cookie_state)
# TODO: make async (in a Thread?)
token = self.hub_auth.token_for_code(code)
user_model = self.hub_auth.user_for_token(token)
if user_model is None:
raise HTTPError(500, "oauth callback failed to identify a user")
app_log.info("Logged-in user %s", user_model)
self.hub_auth.set_cookie(self, token)
self.redirect(next_url or self.hub_auth.base_url)
def delete(self, uid, *args, **kwargs):
# dismiss a user
try:
admin = self.session.query(User).filter(
User.id == self.current_user,
User.is_admin == 1
).one()
except sqlalchemy.orm.exc.NoResultFound:
return self.send_error(status_code=404)
except sqlalchemy.orm.exc.MultipleResultsFound:
return self.finish("multiple uid %s found" % self.current_user)
if not uid:
self.finish("uid needed")
try:
user = self.session.query(User).filter(
User.id == uid,
User.is_present == 1
).one()
user.is_present = 0
self.session.commit()
app_log.info("[{ip}] User Dismiss: {admin} - {user}".format(ip=self.request.remote_ip,
admin=admin.email,
user=user.email))
self.finish("ok")
except sqlalchemy.orm.exc.NoResultFound:
self.finish("uid %s not found" % self.current_user)
except sqlalchemy.orm.exc.MultipleResultsFound:
self.finish("multiple uid %s found" % self.current_user)
def cull_idle(url, api_token, timeout):
"""cull idle single-user servers"""
auth_header = {
'Authorization': 'token %s' % api_token
}
req = HTTPRequest(url=url + '/api/users',
headers=auth_header,
)
now = datetime.datetime.utcnow()
cull_limit = now - datetime.timedelta(seconds=timeout)
client = AsyncHTTPClient()
resp = yield client.fetch(req)
users = json.loads(resp.body.decode('utf8', 'replace'))
futures = []
for user in users:
last_activity = parse_date(user['last_activity'])
if user['server'] and last_activity < cull_limit:
app_log.info("Culling %s (inactive since %s)", user['name'], last_activity)
req = HTTPRequest(url=url + '/api/users/%s/server' % user['name'],
method='DELETE',
headers=auth_header,
)
futures.append((user['name'], client.fetch(req)))
elif user['server'] and last_activity > cull_limit:
app_log.debug("Not culling %s (active since %s)", user['name'], last_activity)
for (name, f) in futures:
yield f
app_log.debug("Finished culling %s", name)
def main():
tornado.options.parse_command_line()
WebSocket.init_pool()
http_server = tornado.httpserver.HTTPServer(Application())
http_server.listen(options.port, address=options.listen)
log.info('Server started {host}:{port}'.format(host=options.listen, port=options.port))
tornado.ioloop.IOLoop.instance().start()
def main():
options.define('port', default=9005,
help="Port for the REST API"
)
options.define('ip', default='127.0.0.1',
help="IP address for the REST API"
)
options.define('host_mount', default='',
help='Path where the host root is mounted in this container'
)
options.define('pool_prefix', default='',
help='Prefix assigned by tmpnb to its pooled containers'
)
options.define('registration_key', default='',
help='Registration key required to create new volumes'
)
options.parse_command_line()
opts = options.options
# regex from docker volume create
api_handlers = [
(r'/api/mounts(/([a-zA-Z0-9][a-zA-Z0-9_.-])+)?', MountsHandler),
(r'/api/volumes', VolumesHander),
]
api_app = web.Application(api_handlers)
api_app.listen(opts.port, opts.ip, xheaders=True)
app_log.info("Listening on {}:{}".format(opts.ip, opts.port))
ioloop.IOLoop.instance().start()
def get(self, user, repo, ref, path):
raw_url = u"https://raw.github.com/{user}/{repo}/{ref}/{path}".format(
user=user, repo=repo, ref=ref, path=quote(path)
)
blob_url = u"https://github.com/{user}/{repo}/blob/{ref}/{path}".format(
user=user, repo=repo, ref=ref, path=quote(path),
)
with self.catch_client_error():
response = yield self.client.fetch(raw_url)
if response.effective_url.startswith("https://github.com/{user}/{repo}/tree".format(
user=user, repo=repo
)):
tree_url = "/github/{user}/{repo}/tree/{ref}/{path}/".format(
user=user, repo=repo, ref=ref, path=quote(path),
)
app_log.info("%s is a directory, redirecting to %s", raw_url, tree_url)
self.redirect(tree_url)
return
filedata = response.body
if path.endswith('.ipynb'):
try:
nbjson = response_text(response)
except Exception as e:
app_log.error("Failed to decode notebook: %s", raw_url, exc_info=True)
raise web.HTTPError(400)
yield self.finish_notebook(nbjson, raw_url,
home_url=blob_url,
msg="file from GitHub: %s" % raw_url,
)
else:
self.set_header("Content-Type", "text/plain")
self.cache_and_finish(filedata)
def get(self):
error = self.get_argument("error", False)
if error:
msg = self.get_argument("error_description", error)
raise HTTPError(400, "Error in oauth: %s" % msg)
code = self.get_argument("code", False)
if not code:
raise HTTPError(400, "oauth callback made without a token")
# validate OAuth state
arg_state = self.get_argument("state", None)
if arg_state is None:
raise HTTPError("oauth state is missing. Try logging in again.")
cookie_name = self.hub_auth.get_state_cookie_name(arg_state)
cookie_state = self.get_secure_cookie(cookie_name)
# clear cookie state now that we've consumed it
self.clear_cookie(cookie_name, path=self.hub_auth.base_url)
if isinstance(cookie_state, bytes):
cookie_state = cookie_state.decode('ascii', 'replace')
# check that state matches
if arg_state != cookie_state:
app_log.warning("oauth state %r != %r", arg_state, cookie_state)
raise HTTPError(403, "oauth state does not match. Try logging in again.")
next_url = self.hub_auth.get_next_url(cookie_state)
# TODO: make async (in a Thread?)
token = self.hub_auth.token_for_code(code)
session_id = self.hub_auth.get_session_id(self)
user_model = self.hub_auth.user_for_token(token, session_id=session_id)
if user_model is None:
raise HTTPError(500, "oauth callback failed to identify a user")
app_log.info("Logged-in user %s", user_model)
self.hub_auth.set_cookie(self, token)
self.redirect(next_url or self.hub_auth.base_url)
def show(handlers, url_width, handler_width):
app_log.info("=" * int(url_width + handler_width))
app_log.info("%-*s%-*s" % (url_width, "URL", handler_width, "HANDLER"))
app_log.info("=" * int(url_width + handler_width))
for url, _ in handlers:
app_log.info("%-*s%-*s" % (url_width, url, handler_width, _))
app_log.info("=" * int(url_width + handler_width))
def post(self):
'''
Creates a new docker volume for the user protected by the given password
if the proper registration key is provided. Errors if the key is
incorrect/missing, if the volume already exists, or if the volume cannot
be created for any other reason.
'''
body = json.loads(self.request.body)
registration_key = body['registration_key']
username = body['username']
password = body['password']
required_key = options.options.registration_key
if not required_key or required_key == registration_key:
# Hash the username as the volume prefix
volume_prefix = username_to_volume_prefix(username)
exists = yield find_volume(volume_prefix)
if exists:
# Error if the volume already exists
raise web.HTTPError(409, 'volume %s exists' % volume_prefix)
volume_suffix = password_to_volume_suffix(password)
created = yield create_volume(volume_prefix, volume_suffix)
if not created:
# Error if volume creation failed
raise web.HTTPError(500, 'unable to create volume')
else:
raise web.HTTPError(401, 'invalid registration key %s', registration_key)
app_log.info('created volume prefix %s', volume_prefix)
# All good if we get here
self.set_status(201)
self.finish()
def main():
tornado.options.define('cull_period',
default=600,
help="Interval (s) for culling idle containers.")
tornado.options.define('cull_timeout',
default=3600,
help="Timeout (s) for culling idle containers.")
tornado.options.define('cull_max',
default=14400,
help=dedent("""
Maximum age of a container (s), regardless of activity.
Default: 14400 (4 hours)
A container that has been running for this long will be culled,
even if it is not idle.
"""))
tornado.options.define(
'container_ip',
default='127.0.0.1',
help="""Host IP address for containers to bind to. If host_network=True,
the host IP address for notebook servers to bind to.""")
tornado.options.define(
'container_port',
default='8888',
help="""Within container port for notebook servers to bind to.
If host_network=True, the starting port assigned to notebook servers on the host."""
)
tornado.options.define(
'use_tokens',
default=False,
help="""Enable token-authentication of notebook servers.
If host_network=True, the starting port assigned to notebook servers on the host."""
)
command_default = ('jupyter notebook --no-browser'
' --port {port} --ip=0.0.0.0 --allow-root'
' --NotebookApp.base_url={base_path}'
' --NotebookApp.port_retries=0'
' --NotebookApp.token="{token}"')
tornado.options.define(
'command',
default=command_default,
help="""Command to run when booting the image. A placeholder for
{base_path} should be provided. A placeholder for {port} and {ip} can be provided."""
)
tornado.options.define('port',
default=9999,
help="port for the main server to listen on")
tornado.options.define(
'ip',
default=None,
help="ip for the main server to listen on [default: all interfaces]")
tornado.options.define('admin_port',
default=10000,
help="port for the admin server to listen on")
tornado.options.define(
'admin_ip',
default='127.0.0.1',
help="ip for the admin server to listen on [default: 127.0.0.1]")
tornado.options.define('max_dock_workers',
default=2,
help="Maximum number of docker workers")
tornado.options.define('mem_limit',
default="512m",
help="Limit on Memory, per container")
tornado.options.define('cpu_shares',
default=None,
type=int,
help="Limit CPU shares, per container")
tornado.options.define('cpu_quota',
default=None,
type=int,
help=dedent("""
Limit CPU quota, per container.
Units are CPU-µs per 100ms, so 1 CPU/container would be:
--cpu-quota=100000
"""))
tornado.options.define(
'image',
default="wodeai/tensorflow",
help=
"Docker container to spawn for new users. Must be on the system already"
)
tornado.options.define('docker_version',
default="auto",
help="Version of the Docker API to use")
tornado.options.define(
'redirect_uri',
default="/tree",
help="URI to redirect users to upon initial notebook launch")
tornado.options.define(
'pool_size',
default=10,
help=
"Capacity for containers on this system. Will be prelaunched at startup."
)
tornado.options.define(
'pool_name',
default=None,
help=
"Container name fragment used to identity containers that belong to this instance."
)
tornado.options.define(
'static_files',
default=None,
help="Static files to extract from the initial container launch")
tornado.options.define(
'allow_origin',
default=None,
help=
"Set the Access-Control-Allow-Origin header. Use '*' to allow any origin to access."
)
tornado.options.define(
'expose_headers',
default=None,
help="Sets the Access-Control-Expose-Headers header.")
tornado.options.define('max_age',
default=None,
help="Sets the Access-Control-Max-Age header.")
tornado.options.define(
'allow_credentials',
default=None,
help="Sets the Access-Control-Allow-Credentials header.")
tornado.options.define(
'allow_methods',
default=None,
help="Sets the Access-Control-Allow-Methods header.")
tornado.options.define(
'allow_headers',
default=None,
help="Sets the Access-Control-Allow-Headers header.")
tornado.options.define('assert_hostname',
default=False,
help="Verify hostname of Docker daemon.")
tornado.options.define('container_user',
default=None,
help="User to run container command as")
tornado.options.define(
'host_network',
default=False,
help="""Attaches the containers to the host networking instead of the
default docker bridge. Affects the semantics of container_port and container_ip."""
)
tornado.options.define(
'docker_network',
default=None,
help="""Attaches the containers to the specified docker network.
For use when the proxy, tmpnb, and containers are all in docker.""")
tornado.options.define('host_directories',
default=None,
help=dedent("""
Mount the specified directory as a data volume, multiple
directories can be specified by using a comma-delimited string, directory
path must provided in full (eg: /home/steve/data/:r), permissions default to
rw"""))
tornado.options.define(
'user_length',
default=12,
help="Length of the unique /user/:id path generated per container")
tornado.options.define('extra_hosts',
default=[],
multiple=True,
help=dedent("""
Extra hosts for the containers, multiple hosts can be specified
by using a comma-delimited string, specified in the form hostname:IP"""
))
tornado.options.parse_command_line()
opts = tornado.options.options
api_token = os.getenv('API_AUTH_TOKEN')
admin_token = os.getenv('ADMIN_AUTH_TOKEN')
proxy_token = os.environ['CONFIGPROXY_AUTH_TOKEN']
proxy_endpoint = os.environ.get('CONFIGPROXY_ENDPOINT',
"http://127.0.0.1:8001")
docker_host = os.environ.get('DOCKER_HOST', 'unix://var/run/docker.sock')
handlers = [
(r"/api/spawn/?", APISpawnHandler),
(r"/api/stats/?", APIStatsHandler),
(r"/stats/?", RedirectHandler, {
"url": "/api/stats"
}),
]
# Only add human-facing handlers if there's no spawn API key set
if api_token is None:
handlers.extend([
(r"/", LoadingHandler),
(r"/spawn/?(/user/\w+(?:/.*)?)?", SpawnHandler),
(r"/spawn/((?:notebooks|tree|files)(?:/.*)?)", SpawnHandler),
(r"/(user/\w+)(?:/.*)?", LoadingHandler),
(r"/((?:notebooks|tree|files)(?:/.*)?)", LoadingHandler),
(r"/info/?", InfoHandler),
])
admin_handlers = [(r"/api/pool/?", APIPoolHandler)]
max_idle = datetime.timedelta(seconds=opts.cull_timeout)
max_age = datetime.timedelta(seconds=opts.cull_max)
pool_name = opts.pool_name
if pool_name is None:
# Derive a valid container name from the image name by default.
pool_name = re.sub('[^a-zA-Z0_.-]+', '', opts.image.split(':')[0])
container_config = dockworker.ContainerConfig(
image=opts.image,
command=opts.command,
use_tokens=opts.use_tokens,
mem_limit=opts.mem_limit,
cpu_quota=opts.cpu_quota,
cpu_shares=opts.cpu_shares,
container_ip=opts.container_ip,
container_port=opts.container_port,
container_user=opts.container_user,
host_network=opts.host_network,
docker_network=opts.docker_network,
host_directories=opts.host_directories,
extra_hosts=opts.extra_hosts,
)
spawner = dockworker.DockerSpawner(
docker_host,
timeout=30,
version=opts.docker_version,
max_workers=opts.max_dock_workers,
assert_hostname=opts.assert_hostname,
)
static_path = os.path.join(os.path.dirname(__file__), "static")
pool = spawnpool.SpawnPool(proxy_endpoint=proxy_endpoint,
proxy_token=proxy_token,
spawner=spawner,
container_config=container_config,
capacity=opts.pool_size,
max_idle=max_idle,
max_age=max_age,
static_files=opts.static_files,
static_dump_path=static_path,
pool_name=pool_name,
user_length=opts.user_length)
ioloop = tornado.ioloop.IOLoop().current()
settings = dict(
default_handler_class=BaseHandler,
static_path=static_path,
cookie_secret=uuid.uuid4(),
xsrf_cookies=False,
debug=True,
cull_period=opts.cull_period,
allow_origin=opts.allow_origin,
expose_headers=opts.expose_headers,
max_age=opts.max_age,
allow_credentials=opts.allow_credentials,
allow_methods=opts.allow_methods,
allow_headers=opts.allow_headers,
spawner=spawner,
pool=pool,
autoescape=None,
proxy_token=proxy_token,
api_token=api_token,
template_path=os.path.join(os.path.dirname(__file__), 'templates'),
proxy_endpoint=proxy_endpoint,
redirect_uri=opts.redirect_uri.lstrip('/'),
)
admin_settings = dict(pool=pool, admin_token=admin_token)
# Cleanup on a fresh state (likely a restart)
ioloop.run_sync(pool.cleanout)
# Synchronously cull any existing, inactive containers, and pre-launch a set number of
# containers, ready to serve.
ioloop.run_sync(pool.heartbeat)
if (opts.static_files):
ioloop.run_sync(pool.copy_static)
# Periodically execute a heartbeat function to cull used containers and regenerated failed
# ones, self-healing the cluster.
cull_ms = opts.cull_period * 1e3
app_log.info("Culling containers unused for %i seconds every %i seconds.",
opts.cull_timeout, opts.cull_period)
culler = tornado.ioloop.PeriodicCallback(pool.heartbeat, cull_ms)
culler.start()
app_log.info("Listening on {}:{}".format(opts.ip or '*', opts.port))
application = tornado.web.Application(handlers, **settings)
application.listen(opts.port, opts.ip)
app_log.info("Admin listening on {}:{}".format(opts.admin_ip or '*',
opts.admin_port))
admin_application = tornado.web.Application(admin_handlers,
**admin_settings)
admin_application.listen(opts.admin_port, opts.admin_ip)
ioloop.start()
def delete(self):
'''Drains available containers from the pool.'''
n = yield self.pool.drain()
app_log.info('Drained pool of %d containers', n)
self.finish(dict(drained=n))
def show_dir(self, fullpath, path, **namespace):
"""Render the directory view template for a given filesystem path.
Parameters
==========
fullpath: string
Absolute path on disk to show
path: string
URL path equating to the path on disk
Returns
=======
str
Rendered HTML
"""
entries = []
dirs = []
ipynbs = []
try:
contents = os.listdir(fullpath)
except IOError as ex:
if ex.errno == errno.EACCES:
# py2/3: can't access the dir, so don't give away its presence
app_log.info(
"contents of path: '%s' cannot be listed from within nbviewer",
fullpath)
raise web.HTTPError(404)
for f in contents:
absf = os.path.join(fullpath, f)
if not self.can_show(absf):
continue
entry = {}
entry['name'] = f
# We need to make UTC timestamps conform to true ISO-8601 by
# appending Z(ulu). Without a timezone, the spec says it should be
# treated as local time which is not what we want and causes
# moment.js on the frontend to show times in the past or future
# depending on the user's timezone.
# https://en.wikipedia.org/wiki/ISO_8601#Time_zone_designators
if os.path.isdir(absf):
st = os.stat(absf)
dt = datetime.utcfromtimestamp(st.st_mtime)
entry['modtime'] = dt.isoformat() + 'Z'
entry['url'] = url_path_join(self._localfile_path, path, f)
entry['class'] = 'fa fa-folder-open'
dirs.append(entry)
elif f.endswith('.ipynb'):
st = os.stat(absf)
dt = datetime.utcfromtimestamp(st.st_mtime)
entry['modtime'] = dt.isoformat() + 'Z'
entry['url'] = url_path_join(self._localfile_path, path, f)
entry['class'] = 'fa fa-book'
ipynbs.append(entry)
dirs.sort(key=lambda e: e['name'])
ipynbs.sort(key=lambda e: e['name'])
entries.extend(dirs)
entries.extend(ipynbs)
html = self.render_dirview_template(entries=entries,
breadcrumbs=self.breadcrumbs(path),
title=url_path_join(path, '/'),
**namespace)
return html
def on_close(self):
if hasattr(self, 'messager'):
self.messager.destroy()
Statistics.CONNECTIONS -= 1
app_log.info('online connections %d', Statistics.CONNECTIONS)
def on_close(self):
app_log.info('close socket %s', self)
self.sockets.remove(self)
for i in self.subscriptions:
self.subscriptions[i].dispose()
self.subscriptions = {}
def handle_server(user, server_name, server):
"""Handle (maybe) culling a single server
Returns True if server is now stopped (user removable),
False otherwise.
"""
log_name = user['name']
if server_name:
log_name = '%s/%s' % (user['name'], server_name)
if server.get('pending'):
app_log.warning("Not culling server %s with pending %s", log_name,
server['pending'])
return False
# jupyterhub < 0.9 defined 'server.url' once the server was ready
# as an *implicit* signal that the server was ready.
# 0.9 adds a dedicated, explicit 'ready' field.
# By current (0.9) definitions, servers that have no pending
# events and are not ready shouldn't be in the model,
# but let's check just to be safe.
if not server.get('ready', bool(server['url'])):
app_log.warning("Not culling not-ready not-pending server %s: %s",
log_name, server)
return False
if server.get('started'):
age = now - parse_date(server['started'])
else:
# started may be undefined on jupyterhub < 0.9
age = None
# check last activity
# last_activity can be None in 0.9
if server['last_activity']:
inactive = now - parse_date(server['last_activity'])
else:
# no activity yet, use start date
# last_activity may be None with jupyterhub 0.9,
# which introduces the 'started' field which is never None
# for running servers
inactive = age
should_cull = (inactive is not None
and inactive.total_seconds() >= inactive_limit)
if should_cull:
app_log.info("Culling server %s (inactive for %s)", log_name,
format_td(inactive))
if max_age and not should_cull:
# only check started if max_age is specified
# so that we can still be compatible with jupyterhub 0.8
# which doesn't define the 'started' field
if age is not None and age.total_seconds() >= max_age:
app_log.info(
"Culling server %s (age: %s, inactive for %s)",
log_name,
format_td(age),
format_td(inactive),
)
should_cull = True
if not should_cull:
app_log.debug(
"Not culling server %s (age: %s, inactive for %s)",
log_name,
format_td(age),
format_td(inactive),
)
return False
if server_name:
# culling a named server
delete_url = url + "/users/%s/servers/%s" % (
quote(user['name']),
quote(server['name']),
)
else:
delete_url = url + '/users/%s/server' % quote(user['name'])
req = HTTPRequest(url=delete_url, method='DELETE', headers=auth_header)
resp = yield fetch(req)
if resp.code == 202:
app_log.warning("Server %s is slow to stop", log_name)
# return False to prevent culling user with pending shutdowns
return False
return True
health="https://ovh.mybinder.org/health",
# health="https://httpbin.org/status/404",
versions="https://ovh.mybinder.org/versions",
),
"gesis": dict(
url="https://notebooks.gesis.org/binder",
weight=1,
health="https://notebooks.gesis.org/binder/health",
versions="https://notebooks.gesis.org/binder/versions",
),
},
}
config_path = "/etc/federation-redirect/config.json"
if os.path.exists(config_path):
app_log.info("Using config from '{}'.".format(config_path))
with open(config_path) as f:
CONFIG = json.load(f)
else:
app_log.warning("Using default config!")
for h in list(CONFIG["hosts"].keys()):
# Remove empty entries from CONFIG["hosts"], these can happen because we
# can't remove keys in our helm templates/config files. All we can do is
# set them to Null/None. We need to turn the keys into a list so that we
# can modify the dict while iterating over it
if CONFIG["hosts"][h] is None:
CONFIG["hosts"].pop(h)
# remove trailing slashes in host urls
# these can cause 404 after redirection (RedirectHandler) and we don't realize it
else:
async def launch(self, kube, provider):
"""Ask JupyterHub to launch the image."""
# Load the spec-specific configuration if it has been overridden
repo_config = provider.repo_config(self.settings)
# the image name (without tag) is unique per repo
# use this to count the number of pods running with a given repo
# if we added annotations/labels with the repo name via KubeSpawner
# we could do this better
image_no_tag = self.image_name.rsplit(':', 1)[0]
matching_pods = 0
total_pods = 0
# TODO: run a watch to keep this up to date in the background
pool = self.settings['executor']
f = pool.submit(
kube.list_namespaced_pod,
self.settings["build_namespace"],
label_selector='app=jupyterhub,component=singleuser-server',
)
# concurrent.futures.Future isn't awaitable
# wrap in tornado Future
# tornado 5 will have `.run_in_executor`
tf = Future()
chain_future(f, tf)
pods = await tf
for pod in pods.items:
total_pods += 1
for container in pod.spec.containers:
# is the container running the same image as us?
# if so, count one for the current repo.
image = container.image.rsplit(':', 1)[0]
if image == image_no_tag:
matching_pods += 1
break
# TODO: put busy users in a queue rather than fail?
# That would be hard to do without in-memory state.
quota = repo_config.get('quota')
if quota and matching_pods >= quota:
app_log.error("%s has exceeded quota: %s/%s (%s total)",
self.repo_url, matching_pods, quota, total_pods)
await self.fail("Too many users running %s! Try again soon." %
self.repo_url)
return
if quota and matching_pods >= 0.5 * quota:
log = app_log.warning
else:
log = app_log.info
log("Launching pod for %s: %s other pods running this repo (%s total)",
self.repo_url, matching_pods, total_pods)
await self.emit({
'phase': 'launching',
'message': 'Launching server...\n',
})
launcher = self.settings['launcher']
retry_delay = launcher.retry_delay
for i in range(launcher.retries):
launch_starttime = time.perf_counter()
if self.settings['auth_enabled']:
# get logged in user's name
user_model = self.hub_auth.get_user(self)
username = user_model['name']
if launcher.allow_named_servers:
# user can launch multiple servers, so create a unique server name
server_name = launcher.unique_name_from_repo(self.repo_url)
else:
server_name = ''
else:
# create a name for temporary user
username = launcher.unique_name_from_repo(self.repo_url)
server_name = ''
try:
extra_args = {
'binder_ref_url': self.ref_url,
'binder_launch_host': self.binder_launch_host,
'binder_request': self.binder_request,
'binder_persistent_request':
self.binder_persistent_request,
}
server_info = await launcher.launch(image=self.image_name,
username=username,
server_name=server_name,
repo_url=self.repo_url,
extra_args=extra_args)
except Exception as e:
duration = time.perf_counter() - launch_starttime
if i + 1 == launcher.retries:
status = 'failure'
else:
status = 'retry'
# don't count retries in failure/retry
# retry count is only interesting in success
LAUNCH_TIME.labels(
status=status,
retries=-1,
).observe(time.perf_counter() - launch_starttime)
if status == 'failure':
# don't count retries per repo
LAUNCH_COUNT.labels(
status=status,
**self.repo_metric_labels,
).inc()
if i + 1 == launcher.retries:
# last attempt failed, let it raise
raise
# not the last attempt, try again
app_log.error(
"Retrying launch of %s after error (duration=%.0fs, attempt=%s): %r",
self.repo_url,
duration,
i + 1,
e,
)
await self.emit({
"phase":
"launching",
"message":
"Launch attempt {} failed, retrying...\n".format(i + 1),
})
await gen.sleep(retry_delay)
# exponential backoff for consecutive failures
retry_delay *= 2
continue
else:
# success
duration = time.perf_counter() - launch_starttime
LAUNCH_TIME.labels(status="success",
retries=i).observe(duration)
LAUNCH_COUNT.labels(
status='success',
**self.repo_metric_labels,
).inc()
app_log.info("Launched %s in %.0fs", self.repo_url, duration)
break
event = {
'phase': 'ready',
'message': 'server running at %s\n' % server_info['url'],
}
event.update(server_info)
await self.emit(event)
def get_deleter():
"""Get cached global deletion thread"""
app_log.info("Creating global deletion thread")
return Deleter()
def find_inactive_devices(db, cutoff=IDLE_CUTOFF):
"""Check for device ids that need deleting"""
app_log.info(f"Checking for devices inactive since {cutoff}")
cur = db.execute(r"{CALL getLastActivityBefore(?)}", (cutoff,))
return cur.fetchall()
async def process_one(uuid_activity):
uuid, last_activity = uuid_activity
group = await graph.get_group(uuid)
if group is None:
app_log.info(f"No group for inactive device {uuid}")
counts["no_group"] += 1
return
if group.get(consent_revoked):
app_log.info(f"Already marked for deletion: {uuid}")
counts["deleted"] += 1
return
user = await graph.user_for_device(group)
if user is None:
app_log.info(f"No user for inactive device {uuid}")
counts["no_user"] += 1
# FIXME: something went wrong. Mark device id group for deletion?
return
# check other devices on the same user
# in case of new device registrations,
# don't delete data from a user's old phone
other_device_activity = None
device_ids = [uuid]
async for group in graph.device_groups_for_user(user):
device_id = group["displayName"]
if device_id != uuid:
device_ids.append(device_id)
# First check for recent registration (cheap)
if parse_date(group["createdDateTime"]) >= IDLE_CUTOFF:
app_log.info(f"Recently registered device {device_id}")
counts["new"] += 1
if device_id == uuid:
app_log.warning(
f"WRONG activity: recently registered {device_id} not idle"
)
counts["wrong"] += 1
other_device_activity = True
break
try:
device = await devices.get_device(device_id)
except Exception as e:
app_log.warning(f"Failed to get device {device_id}: ({e})")
counts["iot_err"] += 1
pass
else:
if parse_date(device["lastActivityTime"]) >= IDLE_CUTOFF:
counts["iot"] += 1
app_log.info(f"Activity on {device_id} in IoTHub")
if device_id == uuid:
app_log.warning(
f"WRONG activity: iothub active {device_id} not idle"
)
counts["wrong"] += 1
other_device_activity = True
break
if device_id != uuid:
# if not registered since cutoff, check for activity in SQL
if await check_sql_data(device_id, activity_cutoff=IDLE_CUTOFF):
counts["sql"] += 1
app_log.info(f"Activity on {device_id} in SQL")
other_device_activity = True
break
if other_device_activity:
app_log.info(f"{uuid} is associated with other more recent device activity")
counts["active"] += 1
else:
app_log.info(f"User {user['logName']} is inactive since {last_activity}")
app_log.info(f"Inactive devices: {','.join(device_ids)}")
counts["idle"] += 1
return user
async def find_users_to_delete(limit=None):
"""Find users whose devices should be deleted"""
async for user in graph.list_users(filter=f"{consent_revoked} eq true"):
if user.get(consent_revoked):
app_log.warning(f"User {user['logName']} with consent revoked not removed!")
yield user
else:
app_log.warning(
f"User {user['logName']} without consent revoked shouldn't have been returned by query."
)
inactive = await find_inactive_devices()
app_log.info(f"Found {len(inactive)} inactive devices")
counts = defaultdict(int)
async def process_one(uuid_activity):
uuid, last_activity = uuid_activity
group = await graph.get_group(uuid)
if group is None:
app_log.info(f"No group for inactive device {uuid}")
counts["no_group"] += 1
return
if group.get(consent_revoked):
app_log.info(f"Already marked for deletion: {uuid}")
counts["deleted"] += 1
return
user = await graph.user_for_device(group)
if user is None:
app_log.info(f"No user for inactive device {uuid}")
counts["no_user"] += 1
# FIXME: something went wrong. Mark device id group for deletion?
return
# check other devices on the same user
# in case of new device registrations,
# don't delete data from a user's old phone
other_device_activity = None
device_ids = [uuid]
async for group in graph.device_groups_for_user(user):
device_id = group["displayName"]
if device_id != uuid:
device_ids.append(device_id)
# First check for recent registration (cheap)
if parse_date(group["createdDateTime"]) >= IDLE_CUTOFF:
app_log.info(f"Recently registered device {device_id}")
counts["new"] += 1
if device_id == uuid:
app_log.warning(
f"WRONG activity: recently registered {device_id} not idle"
)
counts["wrong"] += 1
other_device_activity = True
break
try:
device = await devices.get_device(device_id)
except Exception as e:
app_log.warning(f"Failed to get device {device_id}: ({e})")
counts["iot_err"] += 1
pass
else:
if parse_date(device["lastActivityTime"]) >= IDLE_CUTOFF:
counts["iot"] += 1
app_log.info(f"Activity on {device_id} in IoTHub")
if device_id == uuid:
app_log.warning(
f"WRONG activity: iothub active {device_id} not idle"
)
counts["wrong"] += 1
other_device_activity = True
break
if device_id != uuid:
# if not registered since cutoff, check for activity in SQL
if await check_sql_data(device_id, activity_cutoff=IDLE_CUTOFF):
counts["sql"] += 1
app_log.info(f"Activity on {device_id} in SQL")
other_device_activity = True
break
if other_device_activity:
app_log.info(f"{uuid} is associated with other more recent device activity")
counts["active"] += 1
else:
app_log.info(f"User {user['logName']} is inactive since {last_activity}")
app_log.info(f"Inactive devices: {','.join(device_ids)}")
counts["idle"] += 1
return user
yielded = 0
async for user in consume_concurrently(
inactive, process_one, counts=counts, label="Inactive users"
):
if user:
yield user
yielded += 1
if limit and yielded >= limit:
app_log.info(f"Reached idle user limit={limit}")
return
async def launch(self, provider):
"""Ask JupyterHub to launch the image."""
# Load the spec-specific configuration if it has been overridden
repo_config = provider.repo_config(self.settings)
# the image name (without tag) is unique per repo
# use this to count the number of pods running with a given repo
# if we added annotations/labels with the repo name via KubeSpawner
# we could do this better
image_no_tag = self.image_name.rsplit(':', 1)[0]
# TODO: put busy users in a queue rather than fail?
# That would be hard to do without in-memory state.
quota = repo_config.get('quota')
if quota:
# Fetch info on currently running users *only* if quotas are set
matching_pods = 0
total_pods = 0
# TODO: run a watch to keep this up to date in the background
f = self.settings['executor'].submit(
self.settings['kubernetes_client'].list_namespaced_pod,
self.settings['build_namespace'],
label_selector='app=jupyterhub,component=singleuser-server',
_request_timeout=KUBE_REQUEST_TIMEOUT,
_preload_content=False,
)
resp = await asyncio.wrap_future(f)
pods = json.loads(resp.read())
for pod in pods["items"]:
total_pods += 1
for container in pod["spec"]["containers"]:
# is the container running the same image as us?
# if so, count one for the current repo.
image = container["image"].rsplit(":", 1)[0]
if image == image_no_tag:
matching_pods += 1
break
if matching_pods >= quota:
app_log.error(
f"{self.repo_url} has exceeded quota: {matching_pods}/{quota} ({total_pods} total)"
)
await self.fail(
f"Too many users running {self.repo_url}! Try again soon.")
return
if matching_pods >= 0.5 * quota:
log = app_log.warning
else:
log = app_log.info
log(
"Launching pod for %s: %s other pods running this repo (%s total)",
self.repo_url, matching_pods, total_pods)
await self.emit({
'phase': 'launching',
'message': 'Launching server...\n',
})
launcher = self.settings['launcher']
retry_delay = launcher.retry_delay
for i in range(launcher.retries):
launch_starttime = time.perf_counter()
if self.settings['auth_enabled']:
# get logged in user's name
user_model = self.hub_auth.get_user(self)
username = user_model['name']
if launcher.allow_named_servers:
# user can launch multiple servers, so create a unique server name
server_name = launcher.unique_name_from_repo(self.repo_url)
else:
server_name = ''
else:
# create a name for temporary user
username = launcher.unique_name_from_repo(self.repo_url)
server_name = ''
try:
extra_args = {
'binder_ref_url': self.ref_url,
'binder_launch_host': self.binder_launch_host,
'binder_request': self.binder_request,
'binder_persistent_request':
self.binder_persistent_request,
}
server_info = await launcher.launch(image=self.image_name,
username=username,
server_name=server_name,
repo_url=self.repo_url,
extra_args=extra_args)
except Exception as e:
duration = time.perf_counter() - launch_starttime
if i + 1 == launcher.retries:
status = 'failure'
else:
status = 'retry'
# don't count retries in failure/retry
# retry count is only interesting in success
LAUNCH_TIME.labels(
status=status,
retries=-1,
).observe(time.perf_counter() - launch_starttime)
if status == 'failure':
# don't count retries per repo
LAUNCH_COUNT.labels(
status=status,
**self.repo_metric_labels,
).inc()
if i + 1 == launcher.retries:
# last attempt failed, let it raise
raise
# not the last attempt, try again
app_log.error(
"Retrying launch of %s after error (duration=%.0fs, attempt=%s): %r",
self.repo_url,
duration,
i + 1,
e,
)
await self.emit({
"phase":
"launching",
"message":
f"Launch attempt {i + 1} failed, retrying...\n",
})
await gen.sleep(retry_delay)
# exponential backoff for consecutive failures
retry_delay *= 2
continue
else:
# success
duration = time.perf_counter() - launch_starttime
LAUNCH_TIME.labels(status="success",
retries=i).observe(duration)
LAUNCH_COUNT.labels(
status='success',
**self.repo_metric_labels,
).inc()
app_log.info("Launched %s in %.0fs", self.repo_url, duration)
break
event = {
'phase': 'ready',
'message': f"server running at {server_info['url']}\n",
}
event.update(server_info)
await self.emit(event)
def recv(self, msg):
app_log.info('Message: {}'.format(msg))
self.local_pub.send_multipart(msg)
def create_notebook_server(self, base_path, container_name,
container_config):
'''Creates a notebook_server running off of `base_path`.
Returns the (container_id, ip, port) tuple in a Future.'''
if container_config.host_network:
# Start with specified container port
if self.port == 0:
self.port = int(container_config.container_port)
port = self.port
self.port += 1
# No bindings when using the host network
port_bindings = None
else:
# Bind the specified within-container port to a random port
# on the container-host IP address
port = container_config.container_port
port_bindings = {
container_config.container_port:
(container_config.container_ip, )
}
app_log.debug(container_config)
# Assumes that the container_config.command is of a format like:
#
# ipython notebook --no-browser --port {port} --ip=0.0.0.0
# --NotebookApp.base_path=/{base_path}
# --NotebookApp.tornado_settings=\"{ \"template_path\": [ \"/srv/ga\",
# \"/srv/ipython/IPython/html\",
# \"/srv/ipython/IPython/html/templates\" ] }\""
#
# Important piece here is the parametrized base_path to let the
# underlying process know where the proxy is routing it.
rendered_command = container_config.command.format(
base_path=base_path, port=port, ip=container_config.container_ip)
command = ["/bin/sh", "-c", rendered_command]
volume_bindings = {}
volumes = []
if container_config.host_directories:
directories = container_config.host_directories.split(",")
for index, item in enumerate(directories):
directory = item.split(":")[0]
try:
permissions = item.split(":")[1]
except IndexError:
permissions = 'rw'
volumes.append('/mnt/vol' + str(index))
volume_bindings[directory] = {
'bind': '/mnt/vol' + str(index),
'mode': permissions
}
host_config = dict(
mem_limit=container_config.mem_limit,
network_mode='host' if container_config.host_network else 'bridge',
binds=volume_bindings,
port_bindings=port_bindings)
host_config = create_host_config(**host_config)
cpu_shares = None
if container_config.cpu_shares:
# Some versions of Docker and docker-py won't cast from string to int
cpu_shares = int(container_config.cpu_shares)
resp = yield self._with_retries(self.docker_client.create_container,
image=container_config.image,
user=container_config.container_user,
command=command,
volumes=volumes,
host_config=host_config,
cpu_shares=cpu_shares,
name=container_name)
docker_warnings = resp.get('Warnings')
if docker_warnings is not None:
app_log.warn(docker_warnings)
container_id = resp['Id']
app_log.info("Created container {}".format(container_id))
yield self._with_retries(self.docker_client.start, container_id)
if container_config.host_network:
host_port = port
host_ip = container_config.container_ip
else:
container_network = yield self._with_retries(
self.docker_client.port, container_id,
container_config.container_port)
host_port = container_network[0]['HostPort']
host_ip = container_network[0]['HostIp']
raise gen.Return((container_id, host_ip, int(host_port)))
def main():
tornado.options.define('cull_period',
default=600,
help="Interval (s) for culling idle containers.")
tornado.options.define('cull_timeout',
default=3600,
help="Timeout (s) for culling idle containers.")
tornado.options.define('container_ip',
default='127.0.0.1',
help="IP address for containers to bind to")
tornado.options.define('container_port',
default='8888',
help="Port for containers to bind to")
command_default = ('ipython3 notebook --no-browser'
' --port {port} --ip=0.0.0.0'
' --NotebookApp.base_url=/{base_path}')
tornado.options.define(
'command',
default=command_default,
help=
"command to run when booting the image. A placeholder for base_path should be provided."
)
tornado.options.define('port',
default=9999,
help="port for the main server to listen on")
tornado.options.define(
'ip',
default=None,
help="ip for the main server to listen on [default: all interfaces]")
tornado.options.define('max_dock_workers',
default=2,
help="Maximum number of docker workers")
tornado.options.define('mem_limit',
default="512m",
help="Limit on Memory, per container")
tornado.options.define('cpu_shares',
default=None,
help="Limit CPU shares, per container")
tornado.options.define(
'image',
default="jupyter/minimal",
help=
"Docker container to spawn for new users. Must be on the system already"
)
tornado.options.define('docker_version',
default="1.13",
help="Version of the Docker API to use")
tornado.options.define(
'redirect_uri',
default="/tree",
help="URI to redirect users to upon initial notebook launch")
tornado.options.define(
'pool_size',
default=10,
help=
"Capacity for containers on this system. Will be prelaunched at startup."
)
tornado.options.define(
'pool_name',
default=None,
help=
"Container name fragment used to identity containers that belong to this instance."
)
tornado.options.define(
'static_files',
default=None,
help="Static files to extract from the initial container launch")
tornado.options.parse_command_line()
opts = tornado.options.options
handlers = [
(r"/", LoadingHandler),
(r"/spawn/?(/user/\w+(?:/.*)?)?", SpawnHandler),
(r"/(user/\w+)(?:/.*)?", LoadingHandler),
(r"/stats", StatsHandler),
]
proxy_token = os.environ['CONFIGPROXY_AUTH_TOKEN']
proxy_endpoint = os.environ.get('CONFIGPROXY_ENDPOINT',
"http://127.0.0.1:8001")
docker_host = os.environ.get('DOCKER_HOST', 'unix://var/run/docker.sock')
max_age = datetime.timedelta(seconds=opts.cull_timeout)
pool_name = opts.pool_name
if pool_name is None:
# Derive a valid container name from the image name by default.
pool_name = re.sub('[^a-zA-Z0_.-]+', '', opts.image.split(':')[0])
container_config = dockworker.ContainerConfig(
image=opts.image,
command=opts.command,
mem_limit=opts.mem_limit,
cpu_shares=opts.cpu_shares,
container_ip=opts.container_ip,
container_port=opts.container_port,
)
spawner = dockworker.DockerSpawner(
docker_host,
version=opts.docker_version,
timeout=30,
max_workers=opts.max_dock_workers,
)
static_path = os.path.join(os.path.dirname(__file__), "static")
pool = spawnpool.SpawnPool(
proxy_endpoint=proxy_endpoint,
proxy_token=proxy_token,
spawner=spawner,
container_config=container_config,
capacity=opts.pool_size,
max_age=max_age,
static_files=opts.static_files,
static_dump_path=static_path,
pool_name=pool_name,
)
ioloop = tornado.ioloop.IOLoop().instance()
settings = dict(
static_path=static_path,
cookie_secret=uuid.uuid4(),
xsrf_cookies=True,
debug=True,
cull_period=opts.cull_period,
spawner=spawner,
pool=pool,
autoescape=None,
proxy_token=proxy_token,
template_path=os.path.join(os.path.dirname(__file__), 'templates'),
proxy_endpoint=proxy_endpoint,
redirect_uri=opts.redirect_uri.lstrip('/'),
)
# Synchronously cull any existing, inactive containers, and pre-launch a set number of
# containers, ready to serve.
ioloop.run_sync(pool.heartbeat)
if (opts.static_files):
ioloop.run_sync(pool.copy_static)
# Periodically execute a heartbeat function to cull used containers and regenerated failed
# ones, self-healing the cluster.
cull_ms = opts.cull_period * 1e3
app_log.info("Culling containers unused for %i seconds every %i seconds.",
opts.cull_timeout, opts.cull_period)
culler = tornado.ioloop.PeriodicCallback(pool.heartbeat, cull_ms)
culler.start()
app_log.info("Listening on {}:{}".format(opts.ip or '*', opts.port))
application = tornado.web.Application(handlers, **settings)
application.listen(opts.port, opts.ip)
ioloop.start()
async def health_check(host, active_hosts):
check_config = CONFIG["check"]
all_hosts = CONFIG["hosts"]
app_log.info("Checking health of {}".format(host))
client = AsyncHTTPClient()
try:
for n in range(check_config["retries"]):
try:
# TODO we could use `asyncio.gather()` and fetch health and versions in parallel
# raises an `HTTPError` if the request returned a non-200 response code
# health url returns 503 if a (hard check) service is unhealthy
response = await client.fetch(
all_hosts[host]["health"], request_timeout=check_config["timeout"]
)
health = json.loads(response.body)
for check in health["checks"]:
# pod quota is a soft check
if check["service"] == "Pod quota":
if not check["ok"]:
raise Exception("{} is over its quota: {}".format(host, check))
break
# check versions
response = await client.fetch(
all_hosts[host]["versions"], request_timeout=check_config["timeout"]
)
versions = json.loads(response.body)
# if this is the prime host store the versions so we can compare to them later
if all_hosts[host].get("prime", False):
all_hosts["versions"] = versions
# check if this cluster is on the same versions as the prime
# w/o information about the prime's version we allow each
# cluster to be on its own versions
if versions != all_hosts.get("versions", versions):
raise Exception("{} has different versions ({}) than prime ({})".
format(host, versions, all_hosts["versions"]))
except Exception as e:
app_log.warning(
"{} failed, attempt {} of {}".format(
host, n + 1, check_config["retries"]
)
)
# raise the exception on the last attempt
if n == check_config["retries"] - 1:
raise
else:
await sleep(1)
# any kind of exception means the host is unhealthy
except Exception as e:
app_log.warning("{} is unhealthy".format(host))
if host in active_hosts:
# prime hosts are never removed, they always get traffic and users
# will see what ever healthy or unhealthy state they are in
# this protects us from the federation ending up with zero active
# hosts because of a glitch somewhere in the health checks
if all_hosts[host].get("prime", False):
app_log.warning(
"{} has NOT been removed because it is a prime ({})".format(
host, str(e)
)
)
else:
# remove the host from the rotation for a while
active_hosts.pop(host)
app_log.warning(
"{} has been removed from the rotation ({})".format(host, str(e))
)
# wait longer than usual to check unhealthy host again
jitter = check_config["jitter"] * (0.5 - random.random())
IOLoop.current().call_later(
30 * (1 + jitter) * check_config["period"], health_check, host, active_hosts
)
else:
if host not in active_hosts:
active_hosts[host] = all_hosts[host]
app_log.warning("{} has been added to the rotation".format(host))
# schedule ourselves to check again later
jitter = check_config["jitter"] * (0.5 - random.random())
IOLoop.current().call_later(
(1 + jitter) * check_config["period"], health_check, host, active_hosts
)
async def delete_test_user(user_id):
app_log.info(f"Delete test user {user_id}")
await graph.graph_request(f"/users/{user_id}", method="DELETE")
app_log.info(f"Deleted test user {user_id}")
def sync_db_tables(self):
dumps = dict()
db_path = os.path.join(static_path, 'syncDB', 'lasTables')
if not os.path.isdir(db_path):
os.makedirs(db_path)
flag_file = os.path.join(db_path, 'TAG')
old_date = ''
if os.path.isfile(flag_file):
with open(flag_file, 'r') as fp:
old_date = fp.read()
now_date = time.strftime('%Y%m%d')
if old_date != now_date:
env_list, total = yield self.project.get_projects_list(
p_type='env', status=1, search='准生产')
for env in env_list:
details, total = yield self.setting.get_settings_list(
s_type='env', name=env.name)
for detail in details:
desc = json.loads(detail.value)
if desc.get('type') != 'APPLICATION' and desc.get(
'title').upper().find('MYSQL') == -1:
continue
dumps[detail.id] = dict(
ip=desc.get('ip').strip(),
port=desc.get('port').strip(),
user=desc.get('user').strip(),
password=desc.get('password').strip(),
dbs=desc.get('description').split(','))
break
with open(flag_file, 'w') as fp:
fp.write(time.strftime('%Y%m%d'))
for key in dumps:
for db in dumps[key]['dbs']:
tmp_path = os.path.join(db_path, '{}_tmp.txt'.format(db))
shell_dump = '''cd {}
/opt/lampp/bin/mysqldump -h{} -P{} -u{} -p{} -d {} > {}'''.format(
root_160, dumps[key]['ip'], dumps[key]['port'],
dumps[key]['user'], dumps[key]['password'], db,
'lasTables/{}_tmp.txt'.format(db))
res, msg = yield self.thread_func.exec_remote_shell(
shell=shell_dump,
host=host_160,
port=port_160,
username=user_160,
password=password_160)
if res and os.path.isfile(tmp_path):
with open(tmp_path, 'r', encoding='utf8') as fp:
lines = fp.readlines()
tables = list()
for line in lines:
if line.find('DROP TABLE IF EXISTS') == -1:
continue
table = re.findall(r'`(\w+)`;', line)
table and tables.append('{}\n'.format(table[0]))
with open(os.path.join(db_path, '{}.txt'.format(db)),
'w') as fp:
fp.writelines(tables)
os.remove(tmp_path)
else:
log.info(msg)
old_time = time.time() - 30 * 24 * 3600
old_path = os.path.join(static_path, 'syncDB',
time.strftime('%Y%m%d', time.gmtime(old_time)))
if os.path.isdir(old_path):
shutil.rmtree(old_path)
async def launch(
self,
image,
username,
server_name="",
repo_url="",
extra_args=None,
event_callback=None,
):
"""Launch a server for a given image
- creates a temporary user on the Hub if authentication is not enabled
- spawns a server for temporary/authenticated user
- generates a token
- returns a dict containing:
- `url`: the URL of the server
- `image`: image spec
- `repo_url`: the url of the repo
- `extra_args`: Dictionary of extra arguments passed to the server
- `token`: the token for the server
"""
# TODO: validate the image argument?
# Matches the escaping that JupyterHub does https://github.com/jupyterhub/jupyterhub/blob/c00c3fa28703669b932eb84549654238ff8995dc/jupyterhub/user.py#L427
escaped_username = quote(username, safe="@~")
if self.create_user:
# create a new user
app_log.info("Creating user %s for image %s", username, image)
try:
await self.api_request(
f"users/{escaped_username}", body=b"", method="POST"
)
except HTTPError as e:
if e.response:
body = e.response.body
else:
body = ""
app_log.error(
"Error creating user %s: %s\n%s",
username,
e,
body,
)
raise web.HTTPError(500, f"Failed to create temporary user for {image}")
elif server_name == "":
# authentication is enabled but not named servers
# check if user has a running server ('')
user_data = await self.get_user_data(escaped_username)
if server_name in user_data["servers"]:
raise web.HTTPError(
409, f"User {username} already has a running server."
)
elif self.named_server_limit_per_user > 0:
# authentication is enabled with named servers
# check if user has already reached to the limit of named servers
user_data = await self.get_user_data(escaped_username)
len_named_spawners = len([s for s in user_data["servers"] if s != ""])
if self.named_server_limit_per_user <= len_named_spawners:
raise web.HTTPError(
409,
"User {} already has the maximum of {} named servers."
" One must be deleted before a new server can be created".format(
username, self.named_server_limit_per_user
),
)
if self.pre_launch_hook:
await maybe_future(
self.pre_launch_hook(self, image, username, server_name, repo_url)
)
# data to be passed into spawner's user_options during launch
# and also to be returned into 'ready' state
data = {
"image": image,
"repo_url": repo_url,
"token": base64.urlsafe_b64encode(uuid.uuid4().bytes)
.decode("ascii")
.rstrip("=\n"),
}
if extra_args:
data.update(extra_args)
# server name to be used in logs
_server_name = f" {server_name}" if server_name else ""
# start server
app_log.info(
f"Starting server{_server_name} for user {username} with image {image}"
)
ready_event_future = asyncio.Future()
def _cancel_ready_event(f=None):
if not ready_event_future.done():
if f and f.exception():
ready_event_future.set_exception(f.exception())
else:
ready_event_future.cancel()
try:
await self.api_request(
f"users/{escaped_username}/servers/{server_name}",
method="POST",
body=json.dumps(data).encode("utf8"),
)
# listen for pending spawn (launch) events until server is ready
# do this even if previous request finished!
buffer_list = []
async def handle_chunk(chunk):
lines = b"".join(buffer_list + [chunk]).split(b"\n\n")
# the last item in the list is usually an empty line ('')
# but it can be the partial line after the last `\n\n`,
# so put it back on the buffer to handle with the next chunk
buffer_list[:] = [lines[-1]]
for line in lines[:-1]:
if line:
line = line.decode("utf8", "replace")
if line and line.startswith("data:"):
event = json.loads(line.split(":", 1)[1])
if event_callback:
await event_callback(event)
# stream ends when server is ready or fails
if event.get("ready", False):
if not ready_event_future.done():
ready_event_future.set_result(event)
elif event.get("failed", False):
if not ready_event_future.done():
ready_event_future.set_exception(
web.HTTPError(
500, event.get("message", "unknown error")
)
)
url_parts = ["users", username]
if server_name:
url_parts.extend(["servers", server_name, "progress"])
else:
url_parts.extend(["server/progress"])
progress_api_url = url_path_join(*url_parts)
self.log.debug(f"Requesting progress for {username}: {progress_api_url}")
resp_future = self.api_request(
progress_api_url,
streaming_callback=lambda chunk: asyncio.ensure_future(
handle_chunk(chunk)
),
request_timeout=self.launch_timeout,
)
try:
await gen.with_timeout(
timedelta(seconds=self.launch_timeout), resp_future
)
except (gen.TimeoutError, TimeoutError):
_cancel_ready_event()
raise web.HTTPError(
500,
f"Image {image} for user {username} took too long to launch",
)
except HTTPError as e:
_cancel_ready_event()
if e.response:
body = e.response.body
else:
body = ""
app_log.error(
f"Error starting server{_server_name} for user {username}: {e}\n{body}"
)
raise web.HTTPError(500, f"Failed to launch image {image}")
except Exception:
_cancel_ready_event()
raise
# verify that the server is running!
try:
# this should already be done, but it's async so wait a finite time
await gen.with_timeout(timedelta(seconds=5), ready_event_future)
except (gen.TimeoutError, TimeoutError):
raise web.HTTPError(
500, f"Image {image} for user {username} failed to launch"
)
data["url"] = self.hub_url + f"user/{escaped_username}/{server_name}"
self.log.debug(data["url"])
return data
def summarize(message, list):
if list:
app_log.info(message, len(list))
def submit(self):
"""Submit a image spec to openshift's s2i and wait for completion """
volume_mounts = [
client.V1VolumeMount(mount_path="/var/run/docker.sock",
name="docker-socket")
]
docker_socket_path = urlparse(self.docker_host).path
volumes = [
client.V1Volume(name="docker-socket",
host_path=client.V1HostPathVolumeSource(
path=docker_socket_path, type='Socket'))
]
if self.push_secret:
volume_mounts.append(
client.V1VolumeMount(mount_path="/root/.docker",
name='docker-push-secret'))
volumes.append(
client.V1Volume(name='docker-push-secret',
secret=client.V1SecretVolumeSource(
secret_name=self.push_secret)))
self.pod = client.V1Pod(
metadata=client.V1ObjectMeta(
name=self.name,
labels={
"name": self.name,
"component": "binderhub-build",
},
annotations={
"binder-repo": self.repo_url,
},
),
spec=client.V1PodSpec(containers=[
client.V1Container(image=self.builder_image,
name="builder",
args=self.get_cmd(),
image_pull_policy='Always',
volume_mounts=volume_mounts,
resources=client.V1ResourceRequirements(
limits={'memory': self.memory_limit},
requests={'memory': self.memory_limit}))
],
node_selector=self.node_selector,
volumes=volumes,
restart_policy="Never"))
try:
ret = self.api.create_namespaced_pod(self.namespace, self.pod)
except client.rest.ApiException as e:
if e.status == 409:
# Someone else created it!
app_log.info("Build %s already running", self.name)
pass
else:
raise
else:
app_log.info("Started build %s", self.name)
app_log.info("Watching build pod %s", self.name)
while not self.stop_event.is_set():
w = watch.Watch()
try:
for f in w.stream(
self.api.list_namespaced_pod,
self.namespace,
label_selector="name={}".format(self.name),
timeout_seconds=30,
):
if f['type'] == 'DELETED':
self.progress('pod.phasechange', 'Deleted')
return
self.pod = f['object']
if not self.stop_event.is_set():
self.progress('pod.phasechange', self.pod.status.phase)
if self.pod.status.phase == 'Succeeded':
self.cleanup()
elif self.pod.status.phase == 'Failed':
self.cleanup()
except Exception as e:
app_log.exception("Error in watch stream for %s", self.name)
raise
finally:
w.stop()
if self.stop_event.is_set():
app_log.info("Stopping watch of %s", self.name)
return
async def on_message(self, message):
log.info(f"Message received: {message}")
async def on_close(self):
log.info("WebSocket closed by client.")
await self.sub.unsubscribe()
log.info("Unsubscribed from channel.")
import tornado.web
import tornado.ioloop
import tornado.httpserver
from tornado.log import app_log
from tornado.options import define, options, parse_command_line
from uri import uris
from settings.app_setting import SETTINGS
from settings.mongo_setting import MONGODB_NAME, MONGODB_CONFIG
from mongoengine import connect
define("port", default=8888, help="run on the given port", type=int)
class Application(tornado.web.Application):
def __init__(self):
# 连接MongoDB
connect(MONGODB_NAME, **MONGODB_CONFIG)
tornado.web.Application.__init__(self, uris, **SETTINGS)
application = Application()
if __name__ == "__main__":
parse_command_line()
server = tornado.httpserver.HTTPServer(application)
server.listen(options.port)
app_log.info("run on port:{}".format(options.port))
io_loop_ = tornado.ioloop.IOLoop.current()
io_loop_.start()
async def launch(self,
image,
username,
server_name='',
repo_url='',
extra_args=None):
"""Launch a server for a given image
- creates a temporary user on the Hub if authentication is not enabled
- spawns a server for temporary/authenticated user
- generates a token
- returns a dict containing:
- `url`: the URL of the server
- `image`: image spec
- `repo_url`: the url of the repo
- `extra_args`: Dictionary of extra arguments passed to the server
- `token`: the token for the server
"""
# TODO: validate the image argument?
if self.create_user:
# create a new user
app_log.info("Creating user %s for image %s", username, image)
try:
await self.api_request('users/%s' % username,
body=b'',
method='POST')
except HTTPError as e:
if e.response:
body = e.response.body
else:
body = ''
app_log.error(
"Error creating user %s: %s\n%s",
username,
e,
body,
)
raise web.HTTPError(
500, "Failed to create temporary user for %s" % image)
elif server_name == '':
# authentication is enabled but not named servers
# check if user has a running server ('')
user_data = await self.get_user_data(username)
if server_name in user_data['servers']:
raise web.HTTPError(
409, "User %s already has a running server." % username)
if self.pre_launch_hook:
await maybe_future(
self.pre_launch_hook(self, image, username, server_name,
repo_url))
# data to be passed into spawner's user_options during launch
# and also to be returned into 'ready' state
data = {
'image':
image,
'repo_url':
repo_url,
'token':
base64.urlsafe_b64encode(
uuid.uuid4().bytes).decode('ascii').rstrip('=\n')
}
if extra_args:
data.update(extra_args)
# server name to be used in logs
_server_name = " {}".format(server_name) if server_name else ''
# start server
app_log.info("Starting server%s for user %s with image %s",
_server_name, username, image)
try:
resp = await self.api_request(
'users/{}/servers/{}'.format(username, server_name),
method='POST',
body=json.dumps(data).encode('utf8'),
)
if resp.code == 202:
# Server hasn't actually started yet
# We wait for it!
# NOTE: This ends up being about ten minutes
for i in range(64):
user_data = await self.get_user_data(username)
if user_data['servers'][server_name]['ready']:
break
if not user_data['servers'][server_name]['pending']:
raise web.HTTPError(
500, "Image %s for user %s failed to launch" %
(image, username))
# FIXME: make this configurable
# FIXME: Measure how long it takes for servers to start
# and tune this appropriately
await gen.sleep(min(1.4**i, 10))
else:
raise web.HTTPError(
500, "Image %s for user %s took too long to launch" %
(image, username))
except HTTPError as e:
if e.response:
body = e.response.body
else:
body = ''
app_log.error("Error starting server{} for user {}: {}\n{}".format(
_server_name, username, e, body))
raise web.HTTPError(500, "Failed to launch image %s" % image)
data['url'] = self.hub_url + 'user/%s/%s' % (username, server_name)
return data
def get(self):
xdata = self.get_arguments("xdata")
ydata = self.get_arguments("ydata")
sma1 = self.get_arguments("sma1")
sma2 = self.get_arguments("sma2")
pos = self.get_arguments("pos")
kind = self.get_argument("kind", "")
imagelog.info("len(xdata)={}".format(len(xdata)))
imagelog.info("len(xdata)={}".format(len(ydata)))
imagelog.info("mutli={} type={}".format(len(pos) > 1, len(pos)))
mutli = False
if len(pos) > 1:
mutli = True
if len(xdata) != len(ydata) and not mutli:
return self.write(
json.dumps({
"err_code": 1,
"msg": "x y length not the same"
}))
# print(f"{xdata} {ydata}")
# print(xdata)
# print(ydata)
ydata = [float(i) for i in ydata]
if mutli:
imagelog.info("mutli data len(pos)={}".format(len(pos)))
sma1 = [float(i) for i in sma1]
sma2 = [float(i) for i in sma2]
pos = [int(i) for i in pos]
sma_data = [ydata, sma1, sma2]
# plt.plot(xdata, *sma_data, marker='o')
# plt.plot(xdata, *sma_data, color=["blue", "green", "orange"])
plt.plot(xdata, ydata, color='blue')
plt.plot(xdata, sma1, color='green')
plt.plot(xdata, sma2, color='orange')
plt.legend(["ori", "sma1", "sma2"])
ax2 = plt.twinx()
ax2.set_ylim([-1 - 0.1, 1 + 0.1])
ax2.plot(xdata, pos, color='red')
# https://blog.csdn.net/u010440456/article/details/90768681
ax2.legend(['pos'], loc="upper center") # upper right
else:
imagelog.info("single data len(ydata)={}".format(len(ydata)))
if kind == "bar":
plt.bar(xdata, ydata)
else:
plt.plot(xdata, ydata, marker='o')
plt.rcParams['font.sans-serif'] = ['WenQuanYi Zen Hei Mono']
plt.gcf().subplots_adjust(bottom=0.2)
plt.xticks(rotation=75, fontsize=12)
if len(xdata) < 5:
plt.xlim(0, 5)
figfile = BytesIO()
# filename = "xy_{}.png".format(self.request.request_time())
# print(filename)
plt.savefig(figfile, dpi=600, format="png")
figdata_png = base64.b64encode(figfile.getvalue()).decode()
# figfile.close()
plt.cla()
plt.close()
# print(len(figdata_png))
imagelog.info("{} {} \nimgb64 len:{}".format("xdata", "ydata",
len(figdata_png)))
return self.write(json.dumps({"err_code": 0, "img": figdata_png}))
def open(self):
app_log.info('open socket %s', self)
self.sockets.append(self)
self.subscriptions = {}
def handle_user(user):
"""Handle one user.
Create a list of their servers, and async exec them. Wait for
that to be done, and if all servers are stopped, possibly cull
the user.
"""
# shutdown servers first.
# Hub doesn't allow deleting users with running servers.
# jupyterhub 0.9 always provides a 'servers' model.
# 0.8 only does this when named servers are enabled.
if 'servers' in user:
servers = user['servers']
else:
# jupyterhub < 0.9 without named servers enabled.
# create servers dict with one entry for the default server
# from the user model.
# only if the server is running.
servers = {}
if user['server']:
servers[''] = {
'last_activity': user['last_activity'],
'pending': user['pending'],
'url': user['server'],
}
server_futures = [
handle_server(user, server_name, server)
for server_name, server in servers.items()
]
results = yield multi(server_futures)
if not cull_users:
return
# some servers are still running, cannot cull users
still_alive = len(results) - sum(results)
if still_alive:
app_log.debug(
"Not culling user %s with %i servers still alive",
user['name'],
still_alive,
)
return False
should_cull = False
if user.get('created'):
age = now - parse_date(user['created'])
else:
# created may be undefined on jupyterhub < 0.9
age = None
# check last activity
# last_activity can be None in 0.9
if user['last_activity']:
inactive = now - parse_date(user['last_activity'])
else:
# no activity yet, use start date
# last_activity may be None with jupyterhub 0.9,
# which introduces the 'created' field which is never None
inactive = age
should_cull = (inactive is not None
and inactive.total_seconds() >= inactive_limit)
if should_cull:
app_log.info("Culling user %s (inactive for %s)", user['name'],
inactive)
if max_age and not should_cull:
# only check created if max_age is specified
# so that we can still be compatible with jupyterhub 0.8
# which doesn't define the 'started' field
if age is not None and age.total_seconds() >= max_age:
app_log.info(
"Culling user %s (age: %s, inactive for %s)",
user['name'],
format_td(age),
format_td(inactive),
)
should_cull = True
if not should_cull:
app_log.debug(
"Not culling user %s (created: %s, last active: %s)",
user['name'],
format_td(age),
format_td(inactive),
)
return False
req = HTTPRequest(url=url + '/users/%s' % user['name'],
method='DELETE',
headers=auth_header)
yield fetch(req)
return True
def post(self, message_type, **kwargs):
app_log.info(self.request.body)
