def add_friend(self, friend):
if not self.check_friend(friend):
_conn.execute('''INSERT INTO tbl_friends (f_user_id, friend_id) VALUES (?, ?)''', (self.user_id, friend.user_id))
_conn.commit()
else:
app_log.warn("{!r} is already a friend of {!r}".format(friend.username, self.username))
raise FriendAlreadyAdded('{} has already been added as a friend of {}'.format(friend.username, self.username))
def _log_rate_limit(self, future):
"""log GitHub rate limit headers
- error if 0 remaining
- warn if 10% or less remain
- debug otherwise
"""
try:
r = future.result()
except HTTPError as e:
r = e.response
limit_s = r.headers.get("X-RateLimit-Limit", "")
remaining_s = r.headers.get("X-RateLimit-Remaining", "")
if not remaining_s or not limit_s:
if r.code < 300:
app_log.warn("No rate limit headers. Did GitHub change? %s", json.dumps(r.headers, indent=1))
return
remaining = int(remaining_s)
limit = int(limit_s)
if remaining == 0:
jsondata = response_text(r)
data = json.loads(jsondata)
app_log.error("GitHub rate limit (%s) exceeded: %s", limit, data.get("message", "no message"))
return
if 10 * remaining > limit:
log = app_log.debug
else:
log = app_log.warn
log("%i/%i GitHub API requests remaining", remaining, limit)
def wait_for_http_server(url, timeout=10):
"""Wait for an HTTP Server to respond at url
Any non-5XX response code will do, even 404.
"""
loop = ioloop.IOLoop.current()
tic = loop.time()
client = AsyncHTTPClient()
while loop.time() - tic < timeout:
try:
r = yield client.fetch(url, follow_redirects=False)
except HTTPError as e:
if e.code >= 500:
# failed to respond properly, wait and try again
if e.code != 599:
# we expect 599 for no connection,
# but 502 or other proxy error is conceivable
app_log.warn("Server at %s responded with error: %s", url, e.code)
yield gen.Task(loop.add_timeout, loop.time() + 0.25)
else:
app_log.debug("Server at %s responded with %s", url, e.code)
return
except (OSError, socket.error) as e:
if e.errno not in {errno.ECONNABORTED, errno.ECONNREFUSED, errno.ECONNRESET}:
app_log.warn("Failed to connect to %s (%s)", url, e)
yield gen.Task(loop.add_timeout, loop.time() + 0.25)
else:
return
raise TimeoutError("Server at {url} didn't respond in {timeout} seconds".format(
**locals()
))
def reraise_client_error(self, exc):
"""Remote fetch raised an error"""
try:
url = exc.response.request.url
except AttributeError:
url = 'url'
app_log.warn("Fetching %s failed with %s", url, exc)
if exc.code == 599:
str_exc = str(exc)
# strip the unhelpful 599 prefix
if str_exc.startswith('HTTP 599: '):
str_exc = str_exc[10:]
if isinstance(exc, CurlError):
en = getattr(exc, 'errno', -1)
# can't connect to server should be 404
# possibly more here
if en in (pycurl.E_COULDNT_CONNECT, pycurl.E_COULDNT_RESOLVE_HOST):
raise web.HTTPError(404, str_exc)
# otherwise, raise 400 with informative message:
raise web.HTTPError(400, str_exc)
if exc.code >= 500:
# 5XX, server error, but not this server
raise web.HTTPError(502, str(exc))
else:
if exc.code == 404:
raise web.HTTPError(404, "Remote %s" % exc)
else:
# client-side error, blame our client
raise web.HTTPError(400, str(exc))
def get_error_html(self, status_code, **kwargs):
"""render custom error pages"""
exception = kwargs.get('exception')
message = ''
status_message = responses.get(status_code, 'Unknown')
if exception:
# get the custom message, if defined
try:
message = exception.log_message % exception.args
except Exception:
pass
# construct the custom reason, if defined
reason = getattr(exception, 'reason', '')
if reason:
status_message = reason
# build template namespace
ns = dict(
status_code=status_code,
status_message=status_message,
message=message,
exception=exception,
)
# render the template
try:
html = self.render_template('%d.html' % status_code, **ns)
except Exception as e:
app_log.warn("No template for %d", status_code)
html = self.render_template('error.html', **ns)
return html
def reraise_client_error(self, exc):
"""Remote fetch raised an error"""
try:
url = exc.response.request.url.split('?')[0]
body = exc.response.body.decode('utf8', 'replace').strip()
except AttributeError:
url = 'url'
body = ''
msg = self.client_error_message(exc, url, body)
slim_body = escape(body[:300])
app_log.warn("Fetching %s failed with %s. Body=%s", url, msg, slim_body)
if exc.code == 599:
if isinstance(exc, CurlError):
en = getattr(exc, 'errno', -1)
# can't connect to server should be 404
# possibly more here
if en in (pycurl.E_COULDNT_CONNECT, pycurl.E_COULDNT_RESOLVE_HOST):
raise web.HTTPError(404, msg)
# otherwise, raise 400 with informative message:
raise web.HTTPError(400, msg)
if exc.code >= 500:
# 5XX, server error, but not this server
raise web.HTTPError(502, msg)
else:
# client-side error, blame our client
if exc.code == 404:
raise web.HTTPError(404, "Remote %s" % msg)
else:
raise web.HTTPError(400, msg)
def get(self, path):
localfile_path = os.path.abspath(
self.settings.get('localfile_path', ''))
abspath = os.path.abspath(os.path.join(
localfile_path,
path
))
app_log.info("looking for file: '%s'" % abspath)
if not abspath.startswith(localfile_path):
app_log.warn("directory traversal attempt: '%s'" % localfile_path)
raise web.HTTPError(404)
if not os.path.exists(abspath):
raise web.HTTPError(404)
with io.open(abspath, encoding='utf-8') as f:
nbdata = f.read()
yield self.finish_notebook(nbdata, download_url=path,
msg="file from localfile: %s" % path,
public=False,
format=self.format,
request=self.request)
def write_error(self, status_code, **kwargs):
"""render custom error pages"""
exc_info = kwargs.get('exc_info')
message = ''
status_message = responses.get(status_code, 'Unknown')
if exc_info:
# get the custom message, if defined
exception = exc_info[1]
try:
message = exception.log_message % exception.args
except Exception:
pass
# construct the custom reason, if defined
reason = getattr(exception, 'reason', '')
if reason:
status_message = reason
# build template namespace
ns = dict(
status_code=status_code,
status_message=status_message,
message=message,
exception=exception,
)
# render the template
try:
html = self.render_template('%d.html' % status_code, **ns)
except Exception as e:
app_log.warn("No template for %d", status_code)
html = self.render_template('error.html', **ns)
self.set_header('Content-Type', 'text/html')
self.write(html)
def get(self, path):
localfile_path = os.path.abspath(
self.settings.get('localfile_path', ''))
abspath = os.path.abspath(os.path.join(
localfile_path,
path
))
app_log.info("looking for file: '%s'" % abspath)
if not abspath.startswith(localfile_path):
app_log.warn("directory traversal attempt: '%s'" % localfile_path)
raise web.HTTPError(404)
if not os.path.exists(abspath):
raise web.HTTPError(404)
if not os.path.isfile(abspath):
base_url='/localfile'
contents=os.listdir(abspath)
dirs=[]
others=[]
ipynbs=[]
entries=[]
for c in contents:
e={}
e['name']=c
e['url']=os.path.join(base_url,path,c)
subpath=os.path.join(abspath,c)
if os.path.isdir(subpath):
e['class']='fa-folder-open'
dirs.append(e)
elif os.path.isfile(subpath):
if c.endswith('.ipynb'):
e['class']='fa-book'
ipynbs.append(e)
else:
e['url']='javascript:alert("not allowed type!");'
e['class']='fa-folder-close'
others.append(e)
entries.extend(dirs)
entries.extend(ipynbs)
entries.extend(others)
breadcrumbs=self.breadcrumbs(path,base_url)
html=self.render_template('treelist.html',
entries=entries,breadcrumbs=breadcrumbs,tree_type='localfile',
tree_label='localfiels')
yield self.cache_and_finish(html)
else:
with io.open(abspath, encoding='utf-8') as f:
nbdata = f.read()
provider_url=os.path.join(base_url,path)
yield self.finish_notebook(nbdata, download_url=path,
provider_url=provider_url,
msg="file from localfile: %s" % path,
public=False,
format=self.format,
request=self.request)
def on_finish(self):
if self._pkg_fd is not None:
self._pkg_fd.close()
# md5_digest content not found
if self._need_md5:
app_log.warn('md5_digest disposition not found')
self._pkg_md5 = self._pkg_diggest.hexdigest()
self.write_md5()
def create_notebook_server(self, base_path,
image="jupyter/demo",
ipython_executable="ipython3",
mem_limit="512m",
cpu_shares="1",
container_ip="127.0.0.1",
container_port=8888):
'''
Creates a notebook_server running off of `base_path`.
returns the container_id, ip, port in a Future
'''
templates = ['/srv/ga',
'/srv/ipython/IPython/html',
'/srv/ipython/IPython/html/templates']
tornado_settings = {'template_path': templates}
ipython_args = [
"notebook", "--no-browser",
"--port {}".format(container_port),
"--ip=0.0.0.0",
"--NotebookApp.base_url=/{}".format(base_path),
"--NotebookApp.tornado_settings=\"{}\"".format(tornado_settings)
]
ipython_command = ipython_executable + " " + " ".join(ipython_args)
command = [
"/bin/sh",
"-c",
ipython_command
]
resp = yield self.docker_client.create_container(image=image,
command=command,
mem_limit=mem_limit,
cpu_shares=cpu_shares)
docker_warnings = resp['Warnings']
if docker_warnings is not None:
app_log.warn(docker_warnings)
container_id = resp['Id']
app_log.info("Created container {}".format(container_id))
yield self.docker_client.start(container_id,
port_bindings={container_port: (container_ip,)})
container_network = yield self.docker_client.port(container_id,
container_port)
host_port = container_network[0]['HostPort']
host_ip = container_network[0]['HostIp']
raise gen.Return((container_id, host_ip, int(host_port)))
def find_uid(cls, uid):
cur = _conn.execute('''SELECT rowid AS user_id, fname, lname, username, email, image, dob FROM tbl_users WHERE rowid = ? LIMIT 1''', (uid,))
row = cur.fetchone()
if not row:
app_log.warn("user with the uid {} could not be found".format(uid))
raise UserNotFound('Uid {} does not exist'.format(uid))
return cls(**row)
def on_finish(self):
if self._pkg_fd is not None:
self._pkg_fd.close()
# md5_digest content not found
if self._need_md5:
app_log.warn('md5_digest disposition not found')
self._pkg_md5 = self._pkg_diggest.hexdigest()
self.write_md5()
def find(cls, product_id):
cur = _conn.execute('''SELECT p.rowid AS product_id, p.name, p.image, p.link, p.description, p.price, i.checked FROM tbl_products AS p LEFT JOIN tbl_list_item AS i ON p.rowid = i.product_id WHERE p.rowid = ? LIMIT 1''', (product_id,))
row = cur.fetchone()
if not row:
app_log.warn("product {} could not be found".format(product_id))
raise ProductNotFound('{} does not exist'.format(product_id))
return cls(*row)
def create_notebook_server(self, base_path, container_name, container_config):
'''Creates a notebook_server running off of `base_path`.
Returns the (container_id, ip, port) tuple in a Future.'''
port = container_config.container_port
app_log.debug(container_config)
# Assumes that the container_config.command is of a format like:
#
# ipython3 notebook --no-browser --port {port} --ip=0.0.0.0
# --NotebookApp.base_path=/{base_path}
# --NotebookApp.tornado_settings=\"{ \"template_path\": [ \"/srv/ga\",
# \"/srv/ipython/IPython/html\",
# \"/srv/ipython/IPython/html/templates\" ] }\""
#
# Important piece here is the parametrized base_path to let the
# underlying process know where the proxy is routing it.
rendered_command = container_config.command.format(base_path=base_path, port=port)
command = [
"/bin/sh",
"-c",
rendered_command
]
resp = yield self._with_retries(self.docker_client.create_container,
image=container_config.image,
command=command,
mem_limit=container_config.mem_limit,
cpu_shares=container_config.cpu_shares,
name=container_name)
docker_warnings = resp['Warnings']
if docker_warnings is not None:
app_log.warn(docker_warnings)
container_id = resp['Id']
app_log.info("Created container {}".format(container_id))
port_bindings = {
container_config.container_port: (container_config.container_ip,)
}
yield self._with_retries(self.docker_client.start,
container_id,
port_bindings=port_bindings)
container_network = yield self._with_retries(self.docker_client.port,
container_id,
container_config.container_port)
host_port = container_network[0]['HostPort']
host_ip = container_network[0]['HostIp']
raise gen.Return((container_id, host_ip, int(host_port)))
def create_notebook_server(self, base_path, container_name, container_config):
'''Creates a notebook_server running off of `base_path`.
Returns the (container_id, ip, port) tuple in a Future.'''
port = container_config.container_port
app_log.debug(container_config)
# Assumes that the container_config.command is of a format like:
#
# ipython3 notebook --no-browser --port {port} --ip=0.0.0.0
# --NotebookApp.base_path=/{base_path}
# --NotebookApp.tornado_settings=\"{ \"template_path\": [ \"/srv/ga\",
# \"/srv/ipython/IPython/html\",
# \"/srv/ipython/IPython/html/templates\" ] }\""
#
# Important piece here is the parametrized base_path to let the
# underlying process know where the proxy is routing it.
rendered_command = container_config.command.format(base_path=base_path, port=port)
command = [
"/bin/sh",
"-c",
rendered_command
]
resp = yield self._with_retries(self.docker_client.create_container,
image=container_config.image,
command=command,
mem_limit=container_config.mem_limit,
cpu_shares=container_config.cpu_shares,
name=container_name)
docker_warnings = resp['Warnings']
if docker_warnings is not None:
app_log.warn(docker_warnings)
container_id = resp['Id']
app_log.info("Created container {}".format(container_id))
port_bindings = {
container_config.container_port: (container_config.container_ip,)
}
yield self._with_retries(self.docker_client.start,
container_id,
port_bindings=port_bindings)
container_network = yield self._with_retries(self.docker_client.port,
container_id,
container_config.container_port)
host_port = container_network[0]['HostPort']
host_ip = container_network[0]['HostIp']
raise gen.Return((container_id, host_ip, int(host_port)))
def name_matches(container):
try:
names = container['Names']
except Exception:
app_log.warn("Invalid container: %r", container)
return False
for name in names:
if pool_regex.search(name):
return True
return False
def cleanout(self):
'''Completely cleanout containers that are part of this pool.'''
app_log.info("Performing initial pool cleanup")
containers = yield self.spawner.list_notebook_servers(self.container_name_pattern, all=True)
for container in containers:
try:
app_log.debug("Clearing old container [%s] from pool", container['Id'])
yield self.spawner.shutdown_notebook_server(container['Id'])
except Exception as e:
app_log.warn(e)
def cleanout(self):
'''Completely cleanout containers that are part of this pool.'''
app_log.info("Performing initial pool cleanup")
containers = yield self.spawner.list_notebook_servers(self.container_name_pattern, all=True)
for container in containers:
try:
app_log.debug("Clearing old container [%s] from pool", container['Id'])
yield self.spawner.shutdown_notebook_server(container['Id'])
except Exception as e:
app_log.warn(e)
def get_current_user(self):
"""Check admin API token, if any"""
if not self.admin_token:
return 'authorized'
client_token = self.request.headers.get('Authorization')
if client_token != 'token %s' % self.admin_token:
app_log.warn('Rejecting admin request with token %s', client_token)
return
else:
return 'authorized'
def prepare(self):
'''
Responds with a 401 error if the configured admin auth token is not
present in the request headers.
'''
if self.admin_token:
client_token = self.request.headers.get('Authorization')
if client_token != 'token %s' % self.admin_token:
app_log.warn('Rejecting admin request with token %s', client_token)
return self.send_error(401)
return super(AdminHandler, self).prepare()
def get_current_user(self):
"""Check admin API token, if any"""
if not self.admin_token:
return 'authorized'
client_token = self.request.headers.get('Authorization')
if client_token != 'token %s' % self.admin_token:
app_log.warn('Rejecting admin request with token %s', client_token)
return
else:
return 'authorized'
def newService(self, data):
"""
Create a new service mapping and return the new service key.
"""
response = self.table.put_item(Item=data, ReturnValues='ALL_OLD')
if response['ResponseMetadata']['HTTPStatusCode'] != 200:
raise ValueError("Failed to run newService.")
# TODO: store the capacity units consumed
if 'Attributes' in response:
app_log.warn("newService updated an existing service.")
app_log.info("newService.response: %s" % response)
def handle_graqhql(self):
result = self.execute_graphql()
app_log.debug('GraphQL result data: %s errors: %s invalid %s',
result.data, result.errors, result.invalid)
if result and (result.errors or result.invalid):
ex = ExecutionError(errors=result.errors)
app_log.warn('GraphQL Error: %s', ex)
raise ex
response = {'data': result.data}
self.write(json_encode(response))
def create_notebook_server(self,
base_path,
image="jupyter/demo",
ipython_executable="ipython3",
mem_limit="512m",
cpu_shares="1",
container_ip="127.0.0.1",
container_port=8888):
'''
Creates a notebook_server running off of `base_path`.
returns the container_id, ip, port in a Future
'''
templates = [
'/srv/ga', '/srv/ipython/IPython/html',
'/srv/ipython/IPython/html/templates'
]
tornado_settings = {'template_path': templates}
ipython_args = [
"notebook", "--no-browser", "--port {}".format(container_port),
"--ip=0.0.0.0", "--NotebookApp.base_url=/{}".format(base_path),
"--NotebookApp.tornado_settings=\"{}\"".format(tornado_settings)
]
ipython_command = ipython_executable + " " + " ".join(ipython_args)
command = ["/bin/sh", "-c", ipython_command]
resp = yield self.docker_client.create_container(image=image,
command=command,
mem_limit=mem_limit,
cpu_shares=cpu_shares)
docker_warnings = resp['Warnings']
if docker_warnings is not None:
app_log.warn(docker_warnings)
container_id = resp['Id']
app_log.info("Created container {}".format(container_id))
yield self.docker_client.start(
container_id, port_bindings={container_port: (container_ip, )})
container_network = yield self.docker_client.port(
container_id, container_port)
host_port = container_network[0]['HostPort']
host_ip = container_network[0]['HostIp']
raise gen.Return((container_id, host_ip, int(host_port)))
def handle_graqhql(self):
result = self.execute_graphql()
app_log.debug('GraphQL result data: %s errors: %s invalid %s',
result.data, result.errors, result.invalid)
if result and (result.errors or result.invalid):
codes = [ x.original_error.status_code for x in result.errors if hasattr(x, 'original_error') and isinstance(x.original_error, web.HTTPError) ]
ex = ExecutionError(codes[0] if len(codes) == 1 else 400, errors=result.errors)
app_log.warn('GraphQL Error: %s', ex)
raise ex
response = {'data': result.data}
self.write(json_encode(response))
def prepare(self):
'''
Responds with a 401 error if the configured admin auth token is not
present in the request headers.
'''
if self.admin_token:
client_token = self.request.headers.get('Authorization')
if client_token != 'token %s' % self.admin_token:
app_log.warn('Rejecting admin request with token %s',
client_token)
return self.send_error(401)
return super(AdminHandler, self).prepare()
async def save_faces(self, faces, photo_id):
document = {
'_id': photo_id,
'faces': faces
}
app_log.debug(
'FacesRepository: saving faces in cache for photo {}'.format(photo_id))
try:
result = await self.collection.insert_one(document)
except Exception as e:
app_log.warn(
'Cannot save faces for photo {}: {}'.format(photo_id, faces))
def user_for_cookie(self, encrypted_cookie, use_cache=True):
"""Ask the Hub to identify the user for a given cookie.
Args:
encrypted_cookie (str): the cookie value (not decrypted, the Hub will do that)
use_cache (bool): Specify use_cache=False to skip cached cookie values (default: True)
Returns:
user_model (dict): The user model, if a user is identified, None if authentication fails.
The 'name' field contains the user's name.
"""
if use_cache:
cached = self.cookie_cache.get(encrypted_cookie)
if cached is not None:
return cached
try:
r = requests.get(
url_path_join(
self.api_url, "authorizations/cookie", self.cookie_name, quote(encrypted_cookie, safe="")
),
headers={"Authorization": "token %s" % self.api_token},
)
except requests.ConnectionError:
msg = "Failed to connect to Hub API at %r." % self.api_url
msg += " Is the Hub accessible at this URL (from host: %s)?" % socket.gethostname()
if "127.0.0.1" in self.api_url:
msg += (
" Make sure to set c.JupyterHub.hub_ip to an IP accessible to"
+ " single-user servers if the servers are not on the same host as the Hub."
)
raise HTTPError(500, msg)
if r.status_code == 404:
data = None
elif r.status_code == 403:
app_log.error(
"I don't have permission to verify cookies, my auth token may have expired: [%i] %s",
r.status_code,
r.reason,
)
raise HTTPError(500, "Permission failure checking authorization, I may need a new token")
elif r.status_code >= 500:
app_log.error("Upstream failure verifying auth token: [%i] %s", r.status_code, r.reason)
raise HTTPError(502, "Failed to check authorization (upstream problem)")
elif r.status_code >= 400:
app_log.warn("Failed to check authorization: [%i] %s", r.status_code, r.reason)
raise HTTPError(500, "Failed to check authorization")
else:
data = r.json()
self.cookie_cache[encrypted_cookie] = data
return data
def _check_cache_state(self):
from utils import getmc
app_log.debug("start memcached state check thread")
while 1:
try:
client = getmc()
client.get_stats()
app_log.debug("memcached come back!")
self._evthub.emit_system_notify("cacheserver.up", "")
break
except:
app_log.warn("memcached still down!")
time.sleep(5)
def name_matches(container):
try:
names = container['Names']
if names is None:
app_log.warn("Docker API returned null Names, ignoring")
return False
except Exception:
app_log.warn("Invalid container: %r", container)
return False
for name in names:
if pool_regex.search(name):
return True
return False
def name_matches(container):
try:
names = container['Names']
if names is None:
app_log.warn("Docker API returned null Names, ignoring")
return False
except Exception:
app_log.warn("Invalid container: %r", container)
return False
for name in names:
if pool_regex.search(name):
return True
return False
async def handle_graqhql(self):
result = await self.execute_graphql()
app_log.debug("GraphQL result data: %s errors: %s", result.data, result.errors)
if result and result.errors:
ex = ExecutionError(errors=result.errors)
app_log.warn("GraphQL Error: %s", ex.message)
self.write("GraphQL Error: {}".format(ex.message))
if not self.application.settings.get("debug", False):
# Return a 500 server error to the client if we are not running the
# server in debug mode
raise ex
response = {"data": result.data}
self.write(response)
def get(self, path):
"""Get a directory listing, rendered notebook, or raw file
at the given path based on the type and URL query parameters.
If the path points to an accessible directory, render its contents.
If the path points to an accessible notebook file, render it.
If the path points to an accessible file and the URL contains a
'download' query parameter, respond with the file as a download.
Parameters
==========
path: str
Local filesystem path
"""
abspath = os.path.abspath(os.path.join(
self.localfile_path,
path
))
if not abspath.startswith(self.localfile_path):
app_log.warn("directory traversal attempt: '%s'" %
self.localfile_path)
raise web.HTTPError(404)
if not os.path.exists(abspath):
raise web.HTTPError(404)
if os.path.isdir(abspath):
html = self.show_dir(abspath, path)
raise gen.Return(self.cache_and_finish(html))
is_download = self.get_query_arguments('download')
if is_download:
self.download(abspath)
return
try:
with io.open(abspath, encoding='utf-8') as f:
nbdata = f.read()
except IOError as ex:
if ex.errno == errno.EACCES:
# py2/3: can't read the file, so don't give away it exists
raise web.HTTPError(404)
raise ex
yield self.finish_notebook(nbdata,
download_url='?download',
msg="file from localfile: %s" % path,
public=False,
format=self.format,
request=self.request)
def _clean_bad_conn(self):
bad_number = len(self._bad)
while len(self._bad) > 0:
conn = self._bad.pop(0)
if conn in self._busy:
self._busy.remove(conn)
if conn in self._idle:
self._idle.remove(conn)
self._call_whatever(self._close_conn, conn)
if bad_number > 0:
app_log.warn("cleaned %d closed connections" % bad_number)
def user_for_cookie(self, encrypted_cookie, use_cache=True):
"""Ask the Hub to identify the user for a given cookie.
Args:
encrypted_cookie (str): the cookie value (not decrypted, the Hub will do that)
use_cache (bool): Specify use_cache=False to skip cached cookie values (default: True)
Returns:
user_model (dict): The user model, if a user is identified, None if authentication fails.
The 'name' field contains the user's name.
"""
if use_cache:
cached = self.cookie_cache.get(encrypted_cookie)
if cached is not None:
return cached
try:
r = requests.get(
url_path_join(self.api_url,
"authorizations/cookie",
self.cookie_name,
quote(encrypted_cookie, safe=''),
),
headers = {
'Authorization' : 'token %s' % self.api_token,
},
)
except requests.ConnectionError:
msg = "Failed to connect to Hub API at %r." % self.api_url
msg += " Is the Hub accessible at this URL (from host: %s)?" % socket.gethostname()
if '127.0.0.1' in self.api_url:
msg += " Make sure to set c.JupyterHub.hub_ip to an IP accessible to" + \
" single-user servers if the servers are not on the same host as the Hub."
raise HTTPError(500, msg)
if r.status_code == 404:
data = None
elif r.status_code == 403:
app_log.error("I don't have permission to verify cookies, my auth token may have expired: [%i] %s", r.status_code, r.reason)
raise HTTPError(500, "Permission failure checking authorization, I may need a new token")
elif r.status_code >= 500:
app_log.error("Upstream failure verifying auth token: [%i] %s", r.status_code, r.reason)
raise HTTPError(502, "Failed to check authorization (upstream problem)")
elif r.status_code >= 400:
app_log.warn("Failed to check authorization: [%i] %s", r.status_code, r.reason)
raise HTTPError(500, "Failed to check authorization")
else:
data = r.json()
self.cookie_cache[encrypted_cookie] = data
return data
def get(self, path):
"""Get a directory listing, rendered notebook, or raw file
at the given path based on the type and URL query parameters.
If the path points to an accessible directory, render its contents.
If the path points to an accessible notebook file, render it.
If the path points to an accessible file and the URL contains a
'download' query parameter, respond with the file as a download.
Parameters
==========
path: str
Local filesystem path
"""
abspath = os.path.abspath(os.path.join(self.localfile_path, path))
if not abspath.startswith(self.localfile_path):
app_log.warn("directory traversal attempt: '%s'" %
self.localfile_path)
raise web.HTTPError(404)
if not os.path.exists(abspath):
raise web.HTTPError(404)
if os.path.isdir(abspath):
html = self.show_dir(abspath, path)
raise gen.Return(self.cache_and_finish(html))
is_download = self.get_query_arguments('download')
if is_download:
self.download(abspath)
return
try:
with io.open(abspath, encoding='utf-8') as f:
nbdata = f.read()
except IOError as ex:
if ex.errno == errno.EACCES:
# py2/3: can't read the file, so don't give away it exists
raise web.HTTPError(404)
raise ex
yield self.finish_notebook(nbdata,
download_url='?download',
msg="file from localfile: %s" % path,
public=False,
format=self.format,
request=self.request,
breadcrumbs=self.breadcrumbs(path),
title=os.path.basename(path))
def user_for_cookie(self, encrypted_cookie, use_cache=True):
"""Ask the Hub to identify the user for a given cookie.
Args:
encrypted_cookie (str): the cookie value (not decrypted, the Hub will do that)
use_cache (bool): Specify use_cache=False to skip cached cookie values (default: True)
Returns:
user_model (dict): The user model, if a user is identified, None if authentication fails.
The 'name' field contains the user's name.
"""
if use_cache:
cached = self.cookie_cache.get(encrypted_cookie)
if cached is not None:
return cached
r = requests.get(
url_path_join(
self.api_url,
"authorizations/cookie",
self.cookie_name,
quote(encrypted_cookie, safe=''),
),
headers={'Authorization': 'token %s' % self.api_token},
)
if r.status_code == 404:
data = None
elif r.status_code == 403:
app_log.error(
"I don't have permission to verify cookies, my auth token may have expired: [%i] %s",
r.status_code, r.reason)
raise HTTPError(
500,
"Permission failure checking authorization, I may need a new token"
)
elif r.status_code >= 500:
app_log.error("Upstream failure verifying auth token: [%i] %s",
r.status_code, r.reason)
raise HTTPError(
502, "Failed to check authorization (upstream problem)")
elif r.status_code >= 400:
app_log.warn("Failed to check authorization: [%i] %s",
r.status_code, r.reason)
raise HTTPError(500, "Failed to check authorization")
else:
data = r.json()
self.cookie_cache[encrypted_cookie] = data
return data
def validate(self):
pkg_file = self.application.get_upload_path() / self._pkg_name / self._pkg_file.name
if not self.settings['package']:
return pkg_file
setting = self.settings['package'].get(self._pkg_name, {})
if not setting.get('update', True) and pkg_file.exists():
app_log.warn('updating package %s not allowed', self._pkg_file.name)
if pkg_file != self._pkg_file and self._pkg_file.exists():
self._pkg_file.unlink()
raise tornado.web.HTTPError(403)
return pkg_file
def reraise_client_error(self, exc):
"""Remote fetch raised an error"""
try:
url = exc.response.request.url.split('?')[0]
body = exc.response.body.decode('utf8', 'replace').strip()
except AttributeError:
url = 'url'
body = ''
code, msg = self.client_error_message(exc, url, body)
slim_body = escape(body[:300])
app_log.warn("Fetching %s failed with %s. Body=%s", url, msg, slim_body)
raise web.HTTPError(code, msg)
def reraise_client_error(self, exc):
"""Remote fetch raised an error"""
try:
url = exc.response.request.url.split('?')[0]
body = exc.response.body.decode('utf8', 'replace').strip()
except AttributeError:
url = 'url'
body = ''
code, msg = self.client_error_message(exc, url, body)
slim_body = escape(body[:300])
app_log.warn("Fetching %s failed with %s. Body=%s", url, msg, slim_body)
raise web.HTTPError(code, msg)
def reraise_client_error(self, exc):
"""Remote fetch raised an error"""
try:
url = exc.response.request.url
except AttributeError:
url = 'url'
app_log.warn("Fetching %s failed with %s", url, exc)
if exc.code >= 500:
# 5XX, server error
raise web.HTTPError(502, str(exc))
else:
if exc.code == 404:
raise web.HTTPError(404, "Remote %s" % exc)
else:
# client-side error, but we are the client
raise web.HTTPError(500, str(exc))
async def get(self, package):
client = AsyncHTTPClient()
try:
resp = await client.fetch(urljoin(NPM_REGISTRY_URL, self.request.path))
except HTTPClientError as err:
self.set_status(err.code, err.response)
self.finish()
return
except Exception as err:
app_log.warn('Error connecting to npm', exc_info=sys.exc_info)
self.set_status(503, 'Service unavailable')
self.finish()
return
# TODO: stream instead of bufferring
body = rewrite_urls(resp.body)
self.finish(body)
def reraise_client_error(self, exc):
"""Remote fetch raised an error"""
try:
url = exc.response.request.url
except AttributeError:
url = 'url'
app_log.warn("Fetching %s failed with %s", url, exc)
if exc.code >= 500:
# 5XX, server error
raise web.HTTPError(502, str(exc))
else:
if exc.code == 404:
raise web.HTTPError(404, "Remote %s" % exc)
else:
# client-side error, but we are the client
raise web.HTTPError(500, str(exc))
def retry(function, *args, **kwargs):
""" Retries a function up to max_retries, waiting `timeout` seconds between tries.
This function is designed to retry both boto3 and fabric calls. In the
case of boto3, it is necessary because sometimes aws calls return too
early and a resource needed by the next call is not yet available. """
max_retries = kwargs.pop("max_retries", 20)
timeout = kwargs.pop("timeout", 0.25)
for attempt in range(max_retries):
try:
ret = yield thread_pool.submit(function, *args, **kwargs)
return ret
except (ClientError, WaiterError) as e:
app_log.warn("encountered %s, waiting for %s seconds before retrying..." % (type(e), timeout) )
yield sleep(timeout)
else:
print("Failure in %s with args %s and kwargs %s" % (function.__name__, args, kwargs))
def validate(self):
pkg_file = self.application.get_upload_path(
) / self._pkg_name / self._pkg_file.name
if not self.settings['package']:
return pkg_file
setting = self.settings['package'].get(self._pkg_name, {})
if not setting.get('update', True) and pkg_file.exists():
app_log.warn('updating package %s not allowed',
self._pkg_file.name)
if pkg_file != self._pkg_file and self._pkg_file.exists():
self._pkg_file.unlink()
raise tornado.web.HTTPError(403)
return pkg_file
def reraise_client_error(self, exc):
"""Remote fetch raised an error"""
try:
url = exc.response.request.url.split('?')[0]
body = exc.response.body.decode('utf8', 'replace').strip()
except AttributeError:
url = 'url'
body = ''
str_exc = str(exc)
# strip the unhelpful 599 prefix
if str_exc.startswith('HTTP 599: '):
str_exc = str_exc[10:]
if exc.code == 403 and 'too big' in body and 'gist' in url:
msg = "GitHub will not serve raw gists larger than 10MB"
elif body and len(body) < 100:
# if it's a short plain-text error message, include it
msg = "%s (%s)" % (str_exc, escape(body))
else:
msg = str_exc
slim_body = escape(body[:300])
app_log.warn("Fetching %s failed with %s. Body=%s", url, msg,
slim_body)
if exc.code == 599:
if isinstance(exc, CurlError):
en = getattr(exc, 'errno', -1)
# can't connect to server should be 404
# possibly more here
if en in (pycurl.E_COULDNT_CONNECT,
pycurl.E_COULDNT_RESOLVE_HOST):
raise web.HTTPError(404, msg)
# otherwise, raise 400 with informative message:
raise web.HTTPError(400, msg)
if exc.code >= 500:
# 5XX, server error, but not this server
raise web.HTTPError(502, msg)
else:
# client-side error, blame our client
if exc.code == 404:
raise web.HTTPError(404, "Remote %s" % msg)
else:
raise web.HTTPError(400, msg)
def close_network(self):
try:
now_date = time.strftime('%Y-%m-%d')
now_time = time.strftime('%H:%M:%S')
if now_time > '08:30:00':
server, total = yield self.setting.get_settings_list(
s_type='env', status=1)
ips = list()
for svr in server:
desc = json.loads(svr.value)
if (svr.createTime.strftime('%Y-%m-%d') < now_date or
(svr.createTime.strftime('%Y-%m-%d') == now_date
and now_time > svr.createTime.strftime('%H:%M:%S'))
) and desc.get('network') == 'yes' and desc.get('ip'):
ips.append(desc.get('ip').strip())
if set(ips):
title = '测试环境申请关闭外网权限'
mail_content = '''
<p>Hi 你好！</p>
<p style="padding-left:30px;">测试人员已完成测试任务，申请关闭以下测试环境服务器外网权限。请帮忙处理一下，3ks~</p>
<p style="padding-left:30px;">1、服务器</p>
<p style="padding-left:60px;">{}</p>
'''.format('</br>'.join(set(ips)))
res, msg = yield self.common_func.send_email(
subject=title,
content=mail_content,
to=net_mail_to,
cc=net_mail_cc)
if res:
log.info(msg)
for svr in server:
desc = json.loads(svr.value)
if (svr.createTime.strftime('%Y-%m-%d') < now_date
or (svr.createTime.strftime('%Y-%m-%d')
== now_date and now_time >
svr.createTime.strftime('%H:%M:%S'))
) and desc.get(
'network') == 'yes' and desc.get('ip'):
desc['network'] = 'no'
yield self.setting.edit_setting(sid=svr.id,
value=desc)
else:
log.warn(msg)
except Exception as e:
log.error(e)
async def on_start(self, op_id, params):
"""Setup a subscription"""
subscription = await subscribe(
schema=self.schema,
document=parse(params["request_string"]),
root_value=None,
context_value=params["context_value"],
variable_values=params["variable_values"],
operation_name=params["operation_name"],
)
if isinstance(subscription, ExecutionResult):
# There was an error during the subscription graphql call
# TODO: log send errors back to client
app_log.warn("GraphQL Error: %s", subscription.errors)
return False
else:
# The subscription graphql call successfully created a MapAsyncIterator
await self.subscribe(op_id, subscription)
def _wait_for_server(self, ip, port, path, timeout=10, wait_time=0.2):
'''Wait for a server to show up within a newly launched container.'''
app_log.info("Waiting for a container to launch at [%s:%s].", ip, port)
loop = ioloop.IOLoop.current()
tic = loop.time()
# Docker starts listening on a socket before the container is fully launched. Wait for that,
# first.
while loop.time() - tic < timeout:
try:
socket.create_connection((ip, port))
except socket.error as e:
app_log.warn("Socket error on boot: %s", e)
if e.errno != errno.ECONNREFUSED:
app_log.warn("Error attempting to connect to [%s:%i]: %s",
ip, port, e)
yield gen.Task(loop.add_timeout, loop.time() + wait_time)
else:
break
# Fudge factor of IPython notebook bootup.
# TODO: Implement a webhook in IPython proper to call out when the
# notebook server is booted.
yield gen.Task(loop.add_timeout, loop.time() + .5)
# Now, make sure that we can reach the Notebook server.
http_client = AsyncHTTPClient()
req = HTTPRequest("http://{}:{}/{}".format(ip, port, path))
while loop.time() - tic < timeout:
try:
yield http_client.fetch(req)
except HTTPError as http_error:
code = http_error.code
app_log.info(
"Booting server at [%s], getting HTTP status [%s]", path,
code)
yield gen.Task(loop.add_timeout, loop.time() + wait_time)
else:
break
app_log.info("Server [%s] at address [%s:%s] has booted! Have at it.",
path, ip, port)
def reraise_client_error(self, exc):
"""Remote fetch raised an error"""
try:
url = exc.response.request.url.split('?')[0]
body = exc.response.body.decode('utf8', 'replace').strip()
except AttributeError:
url = 'url'
body = ''
str_exc = str(exc)
# strip the unhelpful 599 prefix
if str_exc.startswith('HTTP 599: '):
str_exc = str_exc[10:]
if exc.code == 403 and 'too big' in body and 'gist' in url:
msg = "GitHub will not serve raw gists larger than 10MB"
elif body and len(body) < 100:
# if it's a short plain-text error message, include it
msg = "%s (%s)" % (str_exc, escape(body))
else:
msg = str_exc
slim_body = escape(body[:300])
app_log.warn("Fetching %s failed with %s. Body=%s", url, msg, slim_body)
if exc.code == 599:
if isinstance(exc, CurlError):
en = getattr(exc, 'errno', -1)
# can't connect to server should be 404
# possibly more here
if en in (pycurl.E_COULDNT_CONNECT, pycurl.E_COULDNT_RESOLVE_HOST):
raise web.HTTPError(404, msg)
# otherwise, raise 400 with informative message:
raise web.HTTPError(400, msg)
if exc.code >= 500:
# 5XX, server error, but not this server
raise web.HTTPError(502, msg)
else:
# client-side error, blame our client
if exc.code == 404:
raise web.HTTPError(404, "Remote %s" % msg)
else:
raise web.HTTPError(400, msg)
def can_show(self, path):
"""
Generally determine whether the given path is displayable.
This function is useful for failing fast - further checks may
be applied at notebook render to confirm a file may be shown.
"""
if self.settings.get('localfile_follow_symlinks'):
fullpath = os.path.realpath(os.path.join(
self.localfile_path,
path
))
else:
fullpath = os.path.abspath(os.path.normpath(os.path.join(
self.localfile_path,
path
)))
if not fullpath.startswith(self.localfile_path):
app_log.warn("directory traversal attempt: '%s'" %
fullpath)
return False
if not os.path.exists(fullpath):
return False
if any(part.startswith('.') or part.startswith('_')
for part in fullpath.split(os.sep)):
return False
if not self.settings.get('localfile_any_user'):
fstat = os.stat(fullpath)
# Ensure the file/directory has other read access for all.
if not fstat.st_mode & stat.S_IROTH:
return False
if os.path.isdir(fullpath) and not fstat.st_mode & stat.S_IXOTH:
# skip directories we can't execute (i.e. list)
return False
return True
def _wait_for_server(self, ip, port, path, timeout=10, wait_time=0.2):
'''Wait for a server to show up within a newly launched container.'''
app_log.info("Waiting for a container to launch at [%s:%s].", ip, port)
loop = ioloop.IOLoop.current()
tic = loop.time()
# Docker starts listening on a socket before the container is fully launched. Wait for that,
# first.
while loop.time() - tic < timeout:
try:
socket.create_connection((ip, port))
except socket.error as e:
app_log.warn("Socket error on boot: %s", e)
if e.errno != errno.ECONNREFUSED:
app_log.warn("Error attempting to connect to [%s:%i]: %s",
ip, port, e)
yield gen.Task(loop.add_timeout, loop.time() + wait_time)
else:
break
# Fudge factor of IPython notebook bootup.
# TODO: Implement a webhook in IPython proper to call out when the
# notebook server is booted.
yield gen.Task(loop.add_timeout, loop.time() + .5)
# Now, make sure that we can reach the Notebook server.
http_client = AsyncHTTPClient()
req = HTTPRequest("http://{}:{}/{}".format(ip, port, path))
while loop.time() - tic < timeout:
try:
yield http_client.fetch(req)
except HTTPError as http_error:
code = http_error.code
app_log.info("Booting server at [%s], getting HTTP status [%s]", path, code)
yield gen.Task(loop.add_timeout, loop.time() + wait_time)
else:
break
app_log.info("Server [%s] at address [%s:%s] has booted! Have at it.",
path, ip, port)
def handle_graqhql(self):
result = self.execute_graphql()
app_log.debug('GraphQL result data: %s errors: %s invalid %s',
result.data, result.errors, result.invalid)
if result and (result.errors or result.invalid):
ex = ExecutionError(errors=result.errors)
app_log.warn(f'GraphQL Error: {ex}. Check graphql_errors.log')
for error in result.errors:
if hasattr(error, 'original_error'):
print_graphql_exception(error.original_error)
raise error.original_error
elif isinstance(error, str):
app_log.error(error)
raise ex
response = {'data': result.data}
self.write(json_encode(response))
def _inspect_container_with_retry(self, container_id, retry_count=0, retry_wait=0.1):
"""
Inspect container with retry.
Parameters
----------
container_id : str
Id of the docker container which is related to the
ServiceInstance.
retry_count : int
Retry count.
retry_wait : float
Retry wait time on HTTPError occurred. (sec)
Returns
-------
inspect_result : dict(response)
Dict response from docker client.
Raises
------
tornado.httpclient.HTTPError
When failed to inspect container.
"""
try:
return (yield self._docker_client.inspect_container(container_id))
except HTTPError as e:
app_log.debug(
'Failed to inspect container %s. it may cause on deleting container.', container_id
)
if e.code == 404:
return None
elif retry_count < 10:
app_log.debug('retry inspect container %s...', container_id)
yield gen.sleep(retry_wait)
return (yield self._inspect_container_with_retry(
container_id, retry_count + 1, retry_wait
))
else:
app_log.warn('Retry limit is exceeded. container=%s', container_id)
raise
def manually_kill_server(user_name):
# Get our AWS server db's instance for the user
try:
server = Server.get_server(user_name)
app_log.debug("Checking server for %s manually..." % user_name)
except Server.DoesNotExist:
# it is not necessarily the case that a server will exist, we return early if that is the case.
app_log.warn("There is no matching, allocated server for user %s" % user_name)
return
# get server instance information
resource = yield retry(boto3.resource, "ec2", region_name=SERVER_PARAMS["REGION"])
instance = yield retry(resource.Instance, server.server_id)
# instance object is lazy, run this to get full info...
yield retry(instance.load)
#stop server if state is running (possible states are stopped, stopping, pending, shutting-down, terminated, and running)
if instance.state["Name"] == "running":
retry(instance.stop)
app_log.info("manually killed server for user %s" % user_name)
else:
app_log.debug("server state for user %s is %s, no action taken" % (user_name, instance.state["Name"]))
def check_hub_user(self, user_model):
"""Check whether Hub-authenticated user should be allowed.
Returns the input if the user should be allowed, None otherwise.
Override if you want to check anything other than the username's presence in hub_users list.
Args:
user_model (dict): the user model returned from :class:`HubAuth`
Returns:
user_model (dict): The user model if the user should be allowed, None otherwise.
"""
if self.hub_users is None:
# no users specified, allow any authenticated Hub user
return user_model
name = user_model['name']
if name in self.hub_users:
return user_model
else:
app_log.warn("Not allowing Hub user %s" % name)
return None
def _update_sections(self, section_name):
"""Get info from each section in the user space and port it to the
environment config file.
"""
# point the ConfigManager to the user config_dir and save the info
self.config_dir = self.user_config_dir
cfg_user = self.get(section_name)
if "load_extensions" in cfg_user:
user_extensions = cfg_user["load_extensions"]
else:
user_extensions = None
# point the ConfigManager to the environment config_dir
self.config_dir = self.environment_config_dir
cfg_environment = self.get(section_name)
# load user info into the environment space
if user_extensions is not None:
if "load_extensions" in cfg_environment:
cfg_environment["load_extensions"].update(user_extensions)
else:
cfg_environment["load_extensions"] = user_extensions
try:
# try to write updated environment config
self.update(section_name, cfg_environment)
except IOError as ex:
if ex.errno == EACCES:
# for example, if environment config dir is read-only.
# write to user config location instead.
log.warn('Environment directory %s not writeable. Saving to user directory %s instead.',
self.environment_config_dir, self.user_config_dir)
cfg_user["load_extensions"] = cfg_environment.get("load_extensions")
self.config_dir = self.user_config_dir
self.update(section_name, cfg_user)
# leave config dir pointing here after loading config
self.default_config_dir = self.user_config_dir
else:
raise
def _log_rate_limit(self, future):
"""log GitHub rate limit headers
- error if 0 remaining
- warn if 10% or less remain
- debug otherwise
"""
try:
r = future.result()
except HTTPError as e:
r = e.response
if r is None:
# some errors don't have a response (e.g. failure to build request)
return
limit_s = r.headers.get('X-RateLimit-Limit', '')
remaining_s = r.headers.get('X-RateLimit-Remaining', '')
if not remaining_s or not limit_s:
if r.code < 300:
app_log.warn("No rate limit headers. Did GitHub change? %s",
json.dumps(dict(r.headers), indent=1)
)
return
remaining = int(remaining_s)
limit = int(limit_s)
if remaining == 0 and r.code >= 400:
text = response_text(r)
try:
message = json.loads(text)['message']
except Exception:
# Can't extract message, log full reply
message = text
app_log.error("GitHub rate limit (%s) exceeded: %s", limit, message)
return
if 10 * remaining > limit:
log = app_log.info
else:
log = app_log.warn
log("%i/%i GitHub API requests remaining", remaining, limit)
