async def _kv_atomic_delete_route_parts(self, jupyterhub_routespec, route_keys):
escaped_jupyterhub_routespec = escapism.escape(
jupyterhub_routespec, safe=self.key_safe_chars
)
index, v = await self.kv_client.kv.get(escaped_jupyterhub_routespec)
if v is None:
self.log.warning("Route %s doesn't exist. Nothing to delete", routespec)
return
target = v["Value"]
escaped_target = escapism.escape(target, safe=self.key_safe_chars)
try:
status, response = await self.kv_client.txn.put(
payload=[
{"KV": {"Verb": "delete", "Key": escaped_jupyterhub_routespec}},
{"KV": {"Verb": "delete", "Key": escaped_target}},
{"KV": {"Verb": "delete", "Key": route_keys.backend_url_path}},
{"KV": {"Verb": "delete", "Key": route_keys.backend_weight_path}},
{"KV": {"Verb": "delete", "Key": route_keys.frontend_backend_path}},
{"KV": {"Verb": "delete", "Key": route_keys.frontend_rule_path}},
]
)
status = 1
response = ""
except Exception as e:
status = 0
response = str(e)
return status, response
def fetch(self, url, ref, checkout_path):
"""Fetch the contents of `url` and place it in `checkout_path`.
The `ref` parameter specifies what "version" of the contents should be
fetched. In the case of a git repository `ref` is the SHA-1 of a commit.
Iterate through possible content providers until a valid provider,
based on URL, is found.
"""
picked_content_provider = None
for ContentProvider in self.content_providers:
cp = ContentProvider()
spec = cp.detect(url, ref=ref)
if spec is not None:
picked_content_provider = cp
self.log.info(
"Picked {cp} content "
"provider.\n".format(cp=cp.__class__.__name__)
)
break
if picked_content_provider is None:
self.log.error(
"No matching content provider found for " "{url}.".format(url=url)
)
# I want to use a branch name accessible to the Europa app to display
# the current branch contents. An option is to use the hash part of
# the JupyterHub User but this is not known until after the repo2docker
# image is created.
# if isinstance(picked_content_provider, contentproviders.Git):
# suffix = datetime.now().strftime("%Y%m%d%H%M%S")
# self.branch = "europa-" + suffix
# spec["branch"] = self.branch
self.branch = ref
for log_line in picked_content_provider.fetch(
spec, checkout_path, yield_output=self.json_logs
):
self.log.info(log_line, extra=dict(phase="fetching"))
if not self.output_image_spec:
self.output_image_spec = (
"r2d" + escapism.escape(self.repo, escape_char="-").lower()
)
# if we are building from a subdirectory include that in the
# image name so we can tell builds from different sub-directories
# apart.
if self.subdir:
self.output_image_spec += escapism.escape(
self.subdir, escape_char="-"
).lower()
if picked_content_provider.content_id is not None:
self.output_image_spec += picked_content_provider.content_id
else:
self.output_image_spec += str(int(time.time()))
def get_pvc_manifest(self):
"""
Make a pvc manifest that will spawn current user's pvc.
"""
# Default set of labels, picked up from
# https://github.com/kubernetes/helm/blob/master/docs/chart_best_practices/labels.md
labels = {
'heritage': 'jupyterhub',
'app': 'jupyterhub',
'hub.jupyter.org/userid': escapism.escape(str(self.user.id))
}
labels.update(self._expand_all(self.user_storage_extra_labels))
label_selector = self._expand_all(self.user_storage_pvc_selector)
self.log.info("SEL %s" % label_selector)
self.log.info("HEEEEERE!")
self.log.info("%s %s %s %s %s %s", self.pvc_name,
self.user_storage_class, self.user_storage_access_modes,
self.user_storage_capacity, labels, label_selector)
return make_pvc(name=self.pvc_name,
storage_class=self.user_storage_class,
access_modes=self.user_storage_access_modes,
storage=self.user_storage_capacity,
labels=labels,
label_selector=label_selector)
async def start(self):
if self.repo is None:
raise ValueError("Repo2DockerSpawner.repo must be set")
resolved_ref = await resolve_ref(self.repo, self.ref)
repo_escaped = escape(self.repo, escape_char='-').lower()
image_spec = f'r2dspawner-{repo_escaped}:{resolved_ref}'
image_info = await self.inspect_image(image_spec)
if not image_info:
self.log.info(f'Image {image_spec} not present, building...')
r2d = Repo2Docker()
r2d.repo = self.repo
r2d.ref = resolved_ref
r2d.user_id = 1000
r2d.user_name = 'jovyan'
r2d.output_image_spec = image_spec
r2d.initialize()
await self.run_in_executor(r2d.build)
# HACK: DockerSpawner (and traitlets) don't seem to realize we're setting 'cmd',
# and refuse to use our custom command. Explicitly set this variable for
# now.
self._user_set_cmd = True
self.log.info(
f'Launching with image {image_spec} for {self.user.name}')
self.image = image_spec
return await super().start()
def _escape(self, s):
"""Escape a string to docker-safe characters"""
return escape(
s.lower(),
safe=string.ascii_letters + string.digits + '-',
escape_char='_',
)
def escape_docker(s):
"""Escape a string to docker-safe characters"""
return escapism.escape(
s,
safe=_docker_safe_chars,
escape_char=_docker_escape_char_docker,
)
def get_route(self, routespec):
name = escape(routespec, escape_char='-')
target = self.client.read(
self._k('backends', name, 'servers', 'notebook', 'url'))
data = json.loads(self.client.read(
self._k('frontends', name, 'extra')
))
def test_escape_default():
for s in test_strings:
e = escape(s)
assert isinstance(e, text)
u = unescape(e)
assert isinstance(u, text)
assert u == s
def __init__(self, headers):
self.authenticated = all(
[header in headers.keys() for header in self.auth_headers])
if not self.authenticated:
return
parsed_id_token = self.parse_jwt_from_headers(headers)
self.keycloak_user_id = parsed_id_token["sub"]
self.email = parsed_id_token["email"]
self.full_name = parsed_id_token["name"]
self.username = parsed_id_token["preferred_username"]
self.safe_username = escapism.escape(self.username,
escape_char="-").lower()
self.oidc_issuer = parsed_id_token["iss"]
(
self.git_url,
self.git_auth_header,
self.git_token,
) = self.git_creds_from_headers(headers)
self.gitlab_client = Gitlab(
self.git_url,
api_version=4,
oauth_token=self.git_token,
)
self.setup_k8s()
def add_route(self, routespec, backend, data):
if routespec.startswith('/'):
# url only spec
host = None
url_prefix = routespec
else:
host, url_prefix = routespec.split('/', 1)
if data is None:
data = {}
data.update({'jupyterhub-route': True})
name = escape(routespec, escape_char='-')
self.log.info(name)
self.client.write(
self._k('backends', name, 'servers', 'notebook', 'url'), backend)
self.client.write(
self._k('frontends', name, 'backend'), name)
self.client.write(
self._k('frontends', name, 'routes', 'prefix', 'rule'), 'PathPrefix:{}'.format(url_prefix))
if host:
self.client.write(self._k(
'frontends', name, 'routes', 'host', 'rule'), 'Host:{}'.format(host))
self.client.write(
self._k('frontends', name, 'extra'), json.dumps(data))
async def _kv_get_target(self, jupyterhub_routespec):
escaped_jupyterhub_routespec = escapism.escape(
jupyterhub_routespec, safe=self.key_safe_chars)
_, res = await self.kv_client.kv.get(escaped_jupyterhub_routespec)
if res is None:
return None
return res["Value"].decode()
async def _kv_get_data(self, target):
escaped_target = escapism.escape(target, safe=self.key_safe_chars)
_, res = await self.kv_client.kv.get(escaped_target)
if res is None:
return None
return res["Value"].decode()
def escaped_name(self):
if self._escaped_name is None:
self._escaped_name = escape(self.user.name,
safe=self._container_safe_chars,
escape_char=self._container_escape_char,
)
return self._escaped_name
def create_pv(username, namespace, path, storage_size):
safe_chars = set(string.ascii_lowercase + string.digits)
# Need to format the username that same way jupyterhub does.
username = escapism.escape(username, safe=safe_chars, escape_char='-').lower()
name = 'gpfs-{!s}'.format(username)
claim_name = 'claim-{!s}'.format(username)
path = os.path.join(path, username)
metadata = client.V1ObjectMeta(name=name, namespace=namespace)
claim_ref = client.V1ObjectReference(namespace=namespace, name=claim_name)
host_path = client.V1HostPathVolumeSource(path, 'DirectoryOrCreate')
spec = client.V1PersistentVolumeSpec(
access_modes=[
'ReadWriteOnce',
],
capacity={
'storage': storage_size,
},
claim_ref=claim_ref,
host_path=host_path,
storage_class_name='gpfs',
persistent_volume_reclaim_policy='Retain',
volume_mode='Filesystem')
pv = client.V1PersistentVolume('v1', 'PersistentVolume', metadata, spec)
return pv, path
def escaped_name(self):
if self._escaped_name is None:
self._escaped_name = escape(self.user.name,
safe=self._container_safe_chars,
escape_char=self._container_escape_char,
)
return self._escaped_name
def _get_route_unsafe(self, traefik_routespec):
safe = string.ascii_letters + string.digits + "_-"
escaped_routespec = escapism.escape(traefik_routespec, safe=safe)
routespec = self._routespec_from_traefik_path(traefik_routespec)
result = {"data": "", "target": "", "routespec": routespec}
def get_target_data(d, to_find):
if to_find == "url":
key = "target"
else:
key = to_find
if result[key]:
return
for k, v in d.items():
if k == to_find:
result[key] = v
if isinstance(v, dict):
get_target_data(v, to_find)
for key, value in self.routes_cache["backends"].items():
if escaped_routespec in key:
get_target_data(value, "url")
for key, value in self.routes_cache["frontends"].items():
if escaped_routespec in key:
get_target_data(value, "data")
if not result["data"] and not result["target"]:
self.log.info("No route for {} found!".format(routespec))
result = None
else:
result["data"] = json.loads(result["data"])
return result
def safe_name_for_routespec(self, routespec):
safe_chars = set(string.ascii_lowercase + string.digits)
safe_name = generate_hashed_slug(
'jupyter-' +
escapism.escape(routespec, safe=safe_chars, escape_char='-') +
'-route')
return safe_name
def interpolate_properties(spawner, template):
safe_chars = set(string.ascii_lowercase + string.digits)
username = escapism.escape(spawner.user.name,
safe=safe_chars,
escape_char='-').lower()
return template.format(userid=spawner.user.id, username=username)
def _escape(self, s):
"""Escape a string to docker-safe characters"""
return escape(
s,
safe=self._docker_safe_chars,
escape_char=self._docker_escape_char,
)
def test_subdir_in_image_name():
app = Repo2Docker(repo=TEST_REPO, subdir="a directory")
app.initialize()
app.build()
escaped_dirname = escapism.escape("a directory", escape_char="-").lower()
assert escaped_dirname in app.output_image_spec
def escape_kubernet(s):
"""Escape a string to kubernet-safe characters"""
return escapism.escape(
s,
safe=_docker_safe_chars,
escape_char=_docker_escape_char_kubernet,
)
def get_escaped_string(value):
"""
This allows me to escape names just like kubespawner does.
"""
safe_chars = set(string.ascii_lowercase + string.digits)
return escapism.escape(value, safe=safe_chars,
escape_char='-').lower().rstrip("-")
def _escape(self, s):
"""Escape a string to docker-safe characters"""
return escape(
s,
safe=self._docker_safe_chars,
escape_char=self._docker_escape_char,
)
def get_user_server_pods(user):
safe_username = escapism.escape(user["name"], escape_char="-").lower()
pods = v1.list_namespaced_pod(
kubernetes_namespace,
label_selector=
f"heritage=jupyterhub,renku.io/username={safe_username}",
)
return pods.items
def test_escape_custom_char():
for escape_char in r'\-%+_':
for s in test_strings:
e = escape(s, escape_char=escape_char)
assert isinstance(e, text)
u = unescape(e, escape_char=escape_char)
assert isinstance(u, text)
assert u == s
def test_safe_escape_char():
escape_char = "-"
safe = SAFE.union({escape_char})
with pytest.warns(RuntimeWarning):
e = escape(escape_char, safe=safe, escape_char=escape_char)
assert e == "{}{:02X}".format(escape_char, ord(escape_char))
u = unescape(e, escape_char=escape_char)
assert u == escape_char
def fetch(self, url, ref, checkout_path):
"""Fetch the contents of `url` and place it in `checkout_path`.
The `ref` parameter specifies what "version" of the contents should be
fetched. In the case of a git repository `ref` is the SHA-1 of a commit.
Iterate through possible content providers until a valid provider,
based on URL, is found.
"""
picked_content_provider = None
for ContentProvider in self.content_providers:
cp = ContentProvider()
spec = cp.detect(url, ref=ref)
if spec is not None:
picked_content_provider = cp
self.log.info(
"Picked {cp} content "
"provider.\n".format(cp=cp.__class__.__name__)
)
break
if picked_content_provider is None:
self.log.error(
"No matching content provider found for " "{url}.".format(url=url)
)
for log_line in picked_content_provider.fetch(
spec, checkout_path, yield_output=self.json_logs
):
self.log.info(log_line, extra=dict(phase="fetching"))
if not self.output_image_spec:
self.output_image_spec = (
"r2d" + escapism.escape(self.repo, escape_char="-").lower()
)
# if we are building from a subdirectory include that in the
# image name so we can tell builds from different sub-directories
# apart.
if self.subdir:
self.output_image_spec += escapism.escape(
self.subdir, escape_char="-"
).lower()
if picked_content_provider.content_id is not None:
self.output_image_spec += picked_content_provider.content_id
else:
self.output_image_spec += str(int(time.time()))
def initialize(self):
args = self.get_argparser().parse_args()
if args.debug:
self.log_level = logging.DEBUG
self.load_config_file(args.config)
if os.path.exists(args.repo):
# Let's treat this as a local directory we are building
self.repo_type = 'local'
self.repo = args.repo
self.ref = None
self.cleanup_checkout = False
else:
self.repo_type = 'remote'
self.repo = args.repo
self.ref = args.ref
self.cleanup_checkout = args.clean
if args.json_logs:
# register JSON excepthook to avoid non-JSON output on errors
sys.excepthook = self.json_excepthook
# Need to reset existing handlers, or we repeat messages
logHandler = logging.StreamHandler()
formatter = jsonlogger.JsonFormatter()
logHandler.setFormatter(formatter)
self.log.handlers = []
self.log.addHandler(logHandler)
self.log.setLevel(logging.INFO)
else:
# due to json logger stuff above,
# our log messages include carriage returns, newlines, etc.
# remove the additional newline from the stream handler
self.log.handlers[0].terminator = ''
# We don't want a [Repo2Docker] on all messages
self.log.handlers[0].formatter = logging.Formatter(
fmt='%(message)s')
if args.image_name:
self.output_image_spec = args.image_name
else:
# Attempt to set a sane default!
# HACK: Provide something more descriptive?
self.output_image_spec = 'r2d' + escapism.escape(
self.repo, escape_char='-').lower() + str(int(time.time()))
self.push = args.push
self.run = args.run
self.json_logs = args.json_logs
self.build = args.build
if not self.build:
# Can't push nor run if we aren't building
self.run = False
self.push = False
self.run_cmd = args.cmd
def safe_id(id):
"""
Make sure meeting-ids are safe
We try to keep meeting IDs to a safe subset of characters.
Not sure if Jitsi requires this, but I think it goes on some
URLs so easier to be safe.
"""
return escape(id, safe=string.ascii_letters + string.digits + '-')
def get_pod_manifest(self):
"""
Make a pod manifest that will spawn current user's notebook pod.
"""
if callable(self.singleuser_uid):
singleuser_uid = yield gen.maybe_future(self.singleuser_uid(self))
else:
singleuser_uid = self.singleuser_uid
if callable(self.singleuser_fs_gid):
singleuser_fs_gid = yield gen.maybe_future(
self.singleuser_fs_gid(self))
else:
singleuser_fs_gid = self.singleuser_fs_gid
if self.cmd:
real_cmd = self.cmd + self.get_args()
else:
real_cmd = None
# Default set of labels, picked up from
# https://github.com/kubernetes/helm/blob/master/docs/chart_best_practices/labels.md
labels = {
'heritage': 'jupyterhub',
'component': 'singleuser-server',
'app': 'jupyterhub',
'hub.jupyter.org/username': escapism.escape(self.user.name)
}
if self.name:
# FIXME: Make sure this is dns safe?
labels['hub.jupyter.org/servername'] = self.name
labels.update(self._expand_all(self.singleuser_extra_labels))
return make_pod(name=self.pod_name,
cmd=real_cmd,
port=self.port,
image_spec=self.singleuser_image_spec,
image_pull_policy=self.singleuser_image_pull_policy,
image_pull_secret=self.singleuser_image_pull_secrets,
node_selector=self.singleuser_node_selector,
run_as_uid=singleuser_uid,
fs_gid=singleuser_fs_gid,
run_privileged=self.singleuser_privileged,
env=self.get_env(),
volumes=self._expand_all(self.volumes),
volume_mounts=self._expand_all(self.volume_mounts),
working_dir=self.singleuser_working_dir,
labels=labels,
cpu_limit=self.cpu_limit,
cpu_guarantee=self.cpu_guarantee,
mem_limit=self.mem_limit,
mem_guarantee=self.mem_guarantee,
lifecycle_hooks=self.singleuser_lifecycle_hooks,
init_containers=self.singleuser_init_containers,
service_account=self.singleuser_service_account)
def test_escape_custom_safe():
safe = 'ABCDEFabcdef0123456789'
escape_char = '\\'
safe_set = set(safe + '\\')
for s in test_strings:
e = escape(s, safe=safe, escape_char=escape_char)
assert all(c in safe_set for c in e)
u = unescape(e, escape_char=escape_char)
assert u == s
def escaped_name(self):
"""Escape the username so it's safe for docker objects"""
if self._escaped_name is None:
self._escaped_name = escape(
self.user.name,
safe=self._docker_safe_chars,
escape_char=self._docker_escape_char,
).lower()
return self._escaped_name
def _generate_container_name(realm, user_name, mapping_id):
"""Generates a proper name for the container.
It combines the prefix, username and image name after escaping.
Parameters
----------
realm : string
The docker realm
user_name: string
the user name
mapping_id: string
the mapping id
Return
------
A string combining the three parameters in an appropriate container name,
plus a random token to prevent collisions with a similarly named rogue
container.
NOTE: the container name is not meant for parsing. It's only for human
consumption in the docker list. All information and all searching should
be extracted from labels.
"""
escaped_realm = escape(realm,
safe=_CONTAINER_SAFE_CHARS,
escape_char=_CONTAINER_ESCAPE_CHAR)
escaped_user_name = escape(user_name,
safe=_CONTAINER_SAFE_CHARS,
escape_char=_CONTAINER_ESCAPE_CHAR)
escaped_mapping_id = escape(mapping_id,
safe=_CONTAINER_SAFE_CHARS,
escape_char=_CONTAINER_ESCAPE_CHAR)
random_token = ''.join(random.choice(string.ascii_lowercase)
for _ in range(10))
return "{}-{}-{}-{}".format(escaped_realm,
escaped_user_name,
escaped_mapping_id,
random_token
)
def safe_name_for_routespec(self, routespec):
safe_chars = set(string.ascii_lowercase + string.digits)
safe_name = generate_hashed_slug(
'jupyter-' + escapism.escape(routespec, safe=safe_chars, escape_char='-') + '-route'
)
return safe_name
def escape(s):
"""Trivial escaping wrapper for well established stuff.
Works for containers, file names. Note that it is not destructive,
so it won't generate collisions."""
return escapism.escape(s, _ESCAPE_SAFE_CHARS, _ESCAPE_CHAR)