def test_maybe_raise_for_env(self):
# Test "all goes right"
os.environ['TT'] = "1234"
val = maybe_raise_for_env('TT')
self.assertEqual("1234", val)
# Test type conversion
val = maybe_raise_for_env('TT', typ=int)
self.assertEqual(1234, val)
self.assertIsInstance(val, int)
# Test type conversion goes wrong
os.environ['TT'] = "1234dd"
with self.assertRaises(api.ApiError) as e:
val = maybe_raise_for_env('TT', typ=int)
self.assertEqual("invalid literal for int() with base 10: '1234dd'",
str(e.exception))
# Test env var does not exist
with self.assertRaises(api.ApiError) as e:
val = maybe_raise_for_env('TT1')
self.assertEqual("Environment Variable TT1 does not exist.",
str(e.exception))
# Test env var does not exist but returns default value
val = maybe_raise_for_env('TT1', typ=int, default=0)
self.assertEqual(0, val)
self.assertIsInstance(val, int)
def register(subscription: Subscription, raise_on_exist: bool = True):
user = [{
"user_name": f"geodb_{subscription.guid}",
"start_date": f"{subscription.start_date}",
"subscription": f"{subscription.plan}",
"cells": int(subscription.units)
}]
client_id = util.maybe_raise_for_env("GEODB_ADMIN_CLIENT_ID")
client_secret = util.maybe_raise_for_env("GEODB_ADMIN_CLIENT_SECRET")
server_url = util.maybe_raise_for_env("GEODB_SERVER_URL")
oauth_token = dict(
audience="https://xcube-gen.brockmann-consult.de/api/v2/",
client_id=client_id,
client_secret=client_secret,
grant_type='client_secret'
)
token = oauth.get_token(oauth_token)
headers = {'Authorization': f'Bearer {token}'}
r = requests.post(f"{server_url}/geodb_user_info", json=user, headers=headers)
try:
r.raise_for_status()
except HTTPError as e:
if r.status_code == 409 and raise_on_exist is False:
return r.status_code
raise api.ApiError(r.status_code, str(e))
return True
def get_token(body: JsonObject):
user_client_id = util.maybe_raise_for_env("AUTH0_USER_MANAGEMENT_CLIENT_ID")
oauth_token = OauthToken.from_dict(body)
if oauth_token.client_id == user_client_id:
token = _get_management_token(client_id=oauth_token.client_id, client_secret=oauth_token.client_secret)
return token
aud = util.maybe_raise_for_env("XCUBE_HUB_OAUTH_AUD")
token = _get_management_token()
res = get_user_by_credentials(token=token,
client_id=oauth_token.client_id,
client_secret=oauth_token.client_secret)
user = User.from_dict(res[0])
permissions = users.get_permissions_by_user_id(user.user_id, token=token)
permissions = users.get_permissions(permissions=permissions)
claims = {
"iss": "https://xcube-gen.brockmann-consult.de/",
"aud": [aud],
"scope": " ".join(permissions),
"gty": "client-credentials",
"email": user.email,
"sub": users.create_user_id_from_email(user.email),
"permissions": permissions
}
if user.app_metadata and user.app_metadata.geodb_role:
claims["https://geodb.brockmann-consult.de/dbrole"] = user.app_metadata.geodb_role
return create_token(claims)
def _load_datastores(cls, store_file: Optional[str] = None):
if not cls._datapools_cfg:
data_pools_cfg_file = store_file or util.maybe_raise_for_env(
"XCUBE_HUB_CFG_DATAPOOLS")
data_pools_cfg_dir = util.maybe_raise_for_env("XCUBE_HUB_CFG_DIR")
try:
with open(
os.path.join(data_pools_cfg_dir, data_pools_cfg_file),
'r') as f:
cls._datapools_cfg = yaml.safe_load(f)
cls._validate_datastores(cls._datapools_cfg)
except FileNotFoundError:
raise api.ApiError(404,
"Could not find data pools configuration")
def check_oauthorization(token):
iss = _get_claim_from_token(token=token, tgt='iss')
# Set audience to auth0 user management audience if token claims to be a user management client token.
# Otherwise audience wil be None and defined by environment variables
aud = _get_claim_from_token(token=token, tgt='aud')
user_management_aud = maybe_raise_for_env(
"XCUBE_HUB_OAUTH_USER_MANAGEMENT_AUD")
audience = user_management_aud if user_management_aud == aud else None
# The auth provider is instantiated as a singleton.
auth = Auth(iss=iss, audience=audience)
# Not implemented yet
# AuthApi.instance(end_point=iss, token=token)
auth.verify_token(token=token)
return {
'scopes': auth.permissions,
'user_id': auth.user_id,
'email': auth.email,
'token': token,
'iss': iss,
'sub': _get_claim_from_token(token=token, tgt='sub')
}
def info(user_id: str,
email: str,
body: JsonObject,
token: Optional[str] = None) -> JsonObject:
job = create(user_id=user_id,
email=email,
cfg=body,
info_only=True,
token=token)
apps_v1_api = client.BatchV1Api()
xcube_hub_namespace = maybe_raise_for_env("WORKSPACE_NAMESPACE", "xc-gen")
poller.poll_job_status(apps_v1_api.read_namespaced_job_status,
namespace=xcube_hub_namespace,
name=job['cubegen_id'])
state = get(user_id=user_id, cubegen_id=job['cubegen_id'])
res = state['output'][0]
if "Error" in res:
raise api.ApiError(400, res)
res = res.replace("Awaiting generator configuration JSON from TTY...", "")
res = res.replace(
f"Cube generator configuration loaded from /user-code/{job['cubegen_id']}.yaml.",
"")
res = res.replace("'", '"')
try:
processing_request = json.loads(res)
except JSONDecodeError as e:
raise api.ApiError(400, str(e), output=res)
if 'input_configs' in body:
input_config = body['input_configs'][0]
elif 'input_config' in body:
input_config = body['input_config']
else:
raise api.ApiError(400, "Error. Invalid input configuration.")
store_id = get_json_request_value(input_config,
'store_id',
value_type=str,
default_value="")
store_id = store_id.replace('@', '')
data_store = Cfg.get_datastore(store_id)
available = punits.get_punits(user_id=email)
if 'count' not in available:
raise api.ApiError(
400, "Error. Cannot handle punit data. Entry 'count' is missing.")
cost_est = costs.get_size_and_cost(processing_request=processing_request,
datastore=data_store)
required = cost_est['punits']['total_count']
limit = os.getenv("XCUBE_HUB_PROCESS_LIMIT", 1000)
return dict(dataset_descriptor=cost_est['dataset_descriptor'],
size_estimation=cost_est['size_estimation'],
cost_estimation=dict(required=required,
available=available['count'],
limit=int(limit)))
def _raise_for_no_access(database_id, geodb_user, token):
geodb_server_url = util.maybe_raise_for_env('GEODB_SERVER_URL')
url = f"{geodb_server_url}/rpc/geodb_list_databases"
r = requests.post(url=url, headers={'Authorization': f'Bearer {token}'})
try:
r.raise_for_status()
except HTTPError as e:
raise api.ApiError(400, str(e))
res = r.json()
dbs = res[0]['src']
if dbs is None:
raise api.ApiError(404, f'Database {database_id} not found.')
success = False
for db in dbs:
if db['name'] == database_id and db['owner'] == geodb_user:
success = True
if success is False:
raise api.ApiError(
401,
f'The user {geodb_user} does not have access to database {database_id}'
)
def _validate_datastores(cls, data_pools: JsonObject):
data_pools_cfg_schema_file = util.maybe_raise_for_env(
"XCUBE_HUB_CFG_DATAPOOLS_SCHEMA")
data_pools_cfg_dir = util.maybe_raise_for_env("XCUBE_HUB_CFG_DIR")
try:
with open(
os.path.join(data_pools_cfg_dir,
data_pools_cfg_schema_file), "r") as f:
data_pools_schema = yaml.safe_load(f)
except FileNotFoundError:
raise api.ApiError(404, "Could not find data pools configuration")
try:
cls._validate(js=data_pools, schema=data_pools_schema)
except (ValueError, ValidationError, SchemaError) as e:
raise api.ApiError(
400, "Could not validate data pools configuration. " + str(e))
return True
def create_token(claims: Dict, days_valid: int = 90):
secret = util.maybe_raise_for_env("XCUBE_HUB_TOKEN_SECRET")
if len(secret) < 256:
raise api.ApiError(400, "System Error: Invalid token secret given.")
exp = datetime.datetime.utcnow() + datetime.timedelta(days=days_valid)
claims['exp'] = exp
return jwt.encode(claims, secret, algorithm="HS256")
def __init__(self,
iss: Optional[str] = None,
audience: Optional[str] = None,
**kwargs):
auth0_domain = util.maybe_raise_for_env("AUTH0_DOMAIN")
iss = iss or f"https://{auth0_domain}/"
provider = _ISS_TO_PROVIDER.get(iss)
self._provider = self._new_auth_provider(audience=audience,
provider=provider,
**kwargs)
self._claims = dict()
self._token = ""
def process_user_code(cfg: CubegenConfig,
user_code: Optional[FileStorage] = None):
if user_code is not None:
code_dir = uuid.uuid4().hex
code_root_dir = util.maybe_raise_for_env('XCUBE_HUB_CODE_ROOT_DIR')
filename = user_code.filename
code_dir = os.path.join(code_root_dir, code_dir)
if not os.path.isdir(code_dir):
os.mkdir(code_dir)
code_path = os.path.join(code_dir, filename)
user_code.save(code_path)
cfg.code_config.file_set.path = code_path
return cfg
def delete_cate(user_id: str, prune: bool = False) -> bool:
cate_namespace = util.maybe_raise_for_env("WORKSPACE_NAMESPACE",
default="cate")
user_namespaces.create_if_not_exists(user_namespace=cate_namespace)
deployment = k8s.get_deployment(name=user_id + '-cate',
namespace=cate_namespace)
if deployment:
k8s.delete_deployment(name=user_id + '-cate', namespace=cate_namespace)
if prune:
service_name = user_id + '-cate'
services = k8s.list_services(namespace=cate_namespace)
services = [service.metadata.name for service in services.items]
if service_name in services:
k8s.delete_service(name=service_name, namespace=cate_namespace)
return True
def add_subscription(self,
service_id: str,
subscription: Subscription,
token: str,
prefer: str = "resolution=merge-duplicates"):
user_id = util.create_user_id_from_email(subscription.email)
user = self._get_user(user_id=user_id, raising=False, token=token)
new_user = False
if user is None:
new_user = True
user = User()
user.user_id = util.create_user_id_from_email(subscription.email)
user.username = util.create_user_id_from_email(subscription.email)
user.email = subscription.email
user.first_name = subscription.first_name
user.last_name = subscription.last_name
user.user_metadata = UserUserMetadata(subscriptions={})
user = users.supplement_user(user=user, subscription=subscription)
user.blocked = False
user.email_verified = True
user.connection = "Username-Password-Xcube"
if user.app_metadata is None:
user.app_metadata = UserAppMetadata()
subscription.subscription_id = user.username
subscription.client_id = subscription.client_id or user.user_metadata.client_id
subscription.client_secret = subscription.client_secret or user.user_metadata.client_secret
if subscription.start_date is None:
subscription.start_date = datetime.datetime.now().strftime(
"%Y-%m-%d")
# EOX requires idempotent adding. However, we should think about reintroducing that as returning 409 would
# if service_id in user.user_metadata.subscriptions:
# raise api.ApiError(409, f"The subscription {subscription.subscription_id} exists for service {service_id}.")
role_id = None
if service_id == "xcube_geodb":
role_id_manage = util.maybe_raise_for_env(
"GEODB_AUTH_ROLE_ID_MANAGE")
role_id_free = util.maybe_raise_for_env("GEODB_AUTH_ROLE_ID_FREE")
role_id_user = util.maybe_raise_for_env("GEODB_AUTH_ROLE_ID_USER")
if subscription.unit != "cells":
raise api.ApiError(400, "Wrong unit for a geodb subscription")
if subscription.plan == "manage":
role_id = role_id_manage
elif subscription.plan == "freetrial":
role_id = role_id_free
else:
role_id = role_id_user
user.app_metadata = UserAppMetadata(geodb_role="geodb_" +
subscription.guid)
geodb.register(subscription=subscription, raise_on_exist=False)
roles = {"roles": [role_id_manage, role_id_free, role_id_user]}
requests.delete(
f"https://{self._domain}/users/auth0|{user_id}/roles",
json=roles,
headers=self._get_header(token=token))
if service_id == "xcube_gen":
if subscription.unit != "punits":
raise api.ApiError(400,
"Wrong unit for a xcube gen subscription")
role_id = util.maybe_raise_for_env("XCUBE_GEN_ROLE_ID")
try:
punits.override_punits(
user_id=user.email,
punits_request=dict(punits=dict(
total_count=int(subscription.units))))
except (DatabaseError, ClientError) as e:
raise api.ApiError(400, str(e))
if service_id == "xcube_geoserv":
role_id = util.maybe_raise_for_env("XCUBE_GEOSERV_ID")
subscription.role = role_id
if new_user:
user.user_metadata.subscriptions[service_id] = subscription
user_dict = get_request_body_from_user(user)
r = requests.post(f"https://{self._domain}/users",
json=user_dict,
headers=self._get_header(token=token))
else:
user.user_metadata.subscriptions[service_id] = subscription
user_dict = dict(user_metadata=user.user_metadata.to_dict(),
app_metadata=user.app_metadata.to_dict())
r = requests.patch(f"https://{self._domain}/users/auth0|{user_id}",
json=user_dict,
headers=self._get_header(token=token))
with open('debug.txt', 'a') as f:
f.write('__________________________________\n\n')
f.write(json.dumps(self._get_header(token=token)) + '\n\n')
f.write(self._domain + '\n\n')
f.write('\n\n')
try:
r.raise_for_status()
except HTTPError as e:
raise api.ApiError(r.status_code, str(e))
if new_user:
role = {"roles": [role_id]}
r = requests.post(
f"https://{self._domain}/users/auth0|{user_id}/roles",
json=role,
headers=self._get_header(token=token))
else:
role = {"roles": [role_id]}
r = requests.post(
f"https://{self._domain}/users/auth0|{user_id}/roles",
json=role,
headers=self._get_header(token=token))
try:
r.raise_for_status()
except HTTPError as e:
raise api.ApiError(r.status_code, str(e))
return subscription
def create_cubegen_object(cubegen_id: str,
cfg: AnyDict,
info_only: bool = False) -> client.V1Job:
# Configure Pod template container
sh_client_id = os.environ.get("SH_CLIENT_ID")
sh_client_secret = os.environ.get("SH_CLIENT_SECRET")
sh_instance_id = os.environ.get("SH_INSTANCE_ID")
xcube_repo = util.maybe_raise_for_env("XCUBE_REPO")
xcube_tag = util.maybe_raise_for_env("XCUBE_TAG")
gen_container_pull_policy = os.environ.get("XCUBE_GEN_DOCKER_PULL_POLICY")
cdsapi_url = os.getenv("CDSAPI_URL")
cdsapi_key = os.getenv("CDSAPI_KEY")
aws_access_key_id = os.getenv("AWS_ACCESS_KEY_ID")
aws_secret_access_key = os.getenv("AWS_SECRET_ACCESS_KEY")
xcube_hub_cfg_dir = util.maybe_raise_for_env("XCUBE_HUB_CFG_DIR")
xcube_hub_cfg_datapools = util.maybe_raise_for_env(
"XCUBE_HUB_CFG_DATAPOOLS")
stores_file = os.path.join(xcube_hub_cfg_dir, xcube_hub_cfg_datapools)
gen_image = xcube_repo + ':' + xcube_tag
if not sh_client_secret or not sh_client_id or not sh_instance_id:
raise api.ApiError(400, "SentinelHub credentials not set.")
if not cdsapi_url or not cdsapi_key:
raise api.ApiError(400, "CDS credentials not set.")
if not cfg:
raise api.ApiError(400,
"create_gen_cubegen_object needs a config dict.")
info_flag = " -i " if info_only else ""
xcube_hub_code_root_dir = util.maybe_raise_for_env(
'XCUBE_HUB_CODE_ROOT_DIR')
cfg_file = os.path.join(xcube_hub_code_root_dir, cubegen_id + '.yaml')
with open(cfg_file, 'w') as f:
json.dump(cfg, f)
cmd = [
"/bin/bash", "-c",
f"source activate xcube && xcube --traceback gen2 -vv "
f"{info_flag} --stores {stores_file} {cfg_file}"
]
sh_envs = [
client.V1EnvVar(name="SH_CLIENT_ID", value=sh_client_id),
client.V1EnvVar(name="SH_CLIENT_SECRET", value=sh_client_secret),
client.V1EnvVar(name="SH_INSTANCE_ID", value=sh_instance_id),
client.V1EnvVar(name="CDSAPI_URL", value=cdsapi_url),
client.V1EnvVar(name="CDSAPI_KEY", value=cdsapi_key),
client.V1EnvVar(name="AWS_ACCESS_KEY_ID", value=aws_access_key_id),
client.V1EnvVar(name="AWS_SECRET_ACCESS_KEY",
value=aws_secret_access_key)
]
volume_mounts = [
{
'name': 'xcube-hub-stores',
'mountPath': '/etc/xcube-hub',
'readOnly': True
},
{
'name': 'workspace-pvc',
'mountPath': '/user-code',
},
]
volumes = [
{
'name': 'xcube-hub-stores',
'configMap': {
'name': 'xcube-hub-stores'
}
},
{
'name': 'workspace-pvc',
'persistentVolumeClaim': {
'claimName': 'workspace-pvc',
}
},
]
container = client.V1Container(name="xcube-gen",
image=gen_image,
command=cmd,
volume_mounts=volume_mounts,
image_pull_policy=gen_container_pull_policy,
env=sh_envs)
# Create and configure a spec section
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={"app": "xcube-gen"}),
spec=client.V1PodSpec(volumes=volumes,
restart_policy="Never",
containers=[container]))
# Create the specification of deployment
spec = client.V1JobSpec(template=template, backoff_limit=1)
# Instantiate the cubegen object
cubegen = client.V1Job(api_version="batch/v1",
kind="Job",
metadata=client.V1ObjectMeta(name=cubegen_id),
spec=spec)
return cubegen
def launch_cate(user_id: str) -> JsonObject:
try:
max_pods = util.maybe_raise_for_env("CATE_MAX_WEBAPIS",
default=50,
typ=int)
grace = util.maybe_raise_for_env("CATE_LAUNCH_GRACE",
default=2,
typ=int)
raise_for_invalid_username(user_id)
cate_image = util.maybe_raise_for_env("CATE_IMG")
cate_version = util.maybe_raise_for_env("CATE_VERSION")
# cate_command = util.maybe_raise_for_env("CATE_COMMAND", default=None)
# cate_env_activate_command = util.maybe_raise_for_env("CATE_ENV_ACTIVATE_COMMAND", default=None)
cate_webapi_uri = util.maybe_raise_for_env("CATE_WEBAPI_URI")
cate_namespace = util.maybe_raise_for_env("WORKSPACE_NAMESPACE",
"cate")
cate_stores_config_path = util.maybe_raise_for_env(
"CATE_STORES_CONFIG_PATH", default="/etc/xcube-hub/stores.yaml")
# Not used as the namespace cate has to be created prior to launching cate instances
# user_namespaces.create_if_not_exists(user_namespace=cate_namespace)
if k8s.count_pods(label_selector="purpose=cate-webapi",
namespace=cate_namespace) > max_pods:
raise api.ApiError(413, "Too many pods running.")
cate_command = "cate-webapi-start -b -p 4000 -a 0.0.0.0 -r /home/cate/workspace"
cate_env_activate_command = "source activate cate-env"
cate_image = cate_image + ':' + cate_version
command = [
"/bin/bash", "-c", f"{cate_env_activate_command} && {cate_command}"
]
envs = [
client.V1EnvVar(name='CATE_USER_ROOT',
value="/home/cate/workspace"),
client.V1EnvVar(name='CATE_STORES_CONFIG_PATH',
value=cate_stores_config_path),
client.V1EnvVar(name='JUPYTERHUB_SERVICE_PREFIX',
value='/' + user_id + '/')
]
volume_mounts = [
{
'name': 'workspace-pvc',
'mountPath': '/home/cate/workspace',
'subPath': user_id + '-scratch'
},
{
'name': 'workspace-pvc',
'mountPath': '/home/cate/.cate',
'subPath': user_id + '-cate'
},
{
'name': 'xcube-hub-stores',
'mountPath': '/etc/xcube-hub',
'readOnly': True
},
]
volumes = [
{
'name': 'workspace-pvc',
'persistentVolumeClaim': {
'claimName': 'workspace-pvc',
}
},
{
'name': 'xcube-hub-stores',
'configMap': {
'name': 'xcube-hub-stores'
}
},
]
init_containers = [
{
"name":
"fix-owner",
"image":
"bash",
"command": [
"chown", "-R", "1000.1000", "/home/cate/.cate",
"/home/cate/workspace"
],
"volumeMounts": [
{
"mountPath": "/home/cate/.cate",
"subPath": user_id + '-cate',
"name": "workspace-pvc",
},
{
"mountPath": "/home/cate/workspace",
"subPath": user_id + '-scratch',
"name": "workspace-pvc",
},
]
},
]
deployment = k8s.create_deployment_object(
name=user_id + '-cate',
user_id=user_id,
container_name=user_id + '-cate',
image=cate_image,
envs=envs,
container_port=4000,
command=command,
volumes=volumes,
volume_mounts=volume_mounts,
init_containers=init_containers)
# Make create_if_exists test for broken pods
# pod_status = get_status(user_id)
# if pod_status != "Running":
# create_deployment(namespace=user_id, deployment=deployment)
# else:
k8s.create_deployment_if_not_exists(namespace=cate_namespace,
deployment=deployment)
service = k8s.create_service_object(name=user_id + '-cate',
port=4000,
target_port=4000)
k8s.create_service_if_not_exists(service=service,
namespace=cate_namespace)
host_uri = os.environ.get("CATE_WEBAPI_URI")
service_name = user_id + '-cate'
ingress = k8s.get_ingress(namespace=cate_namespace, name=service_name)
if not ingress:
ingress = k8s.create_ingress_object(name=service_name,
service_name=service_name,
service_port=4000,
user_id=user_id,
host_uri=host_uri)
k8s.create_ingress(ingress, namespace=cate_namespace)
# add_cate_path_to_ingress(
# name='xcubehub-cate',
# namespace=cate_namespace,
# user_id=user_id,
# host_uri=host_uri
# )
poller.poll_pod_phase(get_pod,
namespace=cate_namespace,
prefix=user_id)
try:
grace = int(grace)
except ValueError as e:
raise api.ApiError(400, "Grace wait period must be an integer.")
time.sleep(int(grace))
return dict(serverUrl=f'https://{cate_webapi_uri}/{user_id}')
except ApiException as e:
raise api.ApiError(e.status, str(e))