def publish(self, collection_id: str,
database_id: str) -> Optional[Collection]:
"""
Set a key value
:param collection_id:
:param database_id:
:return: Optional[Collection]
"""
try:
workspace = self._geo.get_workspace(workspace=database_id)
if workspace is None:
self._geo.create_workspace(workspace=database_id)
self._geo.create_featurestore(store_name=database_id,
workspace=database_id,
host=self._pg_host,
port=5432,
db=self._pg_db,
pg_user=self._pg_user,
pg_password=self._pg_password)
pg_table = database_id + '_' + collection_id
self._geo.publish_featurestore(workspace=database_id,
store_name=database_id,
pg_table=pg_table)
return self.get_layer(collection_id=collection_id,
database_id=database_id)
except Exception as e:
raise api.ApiError(400, str(e))
def register(subscription: Subscription, raise_on_exist: bool = True):
user = [{
"user_name": f"geodb_{subscription.guid}",
"start_date": f"{subscription.start_date}",
"subscription": f"{subscription.plan}",
"cells": int(subscription.units)
}]
client_id = util.maybe_raise_for_env("GEODB_ADMIN_CLIENT_ID")
client_secret = util.maybe_raise_for_env("GEODB_ADMIN_CLIENT_SECRET")
server_url = util.maybe_raise_for_env("GEODB_SERVER_URL")
oauth_token = dict(
audience="https://xcube-gen.brockmann-consult.de/api/v2/",
client_id=client_id,
client_secret=client_secret,
grant_type='client_secret'
)
token = oauth.get_token(oauth_token)
headers = {'Authorization': f'Bearer {token}'}
r = requests.post(f"{server_url}/geodb_user_info", json=user, headers=headers)
try:
r.raise_for_status()
except HTTPError as e:
if r.status_code == 409 and raise_on_exist is False:
return r.status_code
raise api.ApiError(r.status_code, str(e))
return True
def test_get_cubegen_info(self, p):
"""Test case for delete_cubegen
Delete a cubegen
"""
p.return_value = dict(dataset_descriptor={},
size_estimation={},
cost_estimation={})
res = cubegens.get_cubegen_info(body={},
token_info={
'user_id': 'drwho',
'email': '*****@*****.**',
'token': 'dscsdc'
})
self.assertEqual(200, res[1])
p.side_effect = api.ApiError(400, 'Error')
res = cubegens.get_cubegen_info(body={},
token_info={
'user_id': 'drwho',
'email': '*****@*****.**',
'token': 'dscsdc'
})
self.assertEqual(400, res[1])
self.assertEqual('Error', res[0]['message'])
def test_get_cubegen(self, p):
"""Test case for delete_cubegen
Delete a cubegen
"""
p.return_value = {
'cubegen_id': 'anid',
'status': 'ready',
'output': [],
'progress': 100
}
res = cubegens.get_cubegen(cubegen_id='anid',
token_info={'user_id': 'drwho'})
self.assertEqual(200, res[1])
p.side_effect = api.ApiError(400, 'Error')
res = cubegens.get_cubegen(cubegen_id='anid',
token_info={'user_id': 'drwho'})
self.assertEqual(400, res[1])
self.assertEqual('Error', res[0]['message'])
def test_get_deployment(self, list_p):
deployment = k8s.create_deployment_object(name='test',
container_name='test',
user_id='drwho',
container_port=8080,
image='cate')
list_p.return_value = V1DeploymentList(items=[deployment])
res = k8s.get_deployment(namespace='test', name='test')
self.assertIsInstance(res, V1Deployment)
self.assertEqual('test', res.metadata.name)
list_p.return_value = V1DeploymentList(items=[])
res = k8s.get_deployment(namespace='test', name='test')
self.assertIsNone(res)
list_p.side_effect = api.ApiError(400, 'Test')
with self.assertRaises(api.ApiError) as e:
k8s.get_deployment(namespace='test', name='test')
self.assertEqual('Test', str(e.exception))
self.assertEqual(400, e.exception.status_code)
def validate_env():
for var in REQUIRED_ENV_VARS:
val = os.getenv(var, None)
if val is None:
raise api.ApiError(500, f"Env var {var} required.")
return True
def _get_management_token(client_id: Optional[str] = None, client_secret: Optional[str] = None,
aud: Optional[str] = None):
client_id = client_id or os.environ.get("AUTH0_USER_MANAGEMENT_CLIENT_ID", None)
if client_id is None:
raise Unauthorized(description="Please configure the env variable AUTH0_USER_MANAGEMENT_CLIENT_ID")
client_secret = client_secret or os.environ.get("AUTH0_USER_MANAGEMENT_CLIENT_SECRET", None)
if client_secret is None:
raise Unauthorized(description="Please configure the env variable AUTH0_USER_MANAGEMENT_CLIENT_SECRET")
audience = aud or os.getenv("XCUBE_HUB_OAUTH_USER_MANAGEMENT_AUD", None)
if audience is None:
raise api.ApiError(401, "Unauthorized. System needs XCUBE_HUB_OAUTH_USER_MANAGEMENT_AUD")
payload = {
"client_id": client_id,
"client_secret": client_secret,
"audience": audience,
"grant_type": "client_credentials"
}
res = requests.post("https://edc.eu.auth0.com/oauth/token", json=payload)
try:
res.raise_for_status()
except HTTPError as e:
raise Unauthorized(description=str(e))
try:
return res.json()["access_token"]
except KeyError:
raise Unauthorized(description="System error: Could not find key 'access_token' in auth0's response")
def get_user_by_credentials(token: str, client_id: str, client_secret: str) -> Sequence:
q = f'(user_metadata.client_id: "{client_id}") AND (user_metadata.client_secret: "{client_secret}")'
headers = {'Authorization': f"Bearer {token}"}
r = requests.get('https://edc.eu.auth0.com/api/v2/users', params={'q': q}, headers=headers)
if r.status_code < 200 or r.status_code >= 300:
raise api.ApiError(400, r.text)
res = r.json()
if len(res) == 0:
raise api.ApiError(404, f"No users found.")
if len(res) > 1:
raise api.ApiError(400, f"More than one user found.")
return res
def __init__(self,
url: Optional[str] = None,
username: Optional[str] = None,
password: Optional[str] = None,
pg_user: Optional[str] = None,
pg_password: Optional[str] = None,
pg_host: Optional[str] = None,
pg_db: Optional[str] = None,
**kwargs):
super().__init__()
try:
from geo.Geoserver import Geoserver
except ImportError:
raise api.ApiError(
500, "Error: Cannot import Geoserver. Please install first.")
self._url = url or os.getenv('XCUBE_HUB_GEOSERVER_URL')
self._username = username or os.getenv('XCUBE_HUB_GEOSERVER_USERNAME')
self._password = password or os.getenv('XCUBE_HUB_GEOSERVER_PASSWORD')
self._pg_host = pg_host or os.getenv("XCUBE_HUB_POSTGIS_HOST")
self._pg_user = pg_user or os.getenv("XCUBE_HUB_POSTGIS_USER")
self._pg_password = pg_password or os.getenv(
"XCUBE_HUB_POSTGIS_PASSWORD")
self._pg_db = pg_db or os.getenv("XCUBE_HUB_POSTGIS_DB")
self._geo = Geoserver(self._url,
username=self._username,
password=self._password)
for prop, value in vars(self).items():
_raise_for_none(prop, value)
def _raise_for_invalid_punits(user_id: str, email: str, cfg: AnyDict,
token: str):
limit = os.getenv("XCUBE_HUB_PROCESS_LIMIT", 1000)
infos = info(user_id=user_id, email=email, body=cfg, token=token)
cost_estimation = infos['cost_estimation']
if cost_estimation['required'] > int(limit):
raise api.ApiError(
413,
f"Number of required punits ({cost_estimation['required']}) is "
f"greater than the absolute limit of {limit}.")
if cost_estimation['required'] > cost_estimation['available']:
raise api.ApiError(
413, f"Number of required punits ({cost_estimation['required']}) "
f"is greater than the available ({cost_estimation['available']}).")
def _validate(cls, js: JsonObject, schema: JsonObject):
try:
validate(instance=js, schema=schema)
except (ValueError, ValidationError, SchemaError) as e:
raise api.ApiError(
400, "Could not validate data pools configuration. " + str(e))
return True
def delete(user_id: str):
api_pod_instance = client.CoreV1Api()
try:
api_pod_instance.delete_namespace(name=user_id)
return True
except ApiException as e:
raise api.ApiError(400, str(e))
def get_datastore(cls, datastore: str):
try:
return cls._datapools_cfg[datastore]
except (TypeError, KeyError) as e:
raise api.ApiError(
400,
f"Error: Could not load datastore configuration. Datastore {datastore} not in config."
)
def list():
api_pod_instance = client.CoreV1Api()
try:
namespaces = api_pod_instance.list_namespace()
return [namespace.metadata.name for namespace in namespaces.items]
except ApiException as e:
raise api.ApiError(400, str(e))
def list_ingresses(namespace: str = 'default', core_api: Optional[client.NetworkingV1beta1Api] = None):
# Creation of the Deployment in specified namespace
# (Can replace "default" with a namespace you may have created)
networking_v1_beta1_api = core_api or client.NetworkingV1beta1Api()
try:
return networking_v1_beta1_api.list_namespaced_ingress(namespace=namespace)
except (ApiException, ApiTypeError) as e:
raise api.ApiError(400, f"Error when listing ingresses in namespace {namespace}: {str(e)}")
def _validate_datastores(cls, data_pools: JsonObject):
data_pools_cfg_schema_file = util.maybe_raise_for_env(
"XCUBE_HUB_CFG_DATAPOOLS_SCHEMA")
data_pools_cfg_dir = util.maybe_raise_for_env("XCUBE_HUB_CFG_DIR")
try:
with open(
os.path.join(data_pools_cfg_dir,
data_pools_cfg_schema_file), "r") as f:
data_pools_schema = yaml.safe_load(f)
except FileNotFoundError:
raise api.ApiError(404, "Could not find data pools configuration")
try:
cls._validate(js=data_pools, schema=data_pools_schema)
except (ValueError, ValidationError, SchemaError) as e:
raise api.ApiError(
400, "Could not validate data pools configuration. " + str(e))
return True
def create(user_id: str, email: str, cfg: AnyDict, token: Optional[str] = None, info_only: bool = False) -> \
Union[AnyDict, Error]:
try:
if 'input_config' not in cfg and 'input_configs' not in cfg:
raise api.ApiError(
400, "Either 'input_config' or 'input_configs' must be given")
if not info_only:
_raise_for_invalid_punits(user_id=user_id,
token=token,
email=email,
cfg=cfg)
xcube_hub_namespace = os.getenv("WORKSPACE_NAMESPACE", "xcube-gen-dev")
# Not used as the namespace cate has to be created prior to launching cubegens instances
# user_namespaces.create_if_not_exists(user_namespace=xcube_hub_namespace)
callback_uri = os.getenv('XCUBE_HUB_CALLBACK_URL', False)
if callback_uri is False:
raise api.ApiError(400, "XCUBE_HUB_CALLBACK_URL must be given")
job_id = f"{user_id}-{str(uuid.uuid4())[:18]}"
cfg['callback_config'] = dict(api_uri=callback_uri +
f'/cubegens/{job_id}/callbacks',
access_token=token)
if 'data_id' not in cfg['output_config']:
cfg['output_config']['data_id'] = job_id + '.zarr'
job = create_cubegen_object(job_id, cfg=cfg, info_only=info_only)
api_instance = client.BatchV1Api()
api_response = api_instance.create_namespaced_job(
body=job, namespace=xcube_hub_namespace)
kvdb = KeyValueDatabase.instance()
kvdb.set(user_id + '__' + job_id + '__cfg', cfg)
kvdb.set(user_id + '__' + job_id, {'progress': []})
return {'cubegen_id': job_id, 'status': api_response.status.to_dict()}
except (ApiException, MaxRetryError) as e:
raise api.ApiError(400, message=str(e))
except Exception as e:
raise api.ApiError(400, message=str(e))
def _get_dim(dims: Dict, tgt: str, tgt_alt: str):
try:
res = get_json_request_value(dims, tgt, value_type=Number)
except api.ApiError:
try:
res = get_json_request_value(dims, tgt_alt, value_type=Number)
except api.ApiError:
raise api.ApiError(400, "Cannot find a valid spatial dimension.")
return res
def get(user_id: str, cubegen_id: str) -> Union[AnyDict, Error]:
try:
outputs = logs(job_id=cubegen_id)
stat = status(job_id=cubegen_id)
if not stat:
raise api.ApiError(404, message=f"Cubegen {cubegen_id} not found")
progress = callbacks.get_callback(user_id=user_id,
cubegen_id=cubegen_id)
return {
'cubegen_id': cubegen_id,
'status': stat,
'output': outputs,
'progress': progress
}
except (ApiException, MaxRetryError) as e:
raise api.ApiError(400, str(e))
def create_configmap(namespace: str, body: client.V1ConfigMap, core_api: Optional[client.CoreV1Api] = None):
# Creation of the Deployment in specified namespace
# (Can replace "default" with a namespace you may have created)
try:
core_api = core_api or client.CoreV1Api()
core_api.create_namespaced_config_map(namespace=namespace, body=body)
except (ApiException, ApiValueError) as e:
raise api.ApiError(400,
f"Error when creating the configmap {body.metadata.name} in {namespace or 'All'}: {str(e)}")
def test_get_services(self):
res = services.get_services()
self.assertEqual((['xcube_gen', 'xcube_serve', 'xcube_geodb'], 200), res)
with patch('xcube_hub.core.services.get_services', side_effect=api.ApiError(400, 'Error')) as p:
res = services.get_services()
self.assertEqual(400, res[1])
self.assertEqual('Error', res[0]['message'])
def set(self, key, value: JsonObject):
"""
Set a key value
:param value:
:param key:
:return:
"""
try:
value = json.dumps(value)
return self._provider.set(key, value)
except JSONDecodeError as e:
raise api.ApiError(
401,
"System error (Cache): Cash contained invalid json " + str(e))
except ValueError as e:
raise api.ApiError(
401,
"System error (Cache): Cash contained invalid json " + str(e))
def admin_client(self):
try:
return KeycloakAdmin(server_url=self._server_url,
username='******',
password='******',
realm_name=self._realm,
client_id=self._client_id,
client_secret_key=self._client_secret,
verify=True)
except KeycloakError as e:
raise api.ApiError(e.response_code, str(e))
def get_subscription(self, service_id: str, subscription_id: str,
token: str):
r = requests.get(
f"https://{self._domain}/users/auth0|{subscription_id}",
headers=self._get_header(token=token))
try:
r.raise_for_status()
except HTTPError as e:
raise api.ApiError(r.status_code, str(e))
user = User.from_dict(r.json())
if service_id not in user.user_metadata.subscriptions:
raise api.ApiError(
404,
f"Subscription {subscription_id} not found in service {service_id}"
)
return user.user_metadata.subscriptions[service_id]
def exists(user_id: str):
api_pod_instance = client.CoreV1Api()
try:
namespaces = api_pod_instance.list_namespace()
user_namespace_names = [
namespace.metadata.name for namespace in namespaces.items
]
return user_id in user_namespace_names
except ApiException as e:
raise api.ApiError(400, str(e))
def create_token(claims: Dict, days_valid: int = 90):
secret = util.maybe_raise_for_env("XCUBE_HUB_TOKEN_SECRET")
if len(secret) < 256:
raise api.ApiError(400, "System Error: Invalid token secret given.")
exp = datetime.datetime.utcnow() + datetime.timedelta(days=days_valid)
claims['exp'] = exp
return jwt.encode(claims, secret, algorithm="HS256")
def put_callback(user_id: str, cubegen_id: str, value: AnyDict, email: str):
if not value or 'state' not in value:
raise api.ApiError(401, 'Callbacks need a "state"')
try:
print(f"Calling progress for {cubegen_id}.")
kvdb = KeyValueDatabase.instance()
kv = kvdb.get(user_id + '__' + cubegen_id)
if kv and 'progress' in kv and isinstance(kv['progress'], list):
kv['progress'].append(value)
else:
kv = dict(progress=[value])
res = kvdb.set(user_id + '__' + cubegen_id, kv)
sender = get_json_request_value(value, "sender", str)
state = get_json_request_value(value, 'state', dict)
if sender == 'on_end':
if 'error' not in state:
processing_request = kvdb.get(user_id + '__' + cubegen_id + '__cfg')
cube_config = processing_request['cube_config']
if 'input_configs' in processing_request:
input_config = processing_request['input_configs'][0]
elif 'input_config' in processing_request:
input_config = processing_request['input_config']
else:
raise api.ApiError(400, "Error in callbacks. Invalid input configuration.")
store_id = input_config['store_id'].replace('@', '')
datastore = Cfg.get_datastore(store_id)
punits_requests = get_size_and_cost(processing_request=cube_config, datastore=datastore)
subtract_punits(user_id=email, punits_request=punits_requests)
return kv
except (TimeoutError, ClientError) as e:
raise api.ApiError(400, "Cache timeout")
def delete_one(cubegen_id: str) -> Union[AnyDict, Error]:
api_instance = client.BatchV1Api()
xcube_hub_namespace = os.getenv("WORKSPACE_NAMESPACE", "xcube-gen-dev")
try:
api_response = api_instance.delete_namespaced_job(
name=cubegen_id,
namespace=xcube_hub_namespace,
body=client.V1DeleteOptions(propagation_policy='Background',
grace_period_seconds=5))
return api_response.status
except (ApiValueError, ApiException, MaxRetryError) as e:
raise api.ApiError(400, str(e))
def create_deployment(deployment: client.V1Deployment,
namespace: str = 'default',
core_api: Optional[client.AppsV1Api] = None):
# Create deployment
apps_v1_api = core_api or client.AppsV1Api()
try:
api_response = apps_v1_api.create_namespaced_deployment(
body=deployment,
namespace=namespace)
print("Deployment created. status='%s'" % str(api_response.status))
except (ApiException, ApiTypeError) as e:
raise api.ApiError(400, f"Error when creating the deployment {deployment.metadata.name}: {str(e)}")
def delete(self, key):
"""
Delete a key
:param key:
:return:
"""
from redis.exceptions import ConnectionError as RedisConnectionError
try:
return self._db.delete(key)
except RedisConnectionError:
raise api.ApiError(400, "System Error: redis cache not ready.")
