def _get_dim(dims: Dict, tgt: str, tgt_alt: str):
try:
res = get_json_request_value(dims, tgt, value_type=Number)
except api.ApiError:
try:
res = get_json_request_value(dims, tgt_alt, value_type=Number)
except api.ApiError:
raise api.ApiError(400, "Cannot find a valid spatial dimension.")
return res
def test_get_json_request_value_to_succeed(self):
self.assertEqual('Bibo',
get_json_request_value(dict(name='Bibo'), 'name'))
self.assertEqual(True, get_json_request_value(dict(ok=True), 'ok'))
self.assertEqual(
'Bibo',
get_json_request_value(dict(name='Bibo'), 'name', value_type=str))
self.assertEqual(
None, get_json_request_value(dict(), 'name', default_value=None))
self.assertEqual(
'Bert', get_json_request_value(dict(),
'name',
default_value='Bert'))
def info(user_id: str,
email: str,
body: JsonObject,
token: Optional[str] = None) -> JsonObject:
job = create(user_id=user_id,
email=email,
cfg=body,
info_only=True,
token=token)
apps_v1_api = client.BatchV1Api()
xcube_hub_namespace = maybe_raise_for_env("WORKSPACE_NAMESPACE", "xc-gen")
poller.poll_job_status(apps_v1_api.read_namespaced_job_status,
namespace=xcube_hub_namespace,
name=job['cubegen_id'])
state = get(user_id=user_id, cubegen_id=job['cubegen_id'])
res = state['output'][0]
if "Error" in res:
raise api.ApiError(400, res)
res = res.replace("Awaiting generator configuration JSON from TTY...", "")
res = res.replace(
f"Cube generator configuration loaded from /user-code/{job['cubegen_id']}.yaml.",
"")
res = res.replace("'", '"')
try:
processing_request = json.loads(res)
except JSONDecodeError as e:
raise api.ApiError(400, str(e), output=res)
if 'input_configs' in body:
input_config = body['input_configs'][0]
elif 'input_config' in body:
input_config = body['input_config']
else:
raise api.ApiError(400, "Error. Invalid input configuration.")
store_id = get_json_request_value(input_config,
'store_id',
value_type=str,
default_value="")
store_id = store_id.replace('@', '')
data_store = Cfg.get_datastore(store_id)
available = punits.get_punits(user_id=email)
if 'count' not in available:
raise api.ApiError(
400, "Error. Cannot handle punit data. Entry 'count' is missing.")
cost_est = costs.get_size_and_cost(processing_request=processing_request,
datastore=data_store)
required = cost_est['punits']['total_count']
limit = os.getenv("XCUBE_HUB_PROCESS_LIMIT", 1000)
return dict(dataset_descriptor=cost_est['dataset_descriptor'],
size_estimation=cost_est['size_estimation'],
cost_estimation=dict(required=required,
available=available['count'],
limit=int(limit)))
def put_callback(user_id: str, cubegen_id: str, value: AnyDict, email: str):
if not value or 'state' not in value:
raise api.ApiError(401, 'Callbacks need a "state"')
try:
print(f"Calling progress for {cubegen_id}.")
kvdb = KeyValueDatabase.instance()
kv = kvdb.get(user_id + '__' + cubegen_id)
if kv and 'progress' in kv and isinstance(kv['progress'], list):
kv['progress'].append(value)
else:
kv = dict(progress=[value])
res = kvdb.set(user_id + '__' + cubegen_id, kv)
sender = get_json_request_value(value, "sender", str)
state = get_json_request_value(value, 'state', dict)
if sender == 'on_end':
if 'error' not in state:
processing_request = kvdb.get(user_id + '__' + cubegen_id + '__cfg')
cube_config = processing_request['cube_config']
if 'input_configs' in processing_request:
input_config = processing_request['input_configs'][0]
elif 'input_config' in processing_request:
input_config = processing_request['input_config']
else:
raise api.ApiError(400, "Error in callbacks. Invalid input configuration.")
store_id = input_config['store_id'].replace('@', '')
datastore = Cfg.get_datastore(store_id)
punits_requests = get_size_and_cost(processing_request=cube_config, datastore=datastore)
subtract_punits(user_id=email, punits_request=punits_requests)
return kv
except (TimeoutError, ClientError) as e:
raise api.ApiError(400, "Cache timeout")
def _update_punits(user_id: str, punits_request: JsonObject, op: str):
update_punits = get_json_request_value(punits_request, 'punits', value_type=dict)
update_count = get_json_request_value(update_punits, 'total_count', value_type=int)
if update_count <= 0:
raise api.ApiError(400, 'Processing unit counts must be greater than zero.')
punits_data_old = get_user_data(user_id, dataset_name='punits') or dict()
count_old = punits_data_old.get('count', 0)
history_old = punits_data_old.get('history', [])
if op == 'add':
count_new = (count_old + update_count)
elif op == "sub":
count_new = (count_old - update_count)
else:
count_new = update_count
if count_new < 0:
raise api.ApiError(400, 'Out of processing units.')
history_new = [[datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), op, punits_request]] + history_old
punits_data_new = dict(punits_data_old)
punits_data_new['count'] = count_new
punits_data_new['history'] = history_new
put_user_data(user_id, punits_data_new, dataset_name='punits')
def get_size_and_cost(processing_request: JsonObject,
datastore: JsonObject) -> JsonObject:
dataset_descriptor = get_json_request_value(processing_request,
'dataset_descriptor',
value_type=dict,
item_type=dict)
data_vars = get_json_request_value(dataset_descriptor,
'data_vars',
value_type=dict,
item_type=dict)
num_variables = len(data_vars.keys())
if num_variables == 0:
raise api.ApiError(400, "Number of variables must be greater than 0.")
dims = get_json_request_value(dataset_descriptor,
'dims',
value_type=dict,
item_type=dict)
time = get_json_request_value(dims, 'time', value_type=int)
lat = _get_dim(dims, 'lat', 'y')
lon = _get_dim(dims, 'lon', 'x')
size_estimation = get_json_request_value(processing_request,
'size_estimation',
value_type=dict,
item_type=dict)
cost_params = get_json_request_value(datastore,
'cost_params',
value_type=dict,
item_type=dict)
scheme = get_json_request_value(cost_params,
'scheme',
value_type=str,
item_type=dict,
default_value='punits')
input_pixels_per_punit = 1
input_punits_weight = 0
output_pixels_per_punit = 1
output_punits_weight = 0
if scheme != 'free':
input_pixels_per_punit = get_json_request_value(
cost_params, 'input_pixels_per_punit', value_type=int)
_check_lower_bound(input_pixels_per_punit, 0)
input_punits_weight = get_json_request_value(cost_params,
'input_punits_weight',
value_type=float,
default_value=1.0)
_check_lower_bound(input_punits_weight, 0)
output_pixels_per_punit = get_json_request_value(
cost_params, 'output_pixels_per_punit', value_type=int)
_check_lower_bound(output_pixels_per_punit, 0)
output_punits_weight = get_json_request_value(cost_params,
'output_punits_weight',
value_type=float,
default_value=1.0)
_check_lower_bound(output_punits_weight, 0)
input_punits_count = _punits(lat, lon, time, num_variables,
input_pixels_per_punit)
output_punits_count = _punits(lat, lon, time, num_variables,
output_pixels_per_punit)
total_punits_count = round(
max(input_punits_weight * input_punits_count,
output_punits_weight * output_punits_count))
return dict(dataset_descriptor=dataset_descriptor,
size_estimation=size_estimation,
data_store=datastore,
punits=dict(input_count=input_punits_count,
input_weight=input_punits_weight,
output_count=output_punits_count,
output_weight=output_punits_weight,
total_count=total_punits_count))
def test_get_json_request_value_to_fail_expectedly(self):
with self.assertRaises(ApiError) as cm:
# noinspection PyTypeChecker
get_json_request_value('test', 'name')
self.assertEqual('request must be a JSON object', f'{cm.exception}')
with self.assertRaises(ApiError) as cm:
get_json_request_value(dict(), 'name')
self.assertEqual('missing request key "name"', f'{cm.exception}')
with self.assertRaises(ApiError) as cm:
get_json_request_value(dict(name=False), 'name', value_type=str)
self.assertEqual(
'value for request key "name" is of wrong type: '
'expected type str, got type bool', f'{cm.exception}')
with self.assertRaises(ApiError) as cm:
get_json_request_value(dict(ratio='zero'),
'ratio',
value_type=(float, int))
self.assertEqual(
'value for request key "ratio" is of wrong type: '
'expected type float or int, got type str', f'{cm.exception}')
with self.assertRaises(ApiError) as cm:
get_json_request_value(dict(tile_size=[5000]),
'tile_size',
value_type=list,
item_count=2,
item_type=int)
self.assertEqual(
'value for request key "tile_size" must be a list of length 2, got length 1',
f'{cm.exception}')
with self.assertRaises(ApiError) as cm:
get_json_request_value(dict(tile_size=[5000, None]),
'tile_size',
value_type=list,
item_count=2,
item_type=int)
self.assertEqual(
'value for request key "tile_size" has an item of wrong type: '
'expected type int, got type NoneType', f'{cm.exception}')