def _get_dim(dims: Dict, tgt: str, tgt_alt: str): try: res = get_json_request_value(dims, tgt, value_type=Number) except api.ApiError: try: res = get_json_request_value(dims, tgt_alt, value_type=Number) except api.ApiError: raise api.ApiError(400, "Cannot find a valid spatial dimension.") return res
def test_get_json_request_value_to_succeed(self): self.assertEqual('Bibo', get_json_request_value(dict(name='Bibo'), 'name')) self.assertEqual(True, get_json_request_value(dict(ok=True), 'ok')) self.assertEqual( 'Bibo', get_json_request_value(dict(name='Bibo'), 'name', value_type=str)) self.assertEqual( None, get_json_request_value(dict(), 'name', default_value=None)) self.assertEqual( 'Bert', get_json_request_value(dict(), 'name', default_value='Bert'))
def info(user_id: str, email: str, body: JsonObject, token: Optional[str] = None) -> JsonObject: job = create(user_id=user_id, email=email, cfg=body, info_only=True, token=token) apps_v1_api = client.BatchV1Api() xcube_hub_namespace = maybe_raise_for_env("WORKSPACE_NAMESPACE", "xc-gen") poller.poll_job_status(apps_v1_api.read_namespaced_job_status, namespace=xcube_hub_namespace, name=job['cubegen_id']) state = get(user_id=user_id, cubegen_id=job['cubegen_id']) res = state['output'][0] if "Error" in res: raise api.ApiError(400, res) res = res.replace("Awaiting generator configuration JSON from TTY...", "") res = res.replace( f"Cube generator configuration loaded from /user-code/{job['cubegen_id']}.yaml.", "") res = res.replace("'", '"') try: processing_request = json.loads(res) except JSONDecodeError as e: raise api.ApiError(400, str(e), output=res) if 'input_configs' in body: input_config = body['input_configs'][0] elif 'input_config' in body: input_config = body['input_config'] else: raise api.ApiError(400, "Error. Invalid input configuration.") store_id = get_json_request_value(input_config, 'store_id', value_type=str, default_value="") store_id = store_id.replace('@', '') data_store = Cfg.get_datastore(store_id) available = punits.get_punits(user_id=email) if 'count' not in available: raise api.ApiError( 400, "Error. Cannot handle punit data. Entry 'count' is missing.") cost_est = costs.get_size_and_cost(processing_request=processing_request, datastore=data_store) required = cost_est['punits']['total_count'] limit = os.getenv("XCUBE_HUB_PROCESS_LIMIT", 1000) return dict(dataset_descriptor=cost_est['dataset_descriptor'], size_estimation=cost_est['size_estimation'], cost_estimation=dict(required=required, available=available['count'], limit=int(limit)))
def put_callback(user_id: str, cubegen_id: str, value: AnyDict, email: str): if not value or 'state' not in value: raise api.ApiError(401, 'Callbacks need a "state"') try: print(f"Calling progress for {cubegen_id}.") kvdb = KeyValueDatabase.instance() kv = kvdb.get(user_id + '__' + cubegen_id) if kv and 'progress' in kv and isinstance(kv['progress'], list): kv['progress'].append(value) else: kv = dict(progress=[value]) res = kvdb.set(user_id + '__' + cubegen_id, kv) sender = get_json_request_value(value, "sender", str) state = get_json_request_value(value, 'state', dict) if sender == 'on_end': if 'error' not in state: processing_request = kvdb.get(user_id + '__' + cubegen_id + '__cfg') cube_config = processing_request['cube_config'] if 'input_configs' in processing_request: input_config = processing_request['input_configs'][0] elif 'input_config' in processing_request: input_config = processing_request['input_config'] else: raise api.ApiError(400, "Error in callbacks. Invalid input configuration.") store_id = input_config['store_id'].replace('@', '') datastore = Cfg.get_datastore(store_id) punits_requests = get_size_and_cost(processing_request=cube_config, datastore=datastore) subtract_punits(user_id=email, punits_request=punits_requests) return kv except (TimeoutError, ClientError) as e: raise api.ApiError(400, "Cache timeout")
def _update_punits(user_id: str, punits_request: JsonObject, op: str): update_punits = get_json_request_value(punits_request, 'punits', value_type=dict) update_count = get_json_request_value(update_punits, 'total_count', value_type=int) if update_count <= 0: raise api.ApiError(400, 'Processing unit counts must be greater than zero.') punits_data_old = get_user_data(user_id, dataset_name='punits') or dict() count_old = punits_data_old.get('count', 0) history_old = punits_data_old.get('history', []) if op == 'add': count_new = (count_old + update_count) elif op == "sub": count_new = (count_old - update_count) else: count_new = update_count if count_new < 0: raise api.ApiError(400, 'Out of processing units.') history_new = [[datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), op, punits_request]] + history_old punits_data_new = dict(punits_data_old) punits_data_new['count'] = count_new punits_data_new['history'] = history_new put_user_data(user_id, punits_data_new, dataset_name='punits')
def get_size_and_cost(processing_request: JsonObject, datastore: JsonObject) -> JsonObject: dataset_descriptor = get_json_request_value(processing_request, 'dataset_descriptor', value_type=dict, item_type=dict) data_vars = get_json_request_value(dataset_descriptor, 'data_vars', value_type=dict, item_type=dict) num_variables = len(data_vars.keys()) if num_variables == 0: raise api.ApiError(400, "Number of variables must be greater than 0.") dims = get_json_request_value(dataset_descriptor, 'dims', value_type=dict, item_type=dict) time = get_json_request_value(dims, 'time', value_type=int) lat = _get_dim(dims, 'lat', 'y') lon = _get_dim(dims, 'lon', 'x') size_estimation = get_json_request_value(processing_request, 'size_estimation', value_type=dict, item_type=dict) cost_params = get_json_request_value(datastore, 'cost_params', value_type=dict, item_type=dict) scheme = get_json_request_value(cost_params, 'scheme', value_type=str, item_type=dict, default_value='punits') input_pixels_per_punit = 1 input_punits_weight = 0 output_pixels_per_punit = 1 output_punits_weight = 0 if scheme != 'free': input_pixels_per_punit = get_json_request_value( cost_params, 'input_pixels_per_punit', value_type=int) _check_lower_bound(input_pixels_per_punit, 0) input_punits_weight = get_json_request_value(cost_params, 'input_punits_weight', value_type=float, default_value=1.0) _check_lower_bound(input_punits_weight, 0) output_pixels_per_punit = get_json_request_value( cost_params, 'output_pixels_per_punit', value_type=int) _check_lower_bound(output_pixels_per_punit, 0) output_punits_weight = get_json_request_value(cost_params, 'output_punits_weight', value_type=float, default_value=1.0) _check_lower_bound(output_punits_weight, 0) input_punits_count = _punits(lat, lon, time, num_variables, input_pixels_per_punit) output_punits_count = _punits(lat, lon, time, num_variables, output_pixels_per_punit) total_punits_count = round( max(input_punits_weight * input_punits_count, output_punits_weight * output_punits_count)) return dict(dataset_descriptor=dataset_descriptor, size_estimation=size_estimation, data_store=datastore, punits=dict(input_count=input_punits_count, input_weight=input_punits_weight, output_count=output_punits_count, output_weight=output_punits_weight, total_count=total_punits_count))
def test_get_json_request_value_to_fail_expectedly(self): with self.assertRaises(ApiError) as cm: # noinspection PyTypeChecker get_json_request_value('test', 'name') self.assertEqual('request must be a JSON object', f'{cm.exception}') with self.assertRaises(ApiError) as cm: get_json_request_value(dict(), 'name') self.assertEqual('missing request key "name"', f'{cm.exception}') with self.assertRaises(ApiError) as cm: get_json_request_value(dict(name=False), 'name', value_type=str) self.assertEqual( 'value for request key "name" is of wrong type: ' 'expected type str, got type bool', f'{cm.exception}') with self.assertRaises(ApiError) as cm: get_json_request_value(dict(ratio='zero'), 'ratio', value_type=(float, int)) self.assertEqual( 'value for request key "ratio" is of wrong type: ' 'expected type float or int, got type str', f'{cm.exception}') with self.assertRaises(ApiError) as cm: get_json_request_value(dict(tile_size=[5000]), 'tile_size', value_type=list, item_count=2, item_type=int) self.assertEqual( 'value for request key "tile_size" must be a list of length 2, got length 1', f'{cm.exception}') with self.assertRaises(ApiError) as cm: get_json_request_value(dict(tile_size=[5000, None]), 'tile_size', value_type=list, item_count=2, item_type=int) self.assertEqual( 'value for request key "tile_size" has an item of wrong type: ' 'expected type int, got type NoneType', f'{cm.exception}')