def _get_default_open_params_schema(self) -> JsonObjectSchema:
params = dict(
dataset_name=JsonStringSchema(min_length=1,
enum=list(
self._handler_registry.keys())),
variable_names=JsonArraySchema(
items=(JsonStringSchema(min_length=0)), unique_items=True),
crs=JsonStringSchema(),
# W, S, E, N
bbox=JsonArraySchema(
items=(JsonNumberSchema(minimum=-180, maximum=180),
JsonNumberSchema(minimum=-90, maximum=90),
JsonNumberSchema(minimum=-180, maximum=180),
JsonNumberSchema(minimum=-90, maximum=90))),
spatial_res=JsonNumberSchema(),
time_range=JsonDateSchema.new_range(),
time_period=JsonStringSchema(),
)
required = [
'variable_names',
'bbox',
'spatial_res',
'time_range',
]
return JsonObjectSchema(properties=params, required=required)
def get_write_data_params_schema(self) -> JsonObjectSchema:
return JsonObjectSchema(properties=dict(
group=JsonStringSchema(
description='Group path. (a.k.a. path in zarr terminology.).',
min_length=1,
),
encoding=JsonObjectSchema(
description='Nested dictionary with variable names as keys and '
'dictionaries of variable specific encodings as values.',
examples=[{
'my_variable': {
'dtype': 'int16',
'scale_factor': 0.1,
}
}],
additional_properties=True,
),
consolidated=JsonBooleanSchema(
description='If True, apply zarr’s consolidate_metadata() '
'function to the store after writing.'),
append_dim=JsonStringSchema(
description=
'If set, the dimension on which the data will be appended.',
min_length=1,
)),
required=[],
additional_properties=False)
def get_schema(cls) -> JsonObjectSchema:
return JsonObjectSchema(properties=dict(
access_token=JsonStringSchema(min_length=1),
token_type=JsonStringSchema(min_length=1)),
required=['access_token', 'token_type'],
additional_properties=False,
factory=cls)
def get_schema(cls):
return JsonObjectSchema(
properties=dict(api_uri=JsonStringSchema(min_length=1),
access_token=JsonStringSchema(min_length=1)),
additional_properties=False,
required=["api_uri", "access_token"],
factory=cls,
)
def get_open_data_params_schema(self,
data_id: str = None) -> JsonObjectSchema:
return JsonObjectSchema(properties=dict(
group=JsonStringSchema(
description='Group path. (a.k.a. path in zarr terminology.).',
min_length=1,
),
chunks=JsonObjectSchema(
description=
'Optional chunk sizes along each dimension. Chunk size values may '
'be None, "auto" or an integer value.',
examples=[{
'time': None,
'lat': 'auto',
'lon': 90
}, {
'time': 1,
'y': 512,
'x': 512
}],
additional_properties=True,
),
decode_cf=JsonBooleanSchema(
description=
'Whether to decode these variables, assuming they were saved '
'according to CF conventions.',
default=True,
),
mask_and_scale=JsonBooleanSchema(
description=
'If True, replace array values equal to attribute "_FillValue" with NaN. '
'Use "scaling_factor" and "add_offset" attributes to compute actual values.',
default=True,
),
decode_times=JsonBooleanSchema(
description=
'If True, decode times encoded in the standard NetCDF datetime format '
'into datetime objects. Otherwise, leave them encoded as numbers.',
default=True,
),
decode_coords=JsonBooleanSchema(
description=
'If True, decode the \"coordinates\" attribute to identify coordinates in '
'the resulting dataset.',
default=True,
),
drop_variables=JsonArraySchema(
items=JsonStringSchema(min_length=1), ),
consolidated=JsonBooleanSchema(
description=
'Whether to open the store using zarr\'s consolidated metadata '
'capability. Only works for stores that have already been consolidated.',
default=False,
),
),
required=[],
additional_properties=False)
def test_to_instance(self):
self.assertEqual('pieps', JsonStringSchema().to_instance('pieps'))
self.assertEqual(
'pieps',
JsonStringSchema(min_length=0, max_length=10).to_instance('pieps'))
self.assertEqual('pieps',
JsonStringSchema(pattern='.*').to_instance('pieps'))
self.assertEqual(
'2020-01-03T03:30:01.99+03:30',
JsonStringSchema(format='date-time').to_instance(
'2020-01-03T03:30:01.99+03:30'))
def get_schema(cls) -> JsonObjectSchema:
return JsonObjectSchema(properties=dict(
job_id=JsonStringSchema(min_length=1),
job_status=CubeGeneratorJobStatus.get_schema(),
job_result=CubeGeneratorResult.get_schema(),
output=JsonArraySchema(items=JsonStringSchema(), nullable=True),
progress=JsonArraySchema(items=CubeGeneratorProgress.get_schema(),
nullable=True)),
required=['job_id', 'job_status'],
additional_properties=True,
factory=cls)
def get_schema(cls) -> JsonObjectSchema:
return JsonObjectSchema(properties=dict(
name=JsonStringSchema(min_length=1),
dtype=JsonStringSchema(min_length=1),
dims=JsonArraySchema(items=JsonStringSchema(min_length=1)),
chunks=JsonArraySchema(items=JsonIntegerSchema(minimum=0)),
attrs=JsonObjectSchema(additional_properties=True),
),
required=['name', 'dtype', 'dims'],
additional_properties=False,
factory=cls)
def test_to_dict(self):
self.assertEqual({'type': 'string'}, JsonStringSchema().to_dict())
self.assertEqual({'type': 'string'},
JsonStringSchema(nullable=False).to_dict())
self.assertEqual({'type': ['string', 'null']},
JsonStringSchema(nullable=True).to_dict())
self.assertEqual({
'type': 'string',
'format': 'uri'
},
JsonStringSchema(format='uri').to_dict())
def get_schema(cls) -> JsonObjectSchema:
return JsonObjectSchema(properties=dict(
succeeded=JsonIntegerSchema(nullable=True),
failed=JsonIntegerSchema(nullable=True),
active=JsonIntegerSchema(nullable=True),
start_time=JsonStringSchema(nullable=True),
completion_time=JsonStringSchema(nullable=True),
conditions=JsonArraySchema(
items=JsonObjectSchema(additional_properties=True),
nullable=True)),
additional_properties=True,
factory=cls)
def get_schema(cls) -> JsonObjectSchema:
return JsonObjectSchema(
properties=dict(
store_id=JsonStringSchema(min_length=1),
opener_id=JsonStringSchema(min_length=1),
data_id=JsonStringSchema(min_length=1),
store_params=JsonObjectSchema(additional_properties=True),
open_params=JsonObjectSchema(additional_properties=True)),
additional_properties=False,
required=['data_id'],
factory=cls,
)
def get_schema(cls) -> JsonObjectSchema:
return JsonObjectSchema(
properties=dict(
endpoint_url=JsonStringSchema(min_length=1),
client_id=JsonStringSchema(min_length=1),
client_secret=JsonStringSchema(min_length=1),
access_token=JsonStringSchema(min_length=1),
),
additional_properties=False,
required=[],
factory=cls,
)
def get_schema(cls) -> JsonObjectSchema:
"""Get the JSON schema for CodeConfig objects."""
return JsonObjectSchema(
properties=dict(
callable_ref=JsonStringSchema(min_length=1),
callable_params=JsonObjectSchema(additional_properties=True),
inline_code=JsonStringSchema(min_length=1),
file_set=FileSet.get_schema(),
install_required=JsonBooleanSchema(),
),
additional_properties=False,
factory=cls,
)
def test_base_props_validated(self):
with self.assertRaises(ValueError) as cm:
JsonComplexSchema()
self.assertEqual('exactly one of one_of, any_of, all_of must be given',
f'{cm.exception}')
with self.assertRaises(ValueError) as cm:
JsonComplexSchema(one_of=[JsonStringSchema(),
JsonIntegerSchema()],
all_of=[JsonStringSchema(),
JsonIntegerSchema()])
self.assertEqual('exactly one of one_of, any_of, all_of must be given',
f'{cm.exception}')
def get_open_data_params_schema(self, data_id: Optional[str] = None) -> \
JsonObjectSchema:
# If the data_id has a product type suffix, remove it.
dataset_id = data_id.split(':')[0] if ':' in data_id else data_id
ds_info = self._dataset_dicts[dataset_id]
variable_info_table = ds_info['variables']
bbox = ds_info['bbox']
params = dict(
dataset_name=JsonStringSchema(
min_length=1,
enum=list(self._valid_data_ids),
description='identifier of the requested dataset'),
variable_names=JsonArraySchema(
items=(JsonStringSchema(
min_length=0,
enum=[
cds_api_name
for cds_api_name, _, _, _ in variable_info_table
])),
unique_items=True,
nullable=True,
description='identifiers of the requested variables'),
crs=JsonStringSchema(nullable=True,
default=ds_info['crs'],
enum=[None, ds_info['crs']],
description='co-ordinate reference system'),
# W, S, E, N (will be converted to N, W, S, E)
bbox=JsonArraySchema(
items=(JsonNumberSchema(minimum=bbox[0], maximum=bbox[2]),
JsonNumberSchema(minimum=bbox[1], maximum=bbox[3]),
JsonNumberSchema(minimum=bbox[0], maximum=bbox[2]),
JsonNumberSchema(minimum=bbox[1], maximum=bbox[3])),
description='bounding box (min_x, min_y, max_x, max_y)'),
spatial_res=JsonNumberSchema(minimum=ds_info['spatial_res'],
maximum=10,
default=ds_info['spatial_res'],
description='spatial resolution'),
time_range=JsonDateSchema.new_range(),
time_period=JsonStringSchema(
const=ds_info['time_period'],
description='time aggregation period'),
)
required = [
'variable_names',
'bbox',
'spatial_res',
'time_range',
]
return JsonObjectSchema(properties=params, required=required)
def get_schema(cls) -> JsonObjectSchema:
"""Get the JSON-schema for FileSet objects."""
return JsonObjectSchema(
properties=dict(
path=JsonStringSchema(min_length=1),
sub_path=JsonStringSchema(min_length=1),
storage_params=JsonObjectSchema(additional_properties=True),
includes=JsonArraySchema(items=JsonStringSchema(min_length=1)),
excludes=JsonArraySchema(items=JsonStringSchema(min_length=1)),
),
additional_properties=False,
required=['path'],
factory=cls,
)
def get_schema(cls):
return JsonObjectSchema(
properties=dict(
store_id=JsonStringSchema(min_length=1),
writer_id=JsonStringSchema(min_length=1),
data_id=JsonStringSchema(default=None),
store_params=JsonObjectSchema(additional_properties=True),
write_params=JsonObjectSchema(additional_properties=True),
replace=JsonBooleanSchema(default=False),
),
additional_properties=False,
required=[],
factory=cls,
)
def get_schema(cls) -> JsonObjectSchema:
return JsonObjectSchema(
properties=dict(
status=JsonStringSchema(enum=STATUS_IDS),
status_code=JsonIntegerSchema(),
result=cls.get_result_schema(),
message=JsonStringSchema(),
output=JsonArraySchema(items=JsonStringSchema()),
traceback=JsonArraySchema(items=JsonStringSchema()),
versions=JsonObjectSchema(additional_properties=True)),
required=['status'],
additional_properties=True,
factory=cls,
)
def test_from_instance_tuple(self):
self.assertEqual([False, 2, 'U'],
JsonArraySchema(items=[
JsonBooleanSchema(),
JsonIntegerSchema(),
JsonStringSchema()
]).from_instance([False, 2, 'U']))
def get_schema(cls) -> JsonObjectSchema:
return JsonObjectSchema(
properties=dict(data_id=JsonStringSchema(min_length=1), ),
required=['data_id'],
additional_properties=False,
factory=cls,
)
def test_from_json_object_additional_properties_is_schema(self):
Person = namedtuple('Person', ['name', 'age', 'deleted'])
person_schema = JsonObjectSchema(
properties=dict(name=JsonStringSchema(),
age=JsonIntegerSchema(),
deleted=JsonBooleanSchema(default=False)),
factory=Person,
)
schema = JsonObjectSchema(additional_properties=person_schema, )
value = {
'p1': {
'name': 'Bibo',
'age': 15,
'deleted': True
},
'p2': {
'name': 'Ernie',
'age': 12,
'deleted': False
},
}
self.assertEqual(
{
'p1': Person(name='Bibo', age=15, deleted=True),
'p2': Person(name='Ernie', age=12, deleted=False),
}, schema.from_instance(value))
def test_from_json_object_object(self):
person_schema = JsonObjectSchema(
properties=dict(name=JsonStringSchema(),
age=JsonIntegerSchema(),
deleted=JsonBooleanSchema(default=False)))
schema = JsonObjectSchema(properties=dict(person=person_schema))
value = {'person': {'name': 'Bibo', 'age': 15}}
self.assertEqual(
{'person': {
'name': 'Bibo',
'age': 15,
'deleted': False
}}, schema.from_instance(value))
Assignment = namedtuple('Assignment', ['person'])
schema.factory = Assignment
self.assertEqual(
Assignment(person={
'name': 'Bibo',
'age': 15,
'deleted': False
}), schema.from_instance(value))
Person = namedtuple('Person', ['name', 'age', 'deleted'])
person_schema.factory = Person
self.assertEqual(
Assignment(person=Person(name='Bibo', age=15, deleted=False)),
schema.from_instance(value))
def test_to_json_object(self):
person_schema = JsonObjectSchema(
properties=dict(name=JsonStringSchema(),
age=JsonIntegerSchema(),
deleted=JsonBooleanSchema(default=False)))
value = {'name': 'Bibo', 'age': 12, 'deleted': True}
self.assertEqual(value, person_schema.to_instance(value))
# ok, because person_schema does not explicitly say additional_properties=False
value_extra = {
'name': 'Bibo',
'age': 12,
'deleted': True,
'comment': 'Hello!'
}
self.assertEqual(value_extra, person_schema.to_instance(value_extra))
Person = namedtuple('Person', ['name', 'age', 'deleted'])
def serialize(person: Person) -> Dict[str, Any]:
return person._asdict()
person_schema.serializer = serialize
person = Person(**value)
self.assertEqual(value, person_schema.to_instance(person))
def _get_open_data_params_schema(
dsd: DatasetDescriptor = None) -> JsonObjectSchema:
min_date = dsd.time_range[0] if dsd and dsd.time_range else None
max_date = dsd.time_range[1] if dsd and dsd.time_range else None
# noinspection PyUnresolvedReferences
cube_params = dict(
variable_names=JsonArraySchema(items=JsonStringSchema(
enum=dsd.data_vars.keys() if dsd and dsd.data_vars else None)),
time_range=JsonDateSchema.new_range(min_date, max_date))
if dsd and (('lat' in dsd.dims and 'lon' in dsd.dims) or
('latitude' in dsd.dims and 'longitude' in dsd.dims)):
min_lon = dsd.bbox[0] if dsd and dsd.bbox else -180
min_lat = dsd.bbox[1] if dsd and dsd.bbox else -90
max_lon = dsd.bbox[2] if dsd and dsd.bbox else 180
max_lat = dsd.bbox[3] if dsd and dsd.bbox else 90
bbox = JsonArraySchema(
items=(JsonNumberSchema(minimum=min_lon, maximum=max_lon),
JsonNumberSchema(minimum=min_lat, maximum=max_lat),
JsonNumberSchema(minimum=min_lon, maximum=max_lon),
JsonNumberSchema(minimum=min_lat, maximum=max_lat)))
cube_params['bbox'] = bbox
cci_schema = JsonObjectSchema(properties=dict(**cube_params),
required=[],
additional_properties=False)
return cci_schema
def get_data_store_params_schema(cls) -> JsonObjectSchema:
return JsonObjectSchema(properties=dict(
root=JsonStringSchema(default=''),
max_depth=JsonIntegerSchema(nullable=True, default=1),
read_only=JsonBooleanSchema(default=False),
),
additional_properties=False)
def get_process_params_schema(cls) -> JsonObjectSchema:
return JsonObjectSchema(
properties=dict(
name=JsonStringSchema(min_length=1),
value=JsonIntegerSchema(minimum=1),
),
required=['name', 'value'],
additional_properties=False,
)
def get_data_store_params_schema(cls) -> JsonObjectSchema:
cciodp_params = dict(
endpoint_url=JsonStringSchema(default=OPENSEARCH_CEDA_URL),
endpoint_description_url=JsonStringSchema(default=CCI_ODD_URL),
enable_warnings=JsonBooleanSchema(
default=False, title='Whether to output warnings'),
num_retries=JsonIntegerSchema(
default=DEFAULT_NUM_RETRIES,
minimum=0,
title='Number of retries when requesting data fails'),
retry_backoff_max=JsonIntegerSchema(
default=DEFAULT_RETRY_BACKOFF_MAX, minimum=0),
retry_backoff_base=JsonNumberSchema(
default=DEFAULT_RETRY_BACKOFF_BASE, exclusive_minimum=1.0),
user_agent=JsonStringSchema(default=None))
return JsonObjectSchema(properties=dict(**cciodp_params),
required=None,
additional_properties=False)
def get_schema(cls) -> JsonObjectSchema:
return JsonObjectSchema(properties=dict(
data_id=JsonStringSchema(min_length=1),
data_type=DataType.get_schema(),
crs=JsonStringSchema(min_length=1),
bbox=JsonArraySchema(items=[
JsonNumberSchema(),
JsonNumberSchema(),
JsonNumberSchema(),
JsonNumberSchema()
]),
time_range=JsonDateSchema.new_range(nullable=True),
time_period=JsonStringSchema(min_length=1),
open_params_schema=JsonObjectSchema(additional_properties=True),
),
required=['data_id', 'data_type'],
additional_properties=True,
factory=cls)
def get_schema(cls) -> JsonObjectSchema:
return JsonObjectSchema(properties=dict(
sender=JsonStringSchema(),
state=CubeGeneratorProgressState.get_schema(),
),
required=[
'sender',
'state',
],
additional_properties=False,
factory=cls)
def get_schema(cls):
return JsonObjectSchema(properties=dict(
variable_names=JsonArraySchema(
items=JsonStringSchema(min_length=1), min_items=0),
crs=JsonStringSchema(nullable=True, min_length=1),
bbox=JsonArraySchema(nullable=True,
items=[
JsonNumberSchema(),
JsonNumberSchema(),
JsonNumberSchema(),
JsonNumberSchema()
]),
spatial_res=JsonNumberSchema(nullable=True, exclusive_minimum=0.0),
time_range=JsonDateSchema.new_range(nullable=True),
time_period=JsonStringSchema(nullable=True,
pattern=r'^([1-9][0-9]*)?[DWMY]$'),
),
required=['variable_names'],
additional_properties=False,
factory=cls)
