def test_python_primitive_type_names(self):
"""
Python primitive types
"""
self.assertEqual(object_to_qualified_name(bool), 'bool')
self.assertEqual(object_to_qualified_name(int), 'int')
self.assertEqual(object_to_qualified_name(float), 'float')
self.assertEqual(object_to_qualified_name(str), 'str')
def _get_pandas_variable_descriptor(self, variable: pd.Series):
return {
'name': variable.name,
'dataType': object_to_qualified_name(variable.dtype),
'ndim': variable.ndim,
'shape': variable.shape,
'dimensions': variable.dims,
}
def introspect_operation(cls, operation) -> 'OpMetaInfo':
if not operation:
raise ValueError("'operation' argument must be given")
op_qualified_name = object_to_qualified_name(operation, fail=True)
input_dict, has_monitor = OrderedDict(), False
if hasattr(operation, '__code__'):
input_dict, has_monitor = cls._introspect_inputs_from_callable(operation, False)
elif isclass(operation):
if hasattr(operation, '__call__'):
call_method = getattr(operation, '__call__')
input_dict, has_monitor = cls._introspect_inputs_from_callable(call_method, True)
else:
raise ValueError('operations of type class must define a __call__(self, ...) method')
return_name = OpMetaInfo.RETURN_OUTPUT_NAME
output_dict = OrderedDict()
if hasattr(operation, '__annotations__'):
# mapping of parameters names to annotations; 'return' key is reserved for return annotations.
annotations = operation.__annotations__
for annotated_name, annotated_type in annotations.items():
if annotated_name == 'return':
# op_meta_info.output can't be present so far -> assign new dict
output_dict[return_name] = dict(data_type=annotated_type)
elif annotated_name != cls.MONITOR_INPUT_NAME:
# input_dict[annotated_name] should be present through _introspect_inputs_from_callable() call
input_dict[annotated_name]['data_type'] = annotated_type
if len(output_dict) == 0:
output_dict[return_name] = dict()
header = dict()
# Introspect the operation instance (see https://docs.python.org/3.5/library/inspect.html)
if hasattr(operation, '__doc__'):
# documentation string
docstring = operation.__doc__
if docstring:
description, param_descriptions, return_description = cls._parse_docstring(docstring)
if description:
header['description'] = description
if param_descriptions:
for param_name, param_description in param_descriptions.items():
if param_name in input_dict:
input_dict[param_name]['description'] = param_descriptions[param_name]
if return_description and return_name in output_dict:
output_dict[return_name]['description'] = return_description
return OpMetaInfo(op_qualified_name,
header_dict=header,
has_monitor=has_monitor,
input_dict=input_dict,
output_dict=output_dict)
def get_op_key(self, operation: Union[str, Callable]):
"""
Get a key under which the given operation will be registered.
:param operation: A fully qualified operation name or a callable object
:return: The operation key
"""
if isinstance(operation, str):
qualified_name = operation
else:
qualified_name = object_to_qualified_name(operation)
if qualified_name.startswith('cate.ops.'):
return qualified_name.rsplit('.', maxsplit=1)[1]
else:
return qualified_name
def test_cate_cdm_type_names(self):
"""
Cate Common Data Model (CDM) types
"""
self.assertEqual(object_to_qualified_name(np.ndarray), 'numpy.ndarray')
self.assertEqual(object_to_qualified_name(xr.Dataset),
'xarray.core.dataset.Dataset')
self.assertEqual(object_to_qualified_name(xr.DataArray),
'xarray.core.dataarray.DataArray')
self.assertEqual(object_to_qualified_name(gpd.GeoDataFrame),
'geopandas.geodataframe.GeoDataFrame')
self.assertEqual(object_to_qualified_name(gpd.GeoSeries),
'geopandas.geoseries.GeoSeries')
self.assertEqual(object_to_qualified_name(pd.DataFrame),
'pandas.core.frame.DataFrame')
self.assertEqual(object_to_qualified_name(pd.Series),
'pandas.core.series.Series')
def _get_resource_descriptor(self, res_name: str, resource):
variable_descriptors = []
data_type_name = object_to_qualified_name(type(resource))
if isinstance(resource, xr.Dataset):
var_names = sorted(resource.data_vars.keys())
for var_name in var_names:
variable = resource.data_vars[var_name]
variable_descriptors.append(
self._get_xarray_variable_descriptor(variable))
return dict(name=res_name,
dataType=data_type_name,
dims=to_json(resource.dims),
attrs=self._get_dataset_attr_list(resource.attrs),
variables=variable_descriptors)
elif isinstance(resource, pd.DataFrame):
var_names = list(resource.columns)
for var_name in var_names:
variable = resource[var_name]
variable_descriptors.append(
self._get_pandas_variable_descriptor(variable))
return dict(name=res_name,
dataType=data_type_name,
variables=variable_descriptors)
elif isinstance(resource, fiona.Collection):
num_features = len(resource)
properties = resource.schema.get('properties')
if properties:
for var_name, var_type in properties.items():
variable_descriptors.append({
'name': var_name,
'dataType': var_type,
'isFeatureAttribute': True,
})
geometry = resource.schema.get('geometry')
return dict(name=res_name,
dataType=data_type_name,
variables=variable_descriptors,
geometry=geometry,
numFeatures=num_features)
return dict(name=res_name, dataType=data_type_name)
def test_cate_op_api_type_names(self):
"""
Additional Cate types used by operations API.
"""
self.assertEqual(object_to_qualified_name(VarName),
'cate.core.types.VarName')
self.assertEqual(object_to_qualified_name(VarNamesLike),
'cate.core.types.VarNamesLike')
self.assertEqual(object_to_qualified_name(PointLike),
'cate.core.types.PointLike')
self.assertEqual(object_to_qualified_name(PolygonLike),
'cate.core.types.PolygonLike')
self.assertEqual(object_to_qualified_name(GeometryLike),
'cate.core.types.GeometryLike')
self.assertEqual(object_to_qualified_name(TimeRangeLike),
'cate.core.types.TimeRangeLike')
def _get_xarray_variable_descriptor(self, variable: xr.DataArray):
attrs = variable.attrs
variable_info = {
'name': variable.name,
'dataType': object_to_qualified_name(variable.dtype),
'ndim': len(variable.dims),
'shape': variable.shape,
'chunks': get_chunk_size(variable),
'dimensions': variable.dims,
'fill_value': self._get_float_attr(attrs, '_FillValue'),
'valid_min': self._get_float_attr(attrs, 'valid_min'),
'valid_max': self._get_float_attr(attrs, 'valid_max'),
'add_offset': self._get_float_attr(attrs, 'add_offset'),
'scale_factor': self._get_float_attr(attrs, 'scale_factor'),
'standard_name': self._get_unicode_attr(attrs, 'standard_name'),
'long_name': self._get_unicode_attr(attrs, 'long_name'),
'units': self._get_unicode_attr(attrs, 'units', default_value='-'),
'comment': self._get_unicode_attr(attrs, 'comment'),
}
image_config = self._get_variable_image_config(variable)
if image_config:
variable_info['imageLayout'] = image_config
variable_info['y_flipped'] = Workspace._is_y_flipped(variable)
return variable_info
def test_registered_op(self):
registered_op = _OP_REGISTRY.get_op(
object_to_qualified_name(scale_point))
point = registered_op(point_like="2.4, 4.8", factor=0.5)
self.assertEqual(point, ExamplePoint(1.2, 2.4))