def register_custom_parameter(value_type, parameter):
value_type = get_value_type_of_type(value_type, inline_value_type=True)
_PARAMS_MAPPER.register_custom_parameter(value_type, parameter)
get_types_registry().register_value_type(value_type)
return parameter
def test_dict_nested_parse_df(self):
r = get_types_registry()
actual = r.get_value_type_of_type_str(
"Dict[str, Dict[ str, pd.DataFrame]]")
assert isinstance(actual, DictValueType)
assert isinstance(actual.sub_value_type, DictValueType)
assert isinstance(actual.sub_value_type.sub_value_type,
DataFrameValueType)
def unknown_value_type_in_parameter(type_):
from targets.values import get_types_registry
return DatabandBuildError(
"The parameter type '{}' is unknown, ".format(type_),
help_msg=
"Use one of: {}, or register the new one via 'register_custom_parameter'"
.format(get_types_registry().list_known_types()),
)
def guess_func_return_type(f_spec):
return_key = "return"
r = get_types_registry()
return_func_spec = f_spec.annotations.get(return_key, NOTHING)
if is_defined(return_func_spec):
if return_func_spec is None:
return None
# for -> (int,DataFrame)
if isinstance(return_func_spec, tuple):
return_func_spec = [
("result_%s" % idx, ret_part_type)
for idx, ret_part_type in enumerate(return_func_spec, start=1)
]
# easy way to check that it's NamedTuple
elif hasattr(return_func_spec, "_field_types"):
return_func_spec = list(
six.iteritems(return_func_spec._field_types))
# case of named tuple
elif is_Tuple(return_func_spec):
# for -> Tuple[int,DataFrame]
return_func_spec = get_Tuple_params(return_func_spec)
return_func_spec = [
("result_%s" % idx, ret_part_type)
for idx, ret_part_type in enumerate(return_func_spec, start=1)
]
if isinstance(return_func_spec, list):
result = []
for field_name, ret_part_type in return_func_spec:
field_value_type = r.get_value_type_of_type(
ret_part_type, inline_value_type=True)
result.append((field_name, field_value_type))
return result
else:
# fallback to regular parsing
return r.get_value_type_of_type(return_func_spec,
inline_value_type=True)
doc_annotation = f_spec.doc_annotations.get(return_key, NOTHING)
if is_defined(doc_annotation):
if doc_annotation == "None":
return None
# if it "return" , it parsed into list
# doc_annotation
if isinstance(doc_annotation, six.string_types):
return r.get_value_type_of_type_str(doc_annotation) or NOTHING
# so we have multiple params
result = []
for idx, ret_part_type in enumerate(doc_annotation, start=1):
field_value_type = r.get_value_type_of_type_str(ret_part_type)
result.append(("result_%s" % idx, field_value_type))
return result
return NOTHING
def no_value_type_defined_in_parameter(context):
from targets.values import get_types_registry
return DatabandBuildError(
"The parameter {context} doesn't have type! Please use parameter[YOUR_TYPE] or parameter[object]"
.format(context=context),
help_msg=
"Use one of: {}, or register the new type via 'register_custom_parameter'"
.format(get_types_registry().list_known_types()),
)
def _update_parameter_from_runtime_value_type(parameter, value):
# type: ( ParameterDefinition, Any)->Optional[ValueType]
original_value_type = parameter.value_type
runtime_value_type = None
if isinstance(value, Target):
if isinstance(value, InlineTarget):
runtime_value_type = value.value_type
elif isinstance(original_value_type, _TargetValueType):
# user expects to get target/path/str
# we will validate that value is good at "parse"
pass
else:
# we are going to "load" the value from target into original_value_type
# let get the "real" value type from the source of it
if value.source and value.source_parameter:
# we can take value type from the creator of it
runtime_value_type = value.source_parameter.value_type
if isinstance(
runtime_value_type, (_TargetValueType, DefaultObjectValueType)
):
return None
else:
# we don't really have value type
return None
else:
if isinstance(value, six.string_types) or isinstance(value, Path):
# str are going to be parsed, or treated as Target
# Path is going to be used as Target
return None
runtime_value_type = get_types_registry().get_value_type_of_obj(value)
# not found or the same
if not runtime_value_type or runtime_value_type == original_value_type:
return None
# value is str, most chances we will parse it into the value
if type(runtime_value_type) in [StrValueType, VersionValueType]:
return None
if isinstance(runtime_value_type, _StructureValueType):
if isinstance(original_value_type, _StructureValueType):
if type(runtime_value_type) == type(original_value_type):
# probably we have difference on sub level,
# there is no a clear way to find sub types of object
if not runtime_value_type.sub_value_type:
return None
if original_value_type.type is object:
if runtime_value_type.type is object:
return None
if not isinstance(original_value_type, DefaultObjectValueType):
return None
return runtime_value_type
def no_value_type_from_default(default_value, context):
from targets.values import get_types_registry
return DatabandBuildError(
"The parameter '{parameter}' has default value '{default_value}' "
"that cannot be resolved into known type!"
"Please use '{parameter} = parameter[YOUR_TYPE]' or '{parameter} = parameter[object]'"
.format(default_value=default_value, parameter=context),
help_msg=
"Use one of: {}, or register the new type via 'register_custom_parameter'"
.format(get_types_registry().list_known_types()),
)
def guess_func_arg_value_type(f_spec, name, default_value):
# type: (_TaskDecoratorSpec, str, Any) -> ValueType
r = get_types_registry()
annotation = f_spec.annotations.get(name)
if annotation is not None:
return r.get_value_type_of_type(annotation)
doc_annotation = f_spec.doc_annotations.get(name)
t = r.get_value_type_of_type_str(doc_annotation)
if t:
return t
if is_defined(default_value):
return r.get_value_type_of_type(type(default_value))
return None
def infer_parameter_value_type(parameter, value):
warnings = []
runtime_value_type = get_types_registry().get_value_type_of_obj(value)
if not runtime_value_type:
runtime_value_type = DefaultObjectValueType()
if runtime_value_type != parameter.value_type:
if not isinstance(parameter.value_type, DefaultObjectValueType):
message = ("{parameter}: type of the value at runtime '{runtime}'"
" doesn't match user defined type '{compile}'".format(
parameter=parameter,
runtime=runtime_value_type,
compile=parameter.value_type,
))
warnings.append(message)
parameter = attr.evolve(
parameter,
value_type=runtime_value_type,
value_type_defined=parameter.value_type,
load_on_build=runtime_value_type.load_on_build,
)
return parameter, warnings
def test_dict_nested_parse_int(self):
r = get_types_registry()
actual = r.get_value_type_of_type_str("Dict[str, Dict[ str,int]]")
assert isinstance(actual, DictValueType)
assert isinstance(actual.sub_value_type, DictValueType)
assert isinstance(actual.sub_value_type.sub_value_type, IntValueType)
def test_List_df_parse(self):
r = get_types_registry()
actual = r.get_value_type_of_type_str("List[pd.DataFrame]")
assert isinstance(actual, _StructureValueType)
assert isinstance(actual.sub_value_type, DataFrameValueType)
def test_List_parse(self):
r = get_types_registry()
actual = r.get_value_type_of_type_str("List")
assert isinstance(actual, ListValueType)
assert actual.sub_value_type is None
def test_dict_Path_parse(self):
r = get_types_registry()
actual = r.get_value_type_of_type_str("Dict[str,Path]")
assert isinstance(actual, DictValueType)
assert isinstance(actual.sub_value_type, TargetPathLibValueType)
def test_dict_simple_parse(self):
r = get_types_registry()
actual = r.get_value_type_of_type_str("Dict[str,str]")
assert isinstance(actual, DictValueType)
assert isinstance(actual.sub_value_type, StrValueType)
def _update_parameter_from_runtime_value_type(parameter, cf_value):
# type: ( ParameterDefinition, ConfigValue)->ParameterDefinition
value = cf_value.value
if value is None or parameter.is_output():
return parameter
compiled_value_type = parameter.value_type
runtime_value_type = None
if isinstance(value, Target):
if isinstance(value, InlineTarget):
runtime_value_type = value.value_type
elif isinstance(compiled_value_type, _TargetValueType):
# user expects to get target/path/str
# we will validate that value is good at "parse"
pass
else:
# we are going to "load" the value from target into compiled_value_type
# let get the "real" value type from the source of it
if value.source and value.source_parameter:
# we can take value type from the creator of it
runtime_value_type = value.source_parameter.value_type
if isinstance(
runtime_value_type, (_TargetValueType, DefaultObjectValueType)
):
return parameter
else:
# we don't really have value type
return parameter
else:
if isinstance(value, six.string_types) or isinstance(value, Path):
# str are going to be parsed, or treated as Target
# Path is going to be used as Target
return parameter
runtime_value_type = get_types_registry().get_value_type_of_obj(value)
# not found or the same
if not runtime_value_type or runtime_value_type == compiled_value_type:
return parameter
# value is str, most chances we will parse it into the value
if type(runtime_value_type) in [StrValueType, VersionValueType]:
return parameter
if isinstance(runtime_value_type, _StructureValueType):
if isinstance(compiled_value_type, _StructureValueType):
if type(runtime_value_type) == type(compiled_value_type):
# probably we have difference on sub level,
# there is no a clear way to find sub types of object
if not runtime_value_type.sub_value_type:
return parameter
if compiled_value_type.type is object:
return parameter
message = (
"{parameter}: type of the value at runtime '{runtime}"
" doesn't match user defined type '{compile}'".format(
parameter=parameter, runtime=runtime_value_type, compile=compiled_value_type
)
)
# we update, only if user didn't specify the value type
# in the task definition
if isinstance(parameter.value_type, DefaultObjectValueType):
message = "%s: updating parameter with the runtime info" % (message)
cf_value.warnings.append(message)
logger.info(message)
parameter = attr.evolve(parameter, value_type=runtime_value_type)
else:
cf_value.warnings.append(message)
logger.warning(message)
return parameter
