def register_custom_parameter(value_type, parameter): value_type = get_value_type_of_type(value_type, inline_value_type=True) _PARAMS_MAPPER.register_custom_parameter(value_type, parameter) get_types_registry().register_value_type(value_type) return parameter
def test_dict_nested_parse_df(self): r = get_types_registry() actual = r.get_value_type_of_type_str( "Dict[str, Dict[ str, pd.DataFrame]]") assert isinstance(actual, DictValueType) assert isinstance(actual.sub_value_type, DictValueType) assert isinstance(actual.sub_value_type.sub_value_type, DataFrameValueType)
def unknown_value_type_in_parameter(type_): from targets.values import get_types_registry return DatabandBuildError( "The parameter type '{}' is unknown, ".format(type_), help_msg= "Use one of: {}, or register the new one via 'register_custom_parameter'" .format(get_types_registry().list_known_types()), )
def guess_func_return_type(f_spec): return_key = "return" r = get_types_registry() return_func_spec = f_spec.annotations.get(return_key, NOTHING) if is_defined(return_func_spec): if return_func_spec is None: return None # for -> (int,DataFrame) if isinstance(return_func_spec, tuple): return_func_spec = [ ("result_%s" % idx, ret_part_type) for idx, ret_part_type in enumerate(return_func_spec, start=1) ] # easy way to check that it's NamedTuple elif hasattr(return_func_spec, "_field_types"): return_func_spec = list( six.iteritems(return_func_spec._field_types)) # case of named tuple elif is_Tuple(return_func_spec): # for -> Tuple[int,DataFrame] return_func_spec = get_Tuple_params(return_func_spec) return_func_spec = [ ("result_%s" % idx, ret_part_type) for idx, ret_part_type in enumerate(return_func_spec, start=1) ] if isinstance(return_func_spec, list): result = [] for field_name, ret_part_type in return_func_spec: field_value_type = r.get_value_type_of_type( ret_part_type, inline_value_type=True) result.append((field_name, field_value_type)) return result else: # fallback to regular parsing return r.get_value_type_of_type(return_func_spec, inline_value_type=True) doc_annotation = f_spec.doc_annotations.get(return_key, NOTHING) if is_defined(doc_annotation): if doc_annotation == "None": return None # if it "return" , it parsed into list # doc_annotation if isinstance(doc_annotation, six.string_types): return r.get_value_type_of_type_str(doc_annotation) or NOTHING # so we have multiple params result = [] for idx, ret_part_type in enumerate(doc_annotation, start=1): field_value_type = r.get_value_type_of_type_str(ret_part_type) result.append(("result_%s" % idx, field_value_type)) return result return NOTHING
def no_value_type_defined_in_parameter(context): from targets.values import get_types_registry return DatabandBuildError( "The parameter {context} doesn't have type! Please use parameter[YOUR_TYPE] or parameter[object]" .format(context=context), help_msg= "Use one of: {}, or register the new type via 'register_custom_parameter'" .format(get_types_registry().list_known_types()), )
def _update_parameter_from_runtime_value_type(parameter, value): # type: ( ParameterDefinition, Any)->Optional[ValueType] original_value_type = parameter.value_type runtime_value_type = None if isinstance(value, Target): if isinstance(value, InlineTarget): runtime_value_type = value.value_type elif isinstance(original_value_type, _TargetValueType): # user expects to get target/path/str # we will validate that value is good at "parse" pass else: # we are going to "load" the value from target into original_value_type # let get the "real" value type from the source of it if value.source and value.source_parameter: # we can take value type from the creator of it runtime_value_type = value.source_parameter.value_type if isinstance( runtime_value_type, (_TargetValueType, DefaultObjectValueType) ): return None else: # we don't really have value type return None else: if isinstance(value, six.string_types) or isinstance(value, Path): # str are going to be parsed, or treated as Target # Path is going to be used as Target return None runtime_value_type = get_types_registry().get_value_type_of_obj(value) # not found or the same if not runtime_value_type or runtime_value_type == original_value_type: return None # value is str, most chances we will parse it into the value if type(runtime_value_type) in [StrValueType, VersionValueType]: return None if isinstance(runtime_value_type, _StructureValueType): if isinstance(original_value_type, _StructureValueType): if type(runtime_value_type) == type(original_value_type): # probably we have difference on sub level, # there is no a clear way to find sub types of object if not runtime_value_type.sub_value_type: return None if original_value_type.type is object: if runtime_value_type.type is object: return None if not isinstance(original_value_type, DefaultObjectValueType): return None return runtime_value_type
def no_value_type_from_default(default_value, context): from targets.values import get_types_registry return DatabandBuildError( "The parameter '{parameter}' has default value '{default_value}' " "that cannot be resolved into known type!" "Please use '{parameter} = parameter[YOUR_TYPE]' or '{parameter} = parameter[object]'" .format(default_value=default_value, parameter=context), help_msg= "Use one of: {}, or register the new type via 'register_custom_parameter'" .format(get_types_registry().list_known_types()), )
def guess_func_arg_value_type(f_spec, name, default_value): # type: (_TaskDecoratorSpec, str, Any) -> ValueType r = get_types_registry() annotation = f_spec.annotations.get(name) if annotation is not None: return r.get_value_type_of_type(annotation) doc_annotation = f_spec.doc_annotations.get(name) t = r.get_value_type_of_type_str(doc_annotation) if t: return t if is_defined(default_value): return r.get_value_type_of_type(type(default_value)) return None
def infer_parameter_value_type(parameter, value): warnings = [] runtime_value_type = get_types_registry().get_value_type_of_obj(value) if not runtime_value_type: runtime_value_type = DefaultObjectValueType() if runtime_value_type != parameter.value_type: if not isinstance(parameter.value_type, DefaultObjectValueType): message = ("{parameter}: type of the value at runtime '{runtime}'" " doesn't match user defined type '{compile}'".format( parameter=parameter, runtime=runtime_value_type, compile=parameter.value_type, )) warnings.append(message) parameter = attr.evolve( parameter, value_type=runtime_value_type, value_type_defined=parameter.value_type, load_on_build=runtime_value_type.load_on_build, ) return parameter, warnings
def test_dict_nested_parse_int(self): r = get_types_registry() actual = r.get_value_type_of_type_str("Dict[str, Dict[ str,int]]") assert isinstance(actual, DictValueType) assert isinstance(actual.sub_value_type, DictValueType) assert isinstance(actual.sub_value_type.sub_value_type, IntValueType)
def test_List_df_parse(self): r = get_types_registry() actual = r.get_value_type_of_type_str("List[pd.DataFrame]") assert isinstance(actual, _StructureValueType) assert isinstance(actual.sub_value_type, DataFrameValueType)
def test_List_parse(self): r = get_types_registry() actual = r.get_value_type_of_type_str("List") assert isinstance(actual, ListValueType) assert actual.sub_value_type is None
def test_dict_Path_parse(self): r = get_types_registry() actual = r.get_value_type_of_type_str("Dict[str,Path]") assert isinstance(actual, DictValueType) assert isinstance(actual.sub_value_type, TargetPathLibValueType)
def test_dict_simple_parse(self): r = get_types_registry() actual = r.get_value_type_of_type_str("Dict[str,str]") assert isinstance(actual, DictValueType) assert isinstance(actual.sub_value_type, StrValueType)
def _update_parameter_from_runtime_value_type(parameter, cf_value): # type: ( ParameterDefinition, ConfigValue)->ParameterDefinition value = cf_value.value if value is None or parameter.is_output(): return parameter compiled_value_type = parameter.value_type runtime_value_type = None if isinstance(value, Target): if isinstance(value, InlineTarget): runtime_value_type = value.value_type elif isinstance(compiled_value_type, _TargetValueType): # user expects to get target/path/str # we will validate that value is good at "parse" pass else: # we are going to "load" the value from target into compiled_value_type # let get the "real" value type from the source of it if value.source and value.source_parameter: # we can take value type from the creator of it runtime_value_type = value.source_parameter.value_type if isinstance( runtime_value_type, (_TargetValueType, DefaultObjectValueType) ): return parameter else: # we don't really have value type return parameter else: if isinstance(value, six.string_types) or isinstance(value, Path): # str are going to be parsed, or treated as Target # Path is going to be used as Target return parameter runtime_value_type = get_types_registry().get_value_type_of_obj(value) # not found or the same if not runtime_value_type or runtime_value_type == compiled_value_type: return parameter # value is str, most chances we will parse it into the value if type(runtime_value_type) in [StrValueType, VersionValueType]: return parameter if isinstance(runtime_value_type, _StructureValueType): if isinstance(compiled_value_type, _StructureValueType): if type(runtime_value_type) == type(compiled_value_type): # probably we have difference on sub level, # there is no a clear way to find sub types of object if not runtime_value_type.sub_value_type: return parameter if compiled_value_type.type is object: return parameter message = ( "{parameter}: type of the value at runtime '{runtime}" " doesn't match user defined type '{compile}'".format( parameter=parameter, runtime=runtime_value_type, compile=compiled_value_type ) ) # we update, only if user didn't specify the value type # in the task definition if isinstance(parameter.value_type, DefaultObjectValueType): message = "%s: updating parameter with the runtime info" % (message) cf_value.warnings.append(message) logger.info(message) parameter = attr.evolve(parameter, value_type=runtime_value_type) else: cf_value.warnings.append(message) logger.warning(message) return parameter