def get_fn(exec_properties: Dict[Text, Any], fn_name: Text) -> Callable[..., Any]: """Loads and returns user-defined function.""" logging.error('udf_utils.get_fn %r %r', exec_properties, fn_name) has_module_file = bool(exec_properties.get(_MODULE_FILE_KEY)) has_module_path = bool(exec_properties.get(_MODULE_PATH_KEY)) has_fn = bool(exec_properties.get(fn_name)) if has_module_path: module_path = exec_properties[_MODULE_PATH_KEY] return import_utils.import_func_from_module(module_path, fn_name) elif has_module_file: if has_fn: return import_utils.import_func_from_source( exec_properties[_MODULE_FILE_KEY], exec_properties[fn_name]) else: return import_utils.import_func_from_source( exec_properties[_MODULE_FILE_KEY], fn_name) elif has_fn: fn_path_split = exec_properties[fn_name].split('.') return import_utils.import_func_from_module( '.'.join(fn_path_split[0:-1]), fn_path_split[-1]) else: raise ValueError( 'Neither module file or user function have been supplied in `exec_properties`.' )
def _GetPreprocessingFn(self, inputs: Mapping[Text, Any], unused_outputs: Mapping[Text, Any]) -> Any: """Returns a user defined preprocessing_fn. Args: inputs: A dictionary of labelled input values. unused_outputs: A dictionary of labelled output values. Returns: User defined function. Raises: ValueError: When neither or both of MODULE_FILE and PREPROCESSING_FN are present in inputs. """ has_module_file = bool( common.GetSoleValue(inputs, labels.MODULE_FILE, strict=False)) has_preprocessing_fn = bool( common.GetSoleValue(inputs, labels.PREPROCESSING_FN, strict=False)) if has_module_file == has_preprocessing_fn: raise ValueError( 'Neither or both of MODULE_FILE and PREPROCESSING_FN have been ' 'supplied in inputs.') if has_module_file: return import_utils.import_func_from_source( common.GetSoleValue(inputs, labels.MODULE_FILE), 'preprocessing_fn') preprocessing_fn_path_split = common.GetSoleValue( inputs, labels.PREPROCESSING_FN).split('.') return import_utils.import_func_from_module( '.'.join(preprocessing_fn_path_split[0:-1]), preprocessing_fn_path_split[-1])
def _GetTrainerFn(self, exec_properties: Dict[Text, Any]) -> Any: """Loads and returns user-defined trainer_fn.""" has_module_file = bool(exec_properties.get('module_file')) has_trainer_fn = bool(exec_properties.get('trainer_fn')) if has_module_file == has_trainer_fn: raise ValueError( "Neither or both of 'module_file' 'trainer_fn' have been supplied in " "'exec_properties'.") if has_module_file: return import_utils.import_func_from_source( exec_properties['module_file'], 'trainer_fn') trainer_fn_path_split = exec_properties['trainer_fn'].split('.') return import_utils.import_func_from_module( '.'.join(trainer_fn_path_split[0:-1]), trainer_fn_path_split[-1])
def _GetFn(self, exec_properties: Dict[Text, Any], fn_name: Text) -> Any: """Loads and returns user-defined function.""" has_module_file = bool(exec_properties.get('module_file')) has_fn = bool(exec_properties.get(fn_name)) if has_module_file == has_fn: raise ValueError( 'Neither or both of module file and user function have been supplied in ' "'exec_properties'.") if has_module_file: return import_utils.import_func_from_source( exec_properties['module_file'], fn_name) fn_path_split = exec_properties[fn_name].split('.') return import_utils.import_func_from_module('.'.join(fn_path_split[0:-1]), fn_path_split[-1])
def get_fn(exec_properties: Dict[Text, Any], fn_name: Text) -> Callable[..., Any]: """Loads and returns user-defined function.""" has_module_file = bool(exec_properties.get(_MODULE_FILE_KEY)) has_fn = bool(exec_properties.get(fn_name)) if has_module_file == has_fn: raise ValueError( 'Neither or both of module file and user function have been supplied ' "in 'exec_properties'.") if has_module_file: return import_utils.import_func_from_source( exec_properties[_MODULE_FILE_KEY], fn_name) fn_path_split = exec_properties[fn_name].split('.') return import_utils.import_func_from_module('.'.join(fn_path_split[0:-1]), fn_path_split[-1])
def _get_test_pipeline_definition(self, module) -> pipeline.Pipeline: """Gets the pipeline definition from module.""" return import_utils.import_func_from_module(module.__name__, "create_test_pipeline")()
def testImportFuncFromModuleModuleMissingFunction(self): with self.assertRaises(AttributeError): _ = import_utils.import_func_from_module( test_fn.test_fn.__module__, 'non_existing_fn')
def testImportFuncFromModuleUnknownModule(self): with self.assertRaises(ImportError): _ = import_utils.import_func_from_module('non_existing_module', 'test_fn')
def testImportFuncFromModule(self): imported_fn = import_utils.import_func_from_module( test_fn.test_fn.__module__, test_fn.test_fn.__name__) self.assertEqual(10, imported_fn([1, 2, 3, 4]))
def try_get_fn(module_path: Text, fn_name: Text) -> Optional[Callable[..., Any]]: try: return import_utils.import_func_from_module(module_path, fn_name) except (ValueError, AttributeError): return None