def get_fn(exec_properties: Dict[Text, Any],
fn_name: Text) -> Callable[..., Any]:
"""Loads and returns user-defined function."""
logging.error('udf_utils.get_fn %r %r', exec_properties, fn_name)
has_module_file = bool(exec_properties.get(_MODULE_FILE_KEY))
has_module_path = bool(exec_properties.get(_MODULE_PATH_KEY))
has_fn = bool(exec_properties.get(fn_name))
if has_module_path:
module_path = exec_properties[_MODULE_PATH_KEY]
return import_utils.import_func_from_module(module_path, fn_name)
elif has_module_file:
if has_fn:
return import_utils.import_func_from_source(
exec_properties[_MODULE_FILE_KEY], exec_properties[fn_name])
else:
return import_utils.import_func_from_source(
exec_properties[_MODULE_FILE_KEY], fn_name)
elif has_fn:
fn_path_split = exec_properties[fn_name].split('.')
return import_utils.import_func_from_module(
'.'.join(fn_path_split[0:-1]), fn_path_split[-1])
else:
raise ValueError(
'Neither module file or user function have been supplied in `exec_properties`.'
)
def _GetPreprocessingFn(self, inputs: Mapping[Text, Any],
unused_outputs: Mapping[Text, Any]) -> Any:
"""Returns a user defined preprocessing_fn.
Args:
inputs: A dictionary of labelled input values.
unused_outputs: A dictionary of labelled output values.
Returns:
User defined function.
Raises:
ValueError: When neither or both of MODULE_FILE and PREPROCESSING_FN
are present in inputs.
"""
has_module_file = bool(
common.GetSoleValue(inputs, labels.MODULE_FILE, strict=False))
has_preprocessing_fn = bool(
common.GetSoleValue(inputs, labels.PREPROCESSING_FN, strict=False))
if has_module_file == has_preprocessing_fn:
raise ValueError(
'Neither or both of MODULE_FILE and PREPROCESSING_FN have been '
'supplied in inputs.')
if has_module_file:
return import_utils.import_func_from_source(
common.GetSoleValue(inputs, labels.MODULE_FILE), 'preprocessing_fn')
preprocessing_fn_path_split = common.GetSoleValue(
inputs, labels.PREPROCESSING_FN).split('.')
return import_utils.import_func_from_module(
'.'.join(preprocessing_fn_path_split[0:-1]),
preprocessing_fn_path_split[-1])
def _GetTrainerFn(self, exec_properties: Dict[Text, Any]) -> Any:
"""Loads and returns user-defined trainer_fn."""
has_module_file = bool(exec_properties.get('module_file'))
has_trainer_fn = bool(exec_properties.get('trainer_fn'))
if has_module_file == has_trainer_fn:
raise ValueError(
"Neither or both of 'module_file' 'trainer_fn' have been supplied in "
"'exec_properties'.")
if has_module_file:
return import_utils.import_func_from_source(
exec_properties['module_file'], 'trainer_fn')
trainer_fn_path_split = exec_properties['trainer_fn'].split('.')
return import_utils.import_func_from_module(
'.'.join(trainer_fn_path_split[0:-1]), trainer_fn_path_split[-1])
def _GetFn(self, exec_properties: Dict[Text, Any], fn_name: Text) -> Any:
"""Loads and returns user-defined function."""
has_module_file = bool(exec_properties.get('module_file'))
has_fn = bool(exec_properties.get(fn_name))
if has_module_file == has_fn:
raise ValueError(
'Neither or both of module file and user function have been supplied in '
"'exec_properties'.")
if has_module_file:
return import_utils.import_func_from_source(
exec_properties['module_file'], fn_name)
fn_path_split = exec_properties[fn_name].split('.')
return import_utils.import_func_from_module('.'.join(fn_path_split[0:-1]),
fn_path_split[-1])
def get_fn(exec_properties: Dict[Text, Any],
fn_name: Text) -> Callable[..., Any]:
"""Loads and returns user-defined function."""
has_module_file = bool(exec_properties.get(_MODULE_FILE_KEY))
has_fn = bool(exec_properties.get(fn_name))
if has_module_file == has_fn:
raise ValueError(
'Neither or both of module file and user function have been supplied '
"in 'exec_properties'.")
if has_module_file:
return import_utils.import_func_from_source(
exec_properties[_MODULE_FILE_KEY], fn_name)
fn_path_split = exec_properties[fn_name].split('.')
return import_utils.import_func_from_module('.'.join(fn_path_split[0:-1]),
fn_path_split[-1])
def _get_test_pipeline_definition(self, module) -> pipeline.Pipeline:
"""Gets the pipeline definition from module."""
return import_utils.import_func_from_module(module.__name__,
"create_test_pipeline")()
def testImportFuncFromModuleModuleMissingFunction(self):
with self.assertRaises(AttributeError):
_ = import_utils.import_func_from_module(
test_fn.test_fn.__module__, 'non_existing_fn')
def testImportFuncFromModuleUnknownModule(self):
with self.assertRaises(ImportError):
_ = import_utils.import_func_from_module('non_existing_module',
'test_fn')
def testImportFuncFromModule(self):
imported_fn = import_utils.import_func_from_module(
test_fn.test_fn.__module__, test_fn.test_fn.__name__)
self.assertEqual(10, imported_fn([1, 2, 3, 4]))
def try_get_fn(module_path: Text,
fn_name: Text) -> Optional[Callable[..., Any]]:
try:
return import_utils.import_func_from_module(module_path, fn_name)
except (ValueError, AttributeError):
return None
