def executor_main():
parser = argparse.ArgumentParser(description='KFP Component Executor.')
parser.add_argument('--component_module_path',
type=str,
help='Path to a module containing the KFP component.')
parser.add_argument('--function_to_execute',
type=str,
help='The name of the component function in '
'--component_module_path file that is to be executed.')
parser.add_argument(
'--executor_input',
type=str,
help='JSON-serialized ExecutorInput from the orchestrator. '
'This should contain inputs and placeholders for outputs.')
args, _ = parser.parse_known_args()
module_directory = os.path.dirname(args.component_module_path)
module_name = os.path.basename(args.component_module_path)[:-len('.py')]
print('Loading KFP component module {} from dir {}'.format(
module_name, module_directory))
module = _load_module(module_name=module_name,
module_directory=module_directory)
executor_input = json.loads(args.executor_input)
function_to_execute = getattr(module, args.function_to_execute)
executor = component_executor.Executor(
executor_input=executor_input, function_to_execute=function_to_execute)
executor.execute()
def _get_executor(self,
func: Callable,
executor_input: Optional[str] = None) -> executor.Executor:
if executor_input is None:
executor_input = _EXECUTOR_INPUT
executor_input_dict = json.loads(executor_input % self._test_dir)
return executor.Executor(executor_input=executor_input_dict,
function_to_execute=func)
def _get_executor(self, func: Callable) -> executor.Executor:
return executor.Executor(executor_input=self._executor_input,
function_to_execute=func)
def executor_main():
_setup_logging()
parser = argparse.ArgumentParser(description='KFP Component Executor.')
parser.add_argument('--component_module_path',
type=str,
help='Path to a module containing the KFP component.')
parser.add_argument('--function_to_execute',
type=str,
required=True,
help='The name of the component function in '
'--component_module_path file that is to be executed.')
parser.add_argument(
'--executor_input',
type=str,
help='JSON-serialized ExecutorInput from the orchestrator. '
'This should contain inputs and placeholders for outputs.')
args, _ = parser.parse_known_args()
func_name = args.function_to_execute
module_path = None
module_directory = None
module_name = None
if args.component_module_path is not None:
logging.info(
'Looking for component `{}` in --component_module_path `{}`'.
format(func_name, args.component_module_path))
module_path = args.component_module_path
module_directory = os.path.dirname(args.component_module_path)
module_name = os.path.basename(
args.component_module_path)[:-len('.py')]
else:
# Look for module directory using kfp_config.ini
logging.info('--component_module_path is not specified. Looking for'
' component `{}` in config file `kfp_config.ini`'
' instead'.format(func_name))
config = kfp_config.KFPConfig()
components = config.get_components()
if not components:
raise RuntimeError('No components found in `kfp_config.ini`')
try:
module_path = components[func_name]
except KeyError:
raise RuntimeError(
'Could not find component `{}` in `kfp_config.ini`. Found the '
' following components instead:\n{}'.format(
func_name, components))
module_directory = str(module_path.parent)
module_name = str(module_path.name)[:-len('.py')]
logging.info(
'Loading KFP component "{}" from {} (directory "{}" and module name'
' "{}")'.format(func_name, module_path, module_directory, module_name))
module = utils.load_module(module_name=module_name,
module_directory=module_directory)
executor_input = json.loads(args.executor_input)
function_to_execute = getattr(module, func_name)
logging.info('Got executor_input:\n{}'.format(
json.dumps(executor_input, indent=4)))
executor = component_executor.Executor(
executor_input=executor_input, function_to_execute=function_to_execute)
executor.execute()