def test_add_process_with_steps(mocker):
mocker.patch('kubernetes.config.load_kube_config')
with patch('builtins.open', mock_open(read_data=workflow_template.format('step-template', workflow_steps))):
main_workflow = ArgoWorkflow.from_yaml('workflow_template')
with patch('builtins.open', mock_open(read_data=process_template.format(''))):
process_workflow = ArgoWorkflow.from_yaml('process_template')
main_workflow.add_process(process_workflow)
spec = main_workflow.body['spec']
assert spec['entrypoint'] == 'step-template'
list_of_templates = spec['templates']
process_template_exists = False
flow_template_exists = False
step_template_exists = False
for template in list_of_templates:
if template['name'] == 'step-template':
step_template_exists = True
assert template.get('steps')
assert len(template.get('steps')) == 2
swt = None
pwt = None
for step in template.get('steps'):
step_name = step[0]['name']
if step_name == 'step1':
swt = step
elif step_name == 'process-template':
pwt = step
parameters = step[0].get('arguments', []).get('parameters', [])
assert parameters
check_parameters(parameters)
assert swt
assert pwt
elif template['name'] == 'workflow-template':
flow_template_exists = True
elif template['name'] == 'process-template':
process_template_exists = True
parameters = template.get('inputs', []).get('parameters')
assert parameters
check_parameters(parameters)
assert process_template_exists
assert flow_template_exists
assert step_template_exists
def export(path: str, format: str, operation_options: Tuple[str, ...]):
additional_params_str = " ".join(operation_options)
format = format.lower()
workflow_exports_files = os.listdir(
f'{Config().config_path}/workflows/exports')
formats = [
file.rstrip('.yaml') for file in workflow_exports_files
if file.endswith('.yaml')
]
if format not in formats:
click.echo(f'Format: {format} does not exist. Choose from: {formats}')
sys.exit(2)
try:
current_namespace = get_kubectl_current_context_namespace()
export_workflow = ArgoWorkflow.from_yaml(
f'{Config().config_path}/workflows/exports/{format}.yaml')
export_workflow.parameters = {
'cluster-registry-address': NAUTAConfigMap().registry,
'saved-model-dir-path': path,
'additional-params': additional_params_str
}
export_workflow.create(namespace=current_namespace)
except Exception:
error_msg = 'Failed to create export workflow.'
click.echo(error_msg)
logger.exception(error_msg)
sys.exit(1)
click.echo(f'Successfully created export workflow: {export_workflow.name}')
def export(path: str, format: str, operation_options: Tuple[str, ...]):
if path == FORMATS_OPTION:
try:
list_of_workflows = get_list_of_workflows(EXPORT_WORKFLOWS_LOCATION)
except Exception:
handle_error(logger, Texts.EXPORT_LIST_ERROR_MSG, Texts.EXPORT_LIST_ERROR_MSG)
sys.exit(1)
click.echo(tabulate(list_of_workflows, headers=EXPORT_LIST_HEADERS,
tablefmt=TBLT_TABLE_FORMAT))
sys.exit(0)
config_path = Config().config_path
formats: List[str] = [] # noqa: E701
if os.path.isdir(config_path):
workflow_exports_files = os.listdir(f'{config_path}/workflows/exports')
formats = [os.path.splitext(file)[0] for file in workflow_exports_files if file.endswith('.yaml')]
if not format:
click.echo(Texts.MISSING_EXPORT_FORMAT.format(formats=formats))
sys.exit(2)
format = format.lower()
if format not in formats:
click.echo(Texts.WRONG_EXPORT_FORMAT.format(format=format, formats=formats))
sys.exit(2)
additional_params_str = " ".join(operation_options)
try:
current_namespace = get_kubectl_current_context_namespace()
export_workflow = ArgoWorkflow.from_yaml(f'{Config().config_path}/workflows/exports/{format}.yaml')
export_workflow.parameters = {
'cluster-registry-address': NAUTAConfigMap().registry,
'saved-model-dir-path': path,
'additional-params': additional_params_str
}
export_workflow.create(namespace=current_namespace)
workflow: ArgoWorkflow = ArgoWorkflow.get(namespace=current_namespace, name=export_workflow.name)
except Exception:
error_msg = 'Failed to create export workflow.'
click.echo(error_msg)
logger.exception(error_msg)
sys.exit(1)
click.echo(tabulate([workflow.cli_representation], headers=MODEL_HEADERS, tablefmt=TBLT_TABLE_FORMAT))
click.echo(f'\nSuccessfully created export workflow')
def submit(state: State, workflow_path: str):
try:
workflow: ArgoWorkflow = ArgoWorkflow.from_yaml(workflow_path)
namespace = get_kubectl_current_context_namespace()
with spinner(text=Texts.PROGRESS_MSG):
workflow.create(namespace=namespace)
workflow.namespace = namespace # Set namespace, to properly display owner in CLI
click.echo(
tabulate([workflow.cli_representation],
headers=HEADERS,
tablefmt=TBLT_TABLE_FORMAT))
except IOError as e:
handle_error(logger, Texts.LOAD_SPEC_ERROR_MSG.format(msg=str(e)),
Texts.LOAD_SPEC_ERROR_MSG.format(msg=str(e)))
exit(1)
except Exception:
handle_error(logger,
Texts.OTHER_ERROR_MSG,
Texts.OTHER_ERROR_MSG,
add_verbosity_msg=True)
exit(1)
def process(path: str, kind: str, options: Tuple[str, ...]):
additional_params_str = " ".join(options)
kind = kind.lower()
config_path = Config().config_path
process_path = f'{config_path}/workflows/processes'
kinds: List[str] = []
if os.path.isdir(process_path):
process_kinds = os.listdir(f'{config_path}/workflows/processes')
kinds = [
os.path.splitext(file)[0] for file in process_kinds
if file.endswith('.yaml')
]
if kind not in kinds:
click.echo(Texts.WRONG_PROCESS_KIND.format(process=kind, kinds=kinds))
sys.exit(2)
try:
current_namespace = get_kubectl_current_context_namespace()
process_workflow = ArgoWorkflow.from_yaml(
f'{Config().config_path}/workflows/processes/{kind}.yaml')
process_workflow.parameters = {
'cluster-registry-address': NAUTAConfigMap().registry,
'saved-model-dir-path': path,
'additional-params': additional_params_str
}
process_workflow.create(namespace=current_namespace)
except Exception:
error_msg = 'Failed to create export workflow.'
click.echo(error_msg)
logger.exception(error_msg)
sys.exit(1)
click.echo(
f'Successfully created process workflow: {process_workflow.name}')