def process_bulk_operation(api_client: RemoteAPIClient, filename: str,
is_removal: bool):
"""
Apply bulk operation helper
:param api_client: base API client to extract connection options from
:param filename: path to file with resources
:param is_removal: is it removal operation
"""
odahuflow_resources = parse_resources_file(filename)
result = api_aggregated_apply(odahuflow_resources, api_client, is_removal)
output = ['Operation completed']
if result.created:
output.append(
f'created resources: {_print_resources_info_counter(result.created)}'
)
if result.changed:
output.append(
f'changed resources: {_print_resources_info_counter(result.changed)}'
)
if result.removed:
output.append(
f'removed resources: {_print_resources_info_counter(result.removed)}'
)
click.echo(', '.join(output))
if result.errors:
click.echo('Some errors detected:')
for error in result.errors:
click.echo(f'\t{error}')
sys.exit(1)
def run(client: ModelPackagingClient, pack_id: str, manifest_file: List[str], manifest_dir: List[str],
artifact_path: str, artifact_name: str, is_target_disabled: bool):
"""
\b
Start a packaging process locally.
\b
Usage example:
* odahuflowctl local pack run --id wine
\f
"""
entities: List[OdahuflowCloudResourceUpdatePair] = []
for file_path in manifest_file:
entities.extend(parse_resources_file(file_path).changes)
for dir_path in manifest_dir:
entities.extend(parse_resources_dir(dir_path))
mp: Optional[ModelPackaging] = None
packagers: Dict[str, PackagingIntegration] = {}
for entity in map(lambda x: x.resource, entities):
if isinstance(entity, PackagingIntegration):
packagers[entity.id] = entity
elif isinstance(entity, ModelPackaging) and entity.id == pack_id:
mp = entity
if not mp:
click.echo(
f'The {pack_id} packaging not found in the manifest files.'
f' Trying to retrieve it from API server'
)
mp = client.get(pack_id)
integration_name = mp.spec.integration_name
packager = packagers.get(integration_name)
if not packager:
click.echo(
f'The {integration_name} packager not found in the manifest files.'
f' Trying to retrieve it from API server'
)
packager = PackagingIntegrationClient.construct_from_other(client).get(integration_name)
if artifact_name:
mp.spec.artifact_name = artifact_name
LOGGER.debug('Override the artifact name')
if is_target_disabled:
mp.spec.targets = []
k8s_packager = K8sPackager(
model_packaging=mp,
packaging_integration=packager,
targets=[],
)
result = start_package(k8s_packager, artifact_path)
click.echo('Packager results:')
for key, value in result.items():
click.echo(f'* {key} - {value}')
def run(client: ModelTrainingClient, train_id: str, manifest_file: List[str],
manifest_dir: List[str], output_dir: str):
"""
\b
Start a training process locally.
\b
Usage example:
* odahuflowctl local train run --id examples-git
\f
"""
entities: List[OdahuflowCloudResourceUpdatePair] = []
for file_path in manifest_file:
entities.extend(parse_resources_file(file_path).changes)
for dir_path in manifest_dir:
entities.extend(parse_resources_dir(dir_path))
mt: Optional[ModelTraining] = None
# find a training
toolchains: Dict[str, ToolchainIntegration] = {}
for entity in map(lambda x: x.resource, entities):
if isinstance(entity, ToolchainIntegration):
toolchains[entity.id] = entity
elif isinstance(entity, ModelTraining) and entity.id == train_id:
mt = entity
if not mt:
click.echo(
f'{train_id} training not found. Trying to retrieve it from API server'
)
mt = client.get(train_id)
toolchain = toolchains.get(mt.spec.toolchain)
if not toolchain:
click.echo(
f'{toolchain} toolchain not found. Trying to retrieve it from API server'
)
toolchain = ToolchainIntegrationClient.construct_from_other(
client).get(mt.spec.toolchain)
trainer = K8sTrainer(
model_training=mt,
toolchain_integration=toolchain,
)
start_train(trainer, output_dir)
def test_parse_resources_yaml_file(conn_manifest_yaml_file: str):
result_conn = parse_resources_file(conn_manifest_yaml_file)
assert len(result_conn.changes) == 1
assert result_conn.changes[0].resource_id == CONNECTION_RES.id
assert result_conn.changes[0].resource == CONNECTION_RES
def test_parse_not_existed_resources_file_with_one_item():
with pytest.raises(FileNotFoundError, match=fr".*'{NOT_EXISTED_FILE}' not found.*"):
parse_resources_file(NOT_EXISTED_FILE)