def process_bulk_operation(api_client: RemoteAPIClient, filename: str, is_removal: bool): """ Apply bulk operation helper :param api_client: base API client to extract connection options from :param filename: path to file with resources :param is_removal: is it removal operation """ odahuflow_resources = parse_resources_file(filename) result = api_aggregated_apply(odahuflow_resources, api_client, is_removal) output = ['Operation completed'] if result.created: output.append( f'created resources: {_print_resources_info_counter(result.created)}' ) if result.changed: output.append( f'changed resources: {_print_resources_info_counter(result.changed)}' ) if result.removed: output.append( f'removed resources: {_print_resources_info_counter(result.removed)}' ) click.echo(', '.join(output)) if result.errors: click.echo('Some errors detected:') for error in result.errors: click.echo(f'\t{error}') sys.exit(1)
def run(client: ModelPackagingClient, pack_id: str, manifest_file: List[str], manifest_dir: List[str], artifact_path: str, artifact_name: str, is_target_disabled: bool): """ \b Start a packaging process locally. \b Usage example: * odahuflowctl local pack run --id wine \f """ entities: List[OdahuflowCloudResourceUpdatePair] = [] for file_path in manifest_file: entities.extend(parse_resources_file(file_path).changes) for dir_path in manifest_dir: entities.extend(parse_resources_dir(dir_path)) mp: Optional[ModelPackaging] = None packagers: Dict[str, PackagingIntegration] = {} for entity in map(lambda x: x.resource, entities): if isinstance(entity, PackagingIntegration): packagers[entity.id] = entity elif isinstance(entity, ModelPackaging) and entity.id == pack_id: mp = entity if not mp: click.echo( f'The {pack_id} packaging not found in the manifest files.' f' Trying to retrieve it from API server' ) mp = client.get(pack_id) integration_name = mp.spec.integration_name packager = packagers.get(integration_name) if not packager: click.echo( f'The {integration_name} packager not found in the manifest files.' f' Trying to retrieve it from API server' ) packager = PackagingIntegrationClient.construct_from_other(client).get(integration_name) if artifact_name: mp.spec.artifact_name = artifact_name LOGGER.debug('Override the artifact name') if is_target_disabled: mp.spec.targets = [] k8s_packager = K8sPackager( model_packaging=mp, packaging_integration=packager, targets=[], ) result = start_package(k8s_packager, artifact_path) click.echo('Packager results:') for key, value in result.items(): click.echo(f'* {key} - {value}')
def run(client: ModelTrainingClient, train_id: str, manifest_file: List[str], manifest_dir: List[str], output_dir: str): """ \b Start a training process locally. \b Usage example: * odahuflowctl local train run --id examples-git \f """ entities: List[OdahuflowCloudResourceUpdatePair] = [] for file_path in manifest_file: entities.extend(parse_resources_file(file_path).changes) for dir_path in manifest_dir: entities.extend(parse_resources_dir(dir_path)) mt: Optional[ModelTraining] = None # find a training toolchains: Dict[str, ToolchainIntegration] = {} for entity in map(lambda x: x.resource, entities): if isinstance(entity, ToolchainIntegration): toolchains[entity.id] = entity elif isinstance(entity, ModelTraining) and entity.id == train_id: mt = entity if not mt: click.echo( f'{train_id} training not found. Trying to retrieve it from API server' ) mt = client.get(train_id) toolchain = toolchains.get(mt.spec.toolchain) if not toolchain: click.echo( f'{toolchain} toolchain not found. Trying to retrieve it from API server' ) toolchain = ToolchainIntegrationClient.construct_from_other( client).get(mt.spec.toolchain) trainer = K8sTrainer( model_training=mt, toolchain_integration=toolchain, ) start_train(trainer, output_dir)
def test_parse_resources_yaml_file(conn_manifest_yaml_file: str): result_conn = parse_resources_file(conn_manifest_yaml_file) assert len(result_conn.changes) == 1 assert result_conn.changes[0].resource_id == CONNECTION_RES.id assert result_conn.changes[0].resource == CONNECTION_RES
def test_parse_not_existed_resources_file_with_one_item(): with pytest.raises(FileNotFoundError, match=fr".*'{NOT_EXISTED_FILE}' not found.*"): parse_resources_file(NOT_EXISTED_FILE)