def manifest(ctx):
"""Generate the manifests."""
try:
ctx.tracker.track_event('command', 'dlis_manifest')
model = Model(os.getcwd(), ctx.verbose)
model.create_manifests()
except Exception as e:
ctx.tracker.track_event('exception', 'dlis_manifest')
ctx.error_log(e)
def publish(ctx, storage, version, increment_type, datasource_id, is_compliant):
"""Publish model into storage."""
try:
ctx.tracker.track_event('command', 'publish')
model = Model(os.getcwd(), ctx.verbose)
model.publish(storage, datasource_id, version, increment_type, is_compliant)
except Exception as e:
ctx.tracker.track_event('exception', 'publish')
ctx.error_log(e)
def test(ctx, file_path, delimiter, mode, endpoint):
"""Running unit tests on current model."""
try:
ctx.tracker.track_event('command', 'test')
model = Model(os.getcwd(), ctx.verbose)
model.test(mode, file_path, delimiter, endpoint, ctx.verbose)
except Exception as e:
ctx.tracker.track_event('exception', 'test')
ctx.error_log(e)
def eval(ctx):
"""Batch scoring on current model."""
try:
ctx.tracker.track_event('command', 'eval')
model = Model(os.getcwd(), ctx.verbose)
model.eval()
except Exception as e:
ctx.tracker.track_event('exception', 'eval')
ctx.error_log(e)
def serve(ctx, port):
"""Serving current model at localhost."""
try:
ctx.tracker.track_event('command', 'serve')
model = Model(os.getcwd(), ctx.verbose)
model.serve(port)
except Exception as e:
ctx.tracker.track_event('exception', 'serve')
ctx.error_log(e)
def package(ctx, platform, skip_archive):
"""Package the model."""
try:
ctx.tracker.track_event('command', 'package')
model = Model(os.getcwd(), ctx.verbose)
if model.language == 'python' and (platform == 'dlisv3' or platform == 'dlisv3binary'):
ctx.log('Warning: Packaging with dlisv3 is only supported for c# models, continue packaging with dlis.')
platform = 'dlis'
model.package(platform, skip_archive)
except Exception as e:
ctx.tracker.track_event('exception', 'package')
ctx.error_log(e)
class TestQASPythonModel(TestCase):
def setUp(self):
self.tmp_dirpath = tempfile.mkdtemp()
factory = TemplateFactory(PYTHON_LANG)
factory.generate(self.tmp_dirpath, 'pytorch', 'Release', 'qas')
os.chdir(self.tmp_dirpath)
self.model = Model(self.tmp_dirpath)
def test_eval(self):
self.model.eval()
self.assertEqual(
os.path.exists(os.path.join(self.tmp_dirpath, '.score')), True)
@mock.patch('oml.models.python.PythonModel.serve')
def test_serve(self, mock_serve):
self.model.serve()
self.assertEqual(mock_serve.call_count, 1)
@skipUnless(sys.platform.startswith('win'), 'requires Windows')
@mock.patch('shutil.rmtree')
@mock.patch('subprocess.run')
def test_QAS_package_without_platform(self, mock_subprocess, mock_rmtree):
package_path = os.path.join(self.tmp_dirpath, '.oml', 'package')
conda_package_path = os.path.join(self.tmp_dirpath, '.oml', 'env',
self.model.model_name)
os.makedirs(conda_package_path, exist_ok=True)
self.assertEqual(os.path.exists(conda_package_path), True)
# hide platform in package command, read from oml.yml
self.assertEqual(self.model.metadata['platform'], 'qas')
self.model.package(None, skip_archive=False)
self.assertEqual(mock_rmtree.call_count, 1)
self.assertEqual(os.path.exists(package_path), True)
self.assertEqual(
os.path.exists(
os.path.join(package_path, 'conda',
'{}.zip'.format(self.model.model_name))), True)
@skipUnless(sys.platform.startswith('win'), 'requires Windows')
@mock.patch('shutil.rmtree')
@mock.patch('subprocess.run')
def test_QAS_package_without_platform_with_skip_archive(
self, mock_subprocess, mock_rmtree):
package_path = os.path.join(self.tmp_dirpath, '.oml', 'package')
conda_package_path = os.path.join(self.tmp_dirpath, '.oml', 'env',
self.model.model_name)
os.makedirs(conda_package_path, exist_ok=True)
self.assertEqual(os.path.exists(conda_package_path), True)
# hide platform in package command, read from oml.yml
self.assertEqual(self.model.metadata['platform'], 'qas')
self.model.package(None, skip_archive=True)
self.assertEqual(mock_rmtree.call_count, 1)
self.assertEqual(os.path.exists(package_path), True)
self.assertEqual(
os.path.exists(
os.path.join(package_path, 'conda',
'{}.zip'.format(self.model.model_name))), False)
def push(ctx, path):
"""Push a model into OXO model repository."""
try:
model = Model(os.getcwd(), ctx.verbose)
if path is None:
path = model.base_path
repo = GitRepositoryHook()
repo.push_model(path)
except Exception as e:
ctx.error_log(e)
def generate_noncomp_template(ctx):
"""Generate non-compliant pipeline template."""
try:
ctx.tracker.track_event('command', 'generate_noncomp_template')
model = Model(os.getcwd(), ctx.verbose)
factory = TemplateFactory(model.language)
factory.generate_noncomp_pipeline_template(model.base_path, model.model_name)
except Exception as e:
ctx.tracker.track_event('exception', 'generate_noncomp_template')
ctx.error_log(e)
def generate_multistage_template(ctx):
"""Generate multistage pipelines template for an existing project."""
try:
ctx.tracker.track_event('command', 'generate_multistage_template')
model = Model(os.getcwd(), ctx.verbose)
factory = TemplateFactory(model.language)
factory.generate_multistage_pipeline_template(model.base_path, model.model_name)
except Exception as e:
ctx.tracker.track_event('exception', 'generate_multistage_template')
ctx.error_log(e)
def list(ctx, model_name):
"""Lists all published versions of a model."""
try:
ctx.tracker.track_event('command', 'list')
model = Model(os.getcwd(), ctx.verbose)
result = model.list_artifacts(model_name)
if result:
tbl = PrettyTable(['Version', 'Path', 'Platform',
'Compliant', 'Created'], align='l')
for row in result:
tbl.add_row([
row['version'],
row['path'],
row['platform'].upper(),
'False' if row['is_compliant'] == 0 else 'True',
row['created_at']])
ctx.log(tbl)
ctx.log('{} published version(s) found.'.format(len(result)))
else:
ctx.log("The model doesn't have any published versions.")
except Exception as e:
ctx.tracker.track_event('exception', 'list')
ctx.error_log(e)
def setUp(self):
self.tmp_dirpath = tempfile.mkdtemp()
factory = TemplateFactory(PYTHON_LANG)
factory.generate(self.tmp_dirpath)
os.chdir(self.tmp_dirpath)
self.model = Model(self.tmp_dirpath)
class TestPythonModel(TestCase):
def setUp(self):
self.tmp_dirpath = tempfile.mkdtemp()
factory = TemplateFactory(PYTHON_LANG)
factory.generate(self.tmp_dirpath)
os.chdir(self.tmp_dirpath)
self.model = Model(self.tmp_dirpath)
def test_eval(self):
self.model.eval()
self.assertEqual(
os.path.exists(os.path.join(self.tmp_dirpath, '.score')), True)
@mock.patch('oml.models.python.PythonModel.serve')
def test_serve(self, mock_serve):
self.model.serve()
self.assertEqual(mock_serve.call_count, 1)
@skipUnless(sys.platform.startswith('win'), 'requires Windows')
@mock.patch('shutil.rmtree')
@mock.patch('subprocess.run')
def test_package(self, mock_subprocess, mock_rmtree):
package_path = os.path.join(self.tmp_dirpath, '.oml', 'package')
conda_package_path = os.path.join(self.tmp_dirpath, '.oml', 'env',
self.model.model_name)
os.makedirs(conda_package_path, exist_ok=True)
self.assertEqual(os.path.exists(conda_package_path), True)
self.model.package('dlis', skip_archive=False)
self.assertEqual(mock_rmtree.call_count, 1)
self.assertEqual(os.path.exists(package_path), True)
self.assertEqual(
os.path.exists(
os.path.join(package_path, 'conda',
'{}.zip'.format(self.model.model_name))), True)
@skipUnless(sys.platform.startswith('win'), 'requires Windows')
@mock.patch('shutil.rmtree')
@mock.patch('subprocess.run')
def test_package_with_skip_archive(self, mock_subprocess, mock_rmtree):
package_path = os.path.join(self.tmp_dirpath, '.oml', 'package')
conda_package_path = os.path.join(self.tmp_dirpath, '.oml', 'env',
self.model.model_name)
os.makedirs(conda_package_path, exist_ok=True)
self.assertEqual(os.path.exists(conda_package_path), True)
self.model.package('dlis', skip_archive=True)
self.assertEqual(mock_rmtree.call_count, 1)
self.assertEqual(os.path.exists(package_path), True)
self.assertEqual(
os.path.exists(
os.path.join(package_path, 'conda',
'{}.zip'.format(self.model.model_name))), False)
@skipUnless(sys.platform.startswith('win'), 'requires Windows')
@mock.patch('oml.util.pipeline.get_pipeline_owner')
@mock.patch('oml.context.Context.is_outdated')
@mock.patch('oml.hooks.catalog.ModelCatalogHook.update_deployment')
@mock.patch('oml.hooks.catalog.ModelCatalogHook.create_deployment')
@mock.patch('oml.hooks.catalog.ModelCatalogHook.register')
@mock.patch('oml.hooks.catalog.ModelCatalogHook.get_model')
@mock.patch('oml.hooks.catalog.ModelCatalogHook.get_model_artifacts')
@mock.patch('oml.hooks.catalog.ModelCatalogHook.get_model_deployments')
@mock.patch('oml.hooks.catalog.ModelCatalogHook.get_model_tests')
@mock.patch('oml.hooks.dlis.DLISHook.get_active_models')
@mock.patch('oml.hooks.dlis.DLISHook.create_deployment')
@mock.patch('oml.hooks.dlis.DLISHook.get_deployments')
@mock.patch('oml.hooks.dlis.DLISHook.get_tests')
@mock.patch('oml.hooks.dlis.DLISHook.create_test')
@mock.patch('subprocess.run')
@mock.patch('oml.util.auth.get_token')
def test_dlis_python_deployment_params(
self, mock_get_token, mock_run, mock_create_test, mock_get_tests,
mock_get_deployments, mock_deployment, mock_get_models,
mock_get_model_tests, mock_get_model_deployments,
mock_get_model_artifacts, mock_get_model, mock_register,
mock_create_deployment, mock_update_deployment, mock_outdated,
mock_pipeline_owner):
mock_get_token.return_value = {'accessToken': '1111'}
artifact_path = 'https://path/to/model/'
mock_get_model.return_value = {
'id': '1',
'owner': 'test\\test',
'name': 'test',
'language': PYTHON_LANG,
}
mock_pipeline_owner.return_value = 'domain\\name'
mock_get_model_artifacts.return_value = [{
'id': '1',
'path': artifact_path,
'is_compliant': False
}]
mock_get_model_tests.return_value = [{'job_id': '1'}]
mock_get_model.return_value.update({'namespace': {'name': 'test'}})
mock_create_test.return_value = 'test'
mock_get_tests.return_value = {'Status': 'Succeeded'}
mock_deployment.return_value = 'deploy'
mock_get_deployments.return_value = [{'ModelVersion': 1}]
# Non-compliant deployment
result = CliRunner().invoke(main, ['dlis', 'deploy', '-ns', 'test'])
self.assertEqual(result.exit_code, 0)
args, kwargs = mock_deployment.call_args
self.assertEqual(args[0]["ModelPath"]["CosmosPath"],
artifact_path + "model/run.cmd")
self.assertEqual(args[0]["CondaPath"]["CosmosPath"],
artifact_path + "conda/test.zip")
self.assertIn('Task submitted', result.output)
# Compliant deployment
result = CliRunner().invoke(
main, ['-v', 'dlis', 'deploy', '-c', '-t', '1234', '-ns', 'test'])
self.assertEqual(result.exit_code, 0)
args, kwargs = mock_deployment.call_args
self.assertEqual(args[0]["ModelPath"]["AzurePath"],
artifact_path + "model/run.cmd")
self.assertEqual(args[0]["CondaPath"]["AzurePath"],
artifact_path + "conda/test.zip")
self.assertIn('Task submitted', result.output)
@skipUnless(sys.platform.startswith('win'), 'requires Windows')
@mock.patch('shutil.rmtree')
@mock.patch('subprocess.run')
def test_manifest(self, mock_subprocess, mock_rmtree):
conda_package_path = os.path.join(self.tmp_dirpath, '.oml', 'env',
self.model.model_name)
os.makedirs(conda_package_path, exist_ok=True)
self.model.package('dlis', skip_archive=True)
self.model.create_manifests()
model_manifest_path = os.path.join(self.tmp_dirpath, '.oml', 'package',
'model', 'SecureManifest.json')
conda_manifest_path = os.path.join(conda_package_path,
'SecureManifest.json')
self.assertEqual(os.path.exists(model_manifest_path), True)
self.assertEqual(os.path.exists(conda_manifest_path), True)
def setUp(self):
self.tmp_dirpath = tempfile.mkdtemp()
factory = TemplateFactory(CSHARP_LANG)
factory.generate(self.tmp_dirpath, 'tensorflow', 'Release')
self.model = Model(self.tmp_dirpath)
class TestCSharpModel(TestCase):
def setUp(self):
self.tmp_dirpath = tempfile.mkdtemp()
factory = TemplateFactory(CSHARP_LANG)
factory.generate(self.tmp_dirpath, 'tensorflow', 'Release')
self.model = Model(self.tmp_dirpath)
@skipUnless(sys.platform.startswith('win'), 'requires Windows')
def test_unit_tests(self):
os.chdir(self.tmp_dirpath)
self.model.test()
self.assertEqual(os.path.exists(os.path.join(self.tmp_dirpath, 'UnitTest', 'TestResults')), True)
@skipUnless(sys.platform.startswith('win'), 'requires Windows')
@mock.patch('oml.models.csharp.CSharpModel.serve')
def test_serve(self, mock_serve):
self.model.serve()
self.assertEqual(mock_serve.call_count, 1)
@skipUnless(sys.platform.startswith('win'), 'requires Windows')
def test_eval(self):
self.model.eval()
self.assertEqual(os.path.exists(os.path.join(self.tmp_dirpath, '.score')), True)
@skipUnless(sys.platform.startswith('win'), 'requires Windows')
def test_package(self):
self.model.package('dlisv3', skip_archive=False)
self.assertEqual(os.path.exists(os.path.join(self.tmp_dirpath, '.oml', 'package')), True)
@skipUnless(sys.platform.startswith('win'), 'requires Windows')
def test_manifest(self):
self.model.package('dlisv3', skip_archive=False)
self.model.create_manifests()
manifest_path = os.path.join(self.tmp_dirpath, '.oml', 'package', 'SecureManifest.json')
self.assertEqual(os.path.exists(manifest_path), True)
@skipUnless(sys.platform.startswith('win'), 'requires Windows')
def test_package_dlisv3binary(self):
self.model.package('dlisv3binary', skip_archive=False)
self.assertEqual(os.path.exists(os.path.join(self.tmp_dirpath, '.oml', 'package')), True)
@skipUnless(sys.platform.startswith('win'), 'requires Windows')
def test_manifest_dlisv3binary(self):
self.model.package('dlisv3binary', skip_archive=False)
self.model.create_manifests()
manifest_path = os.path.join(self.tmp_dirpath, '.oml', 'package', 'SecureManifest.json')
self.assertEqual(os.path.exists(manifest_path), True)