def test_get_full_fn(self, m): m.register_uri('GET', self.MIST_APP_URL + 'endpoints/simple', text=""" { "name": "simple", "execute": { "numbers": { "type": "MList", "args": [ { "type": "MInt", "args": [] } ] }, "multiplier": { "type": "MOption", "args": [ { "type": "MInt", "args": [] } ] } }, "path": "my-awesome-job_0_0_1.jar", "tags": [], "className": "SimpleContext$", "defaultContext": "simple", "lang": "scala" } """) mist = MistApp() res = mist.get_function_json('simple') self.assertIsInstance(res, dict) self.assertIn('execute', res)
def test_validate_methods(self, m): m.register_uri('GET', self.MIST_APP_URL + 'artifacts/test-name.jar/sha', text="SOME_CONTENT") m.register_uri('GET', self.MIST_APP_URL + 'artifacts/unknown.jar/sha', status_code=404) m.register_uri('GET', self.MIST_APP_URL + 'contexts/test-ctx', text=""" { "name": "simple", "maxJobs": 20, "workerMode": "shared", "precreated": false, "sparkConf": { "spark.executor.memory": "256m", "spark.driver.memory": "512m" }, "runOptions": "", "downtime": "Inf", "streamingDuration": "1s" } """) m.register_uri('GET', self.MIST_APP_URL + 'contexts/unknown-ctx', status_code=404) mist = MistApp() with self.assertRaises(ValueError): mist._validate_artifact(models.Artifact('test-name', 'test-name.jar')) mist._validate_artifact(models.Artifact('unknown', 'test-name.jar')) mist._validate_context(models.Context('test')) fn1 = models.Function('test', 'Test', 'test-ctx', path='test-name.jar') mist._validate_function(fn1) fn2 = models.Function('test', 'Test', 'unknown-ctx', path='unknown.jar') with self.assertRaises(ValueError): mist._validate_function(fn2) fn3 = models.Function('test', 'Test', 'test-ctx', path='unknown.jar') with self.assertRaises(ValueError): mist._validate_function(fn3)
def test_deploy_function_exists(self, m): m.register_uri('PUT', self.MIST_APP_URL + 'functions', text=""" { "name": "test-fn", "className": "Test", "path": "test-path.py", "defaultContext": "foo" } """) # returns something by with addr m.register_uri('GET', self.MIST_APP_URL + 'functions/test-fn', text=""" { "name": "test-fn", "className": "Test", "path": "test-path.py", "defaultContext": "foo" } """) fn = models.Function('test-fn', 'Test', 'foo', 'test-path.py') mist = MistApp() res = mist.update_function(fn) self.assertIsInstance(res, models.Function) self.assertEqual(res.name, 'test-fn') self.assertEqual(res.class_name, 'Test') self.assertEqual(res.path, 'test-path.py') self.assertEqual(res.default_context.name, 'foo')
def test_parse_deployment_wo_name(self, m): mist = MistApp() priority, res = mist.parse_deployment(self.apply_artifact_config2) self.assertIsInstance(res, models.Deployment) self.assertEqual(res.name, 'app') self.assertEqual(res.version, '0.0.1') self.assertEqual(res.model_type, 'Artifact')
def test_start_job(self, m): json_str = '{"errors": [], "payload": {"result": [1, 2, 3]}, "success": true}' m.register_uri('POST', self.MIST_APP_URL + 'endpoints/simple/jobs', text=json_str) mist = MistApp() res = mist.start_job('simple', '{"numbers": [1,2,3], "multiplier": 4}') self.assertEqual(res, json.loads(json_str))
def test_update_deployment(self, m): mist = MistApp() artifact = models.Artifact('test-artifact.py', 'test-artifact.py') context = models.Context('test-context') fn = models.Function('test-fn', 'Test', 'test-context', 'test-path.py') mist.artifact_parser.parse = MagicMock(return_value=artifact) mist.context_parser.parse = MagicMock(return_value=context) mist.function_parser.parse = MagicMock(return_value=fn) mist._MistApp__upload_artifact = MagicMock(return_value=artifact) mist.update_context = MagicMock(return_value=context) mist.update_function = MagicMock(return_value=fn) mist._validate_artifact = MagicMock(return_value=None) mist._validate_function = MagicMock(return_value=None) mist._validate_context = MagicMock(return_value=None) mist.update( models.Deployment('test-artifact.py', 'Artifact', ConfigTree(**{'file-path': 'test-path.py'}))) mist.update(models.Deployment('test-context', 'Context', ConfigTree())) mist.update(models.Deployment('test-fn', 'Function', ConfigTree())) call_artifact = mist._MistApp__upload_artifact.call_args[0][0] call_fn = mist.update_function.call_args[0][0] call_context = mist.update_context.call_args[0][0] self.assertEqual(call_artifact.name, 'test-artifact.py') self.assertEqual(call_context.name, 'test-context') self.assertEqual(call_fn.name, 'test-fn')
def test_jobs(self, m): mist = MistApp() m.register_uri('GET', self.MIST_APP_URL + 'jobs?status=started&status=finished', text="""[ { "jobId": "test", "function": "foo", "context": "bar", "source": "http", "status": "started" }, { "jobId": "test2", "function": "foo2", "context": "bar2", "source": "http", "status": "finished" } ]""".strip()) jobs = mist.jobs('started,finished') self.assertEqual(len(jobs), 2) item = jobs[0] self.assertEqual(item.job_id, 'test') self.assertEqual(item.status, 'started') self.assertEqual(jobs[1].job_id, 'test2') self.assertEqual(jobs[1].status, 'finished')
def test_get_sha1(self, m): m.register_uri('GET', self.MIST_APP_URL + 'artifacts/my-artifact.py/sha', text="SOME_CONTENT") m.register_uri('GET', self.MIST_APP_URL + 'artifacts/unknown/sha', status_code=404) mist = MistApp() sha1 = mist.get_sha1('my-artifact.py') self.assertIsNotNone(sha1) res = mist.get_sha1('unknown') self.assertIsNone(res)
def test_functions(self, m): mist = MistApp() m.register_uri('GET', self.MIST_APP_URL + 'endpoints', text='[{"name": "test", "path": "test-path", "defaultContext": "foo", "className": "Test$"}]') res = mist.functions() self.assertEqual(len(res), 1) item = res[0] self.assertEqual(item.name, 'test') self.assertEqual(item.path, 'test-path')
def test_workers(self, m): mist = MistApp() m.register_uri('GET', self.MIST_APP_URL + 'workers', text='[{"name": "test", "address": "localhost:0", "sparkUi": "foo"}]') res = mist.workers() self.assertEqual(len(res), 1) item = res[0] self.assertEqual(item.name, 'test') self.assertEqual(item.address, 'localhost:0') self.assertEqual(item.spark_ui, 'foo')
def test_contexts(self, m): mist = MistApp() m.register_uri('GET', self.MIST_APP_URL + 'contexts', text='[{"name": "foo", "workerMode": "shared"}]') res = mist.contexts() print(res) self.assertEqual(len(res), 1) item = res[0] self.assertEqual(item.name, 'foo') self.assertEqual(item.worker_mode, 'shared')
def test_get_status_return_smth(self, m): m.register_uri('GET', self.MIST_APP_URL + 'status', text=""" { "mistVersion": "1.2.3", "sparkVersion": "1.2.3" } """) mist = MistApp() status = mist.get_status() self.assertIsNotNone(status)
def test_deploy_context(self, m): m.register_uri('POST', self.MIST_APP_URL + 'contexts', text=""" { "name": "test-ctx", "workerMode": "shared" } """) ctx = models.Context('foo', worker_mode='shared') mist = MistApp() res = mist.update_context(ctx) self.assertIsInstance(res, models.Context) self.assertEqual(res.name, 'test-ctx') self.assertEqual(res.worker_mode, 'shared')
def test_update_deployments_should_catch_exceptions(self, m): mist = MistApp(validate=False) context = models.Context('test-context') fn = models.Function('test-fn', 'Test', 'test-context', 'test-path.py') mist.update_function = MagicMock(return_value=fn) mist.update_context = MagicMock(return_value=context) mist.context_parser.parse = MagicMock(return_value=context) mist.function_parser.parse = MagicMock(return_value=fn) depls = [ models.Deployment('simple', 'Function', ConfigTree()), models.Deployment('simple-ctx', 'Context', ConfigTree()) ] mist.update_deployments(depls)
def test_get_fn(self, m): m.register_uri('GET', self.MIST_APP_URL + 'endpoints/simple', text=""" { "name": "simple", "className": "Test", "path": "test-path.py", "defaultContext": "foo" } """) mist = MistApp() res = mist.get_function('simple') self.assertIsInstance(res, models.Function) self.assertEqual(res.name, 'simple') self.assertEqual(res.class_name, 'Test') self.assertEqual(res.path, 'test-path.py') self.assertEqual(res.default_context.name, 'foo')
def test_deploy_function(self, m): m.register_uri('POST', self.MIST_APP_URL + 'endpoints', text=""" { "name": "test-fn", "className": "Test", "path": "test-path.py", "defaultContext": "foo" } """) fn = models.Function('test-fn', 'Test', 'foo', 'test-path.py') mist = MistApp() res = mist.update_function(fn) self.assertIsInstance(res, models.Function) self.assertEqual(res.name, 'test-fn') self.assertEqual(res.class_name, 'Test') self.assertEqual(res.path, 'test-path.py') self.assertEqual(res.default_context.name, 'foo')
def test_get_context(self, m): m.register_uri('GET', self.MIST_APP_URL + 'contexts/simple', text=""" { "name": "simple", "maxJobs": 20, "workerMode": "shared", "precreated": false, "sparkConf": { "spark.executor.memory": "256m", "spark.driver.memory": "512m" }, "runOptions": "", "downtime": "Inf", "streamingDuration": "1s" } """) mist = MistApp() res = mist.get_context('simple') self.assertIsInstance(res, models.Context) self.assertEqual(res.name, 'simple')
def test_mist_cli_get_mist_versions(self): mist = MistApp() actual_status = dict(mistVersion="1.2.3", sparkVersion="4.5.6", javaVersion=dict(runtimeVersion="7.8.9")) mist.get_status = MagicMock(return_value=actual_status) t = get_mist_versions(mist) self.assertEqual(t[0], "1.2.3") self.assertEqual(t[1], "4.5.6") self.assertEqual(t[2], "7.8.9") status_wo_java_version = dict(mistVersion="1.2.3", sparkVersion="4.5.6") mist.get_status = MagicMock(return_value=status_wo_java_version) t = get_mist_versions(mist) self.assertEqual(t[0], "1.2.3") self.assertEqual(t[1], "4.5.6") self.assertEqual(t[2], "UNKNOWN") status_wo_java_version = dict(mistVersion="1.2.3", sparkVersion="4.5.6", javaVersion=dict()) mist.get_status = MagicMock(return_value=status_wo_java_version) t = get_mist_versions(mist) self.assertEqual(t[0], "1.2.3") self.assertEqual(t[1], "4.5.6") self.assertEqual(t[2], "UNKNOWN") status_wo_mist_spark_version = dict(javaVersion=dict(runtimeVersion="7.8.9")) mist.get_status = MagicMock(return_value=status_wo_mist_spark_version) t = get_mist_versions(mist) self.assertEqual(t[0], "UNKNOWN") self.assertEqual(t[1], "UNKNOWN") self.assertEqual(t[2], "7.8.9")
def test_mist_app_creation(self, m): mist = MistApp() self.assertEqual(mist.host, 'localhost') self.assertEqual(mist.port, 2004) self.assertEqual(mist.accept_all, False) self.assertEqual(mist.format_table, False)
def test_get_status_return_empty_dict(self, m): m.register_uri('GET', self.MIST_APP_URL + 'status', status_code=404) mist = MistApp() status = mist.get_status() self.assertIsNotNone(status) self.assertEqual(len(status), 0)
def test_kill_worker(self, m): mist = MistApp() m.register_uri('DELETE', self.MIST_APP_URL + 'workers/some-id-with-dash', text='Cancelled') mist.kill_worker('some-id-with-dash')
def test_cancel_job(self, m): mist = MistApp() m.register_uri('DELETE', self.MIST_APP_URL + 'jobs/some-id-with-dash', text='Cancelled') mist.cancel_job('some-id-with-dash')