def test_get_after_sjs_restart(self, mock_req):
status_url = utils.urljoin(self.TEST_ENDPOINT, self.client.jobs.base_path, job_id)
get_url = utils.urljoin(self.TEST_ENDPOINT, self.client.jobs.base_path, job_id)
mock_req.get(get_url, status_code=404)
mock_req.get(status_url, text=job_status_response)
test_job = self.client.jobs.get(job_id)
self.assertJobFields(test_job, running)
def test_get_after_sjs_restart(self, mock_req):
status_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.jobs.base_path, job_id)
get_url = utils.urljoin(self.TEST_ENDPOINT, self.client.jobs.base_path,
job_id)
mock_req.get(get_url, status_code=404)
mock_req.get(status_url, text=job_status_response)
test_job = self.client.jobs.get(job_id)
self.assertJobFields(test_job, running)
def test_get_config(self, mock_req):
config_url = utils.urljoin(self.TEST_ENDPOINT, self.client.jobs.base_path, job_id, "config")
get_url = utils.urljoin(self.TEST_ENDPOINT, self.client.jobs.base_path, job_id)
mock_req.get(get_url, text=job_get_result_response)
mock_req.get(config_url, text=job_config_response)
jobObj = self.client.jobs.get(job_id)
job_config = jobObj.get_config()
self.assertIsInstance(job_config, job.JobConfig)
self.assertEqual("test_value", job_config["test_config"])
def test_create(self, mock_req):
post_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.apps.base_path, app_name)
get_url = utils.urljoin(self.TEST_ENDPOINT, self.client.apps.base_path)
mock_req.post(post_url, text=app_create_response)
mock_req.get(get_url, text=app_get_response)
test_app = self.client.apps.create(app_name, "fake-data")
self.assertAppFields(test_app)
def test_job_list(self, mock_req):
list_url = utils.urljoin(self.TEST_ENDPOINT, self.client.jobs.base_path)
get_url = utils.urljoin(self.TEST_ENDPOINT, self.client.jobs.base_path, job_id)
mock_req.get(get_url, text=job_get_result_response)
mock_req.get(list_url, text=job_list_response)
job_list = self.client.jobs.list()
test_job = next(job_list)
self.assertJobFields(test_job, running)
def test_context_instance_delete(self, mock_req):
"""Test delete method of context instace"""
get_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.contexts.base_path)
del_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.contexts.base_path, CONTEXT_NAME)
mock_req.get(get_url, text=CONTEXT_GET_RESPONSE)
test_ctx = self.client.contexts.get(CONTEXT_NAME)
mock_req.delete(del_url, text=CONTEXT_CREATE_RESPONSE)
test_ctx.delete()
def test_context_instance_delete(self, mock_req):
get_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.contexts.base_path)
del_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.contexts.base_path,
context_name)
mock_req.get(get_url, text=context_get_response)
test_ctx = self.client.contexts.get(context_name)
mock_req.delete(del_url, text=context_create_response)
test_ctx.delete()
def test_create(self, mock_req):
post_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.apps.base_path,
app_name)
get_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.apps.base_path)
mock_req.post(post_url, text=app_create_response)
mock_req.get(get_url, text=app_get_response)
test_app = self.client.apps.create(app_name, "fake-data")
self.assertAppFields(test_app)
def test_get(self, mock_req):
status_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.jobs.base_path,
job_id)
get_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.jobs.base_path,
job_id)
mock_req.get(get_url, text=job_get_result_response)
mock_req.get(status_url, text=job_status_response)
test_job = self.client.jobs.get(job_id)
self.assertJobFields(test_job)
def test_job_list(self, mock_req):
list_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.jobs.base_path)
get_url = utils.urljoin(self.TEST_ENDPOINT, self.client.jobs.base_path,
job_id)
mock_req.get(get_url, text=job_get_result_response)
mock_req.get(list_url, text=job_list_response)
job_list = self.client.jobs.list()
test_job = next(job_list)
self.assertJobFields(test_job, running)
def test_job_instance_delete(self, mock_req):
delete_url = utils.urljoin(self.TEST_ENDPOINT, self.client.jobs.base_path, job_id)
status_url = utils.urljoin(self.TEST_ENDPOINT, self.client.jobs.base_path, job_id)
get_url = utils.urljoin(self.TEST_ENDPOINT, self.client.jobs.base_path, job_id)
mock_req.get(get_url, text=job_get_result_response)
mock_req.get(status_url, text=job_status_response)
mock_req.delete(delete_url, text=job_delete_response)
test_job = self.client.jobs.get(job_id)
resp = test_job.delete()
self.assertEqual(200, resp.status_code)
resp = resp.json()
self.assertEqual("KILLED", resp["status"])
def test_get_config(self, mock_req):
config_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.jobs.base_path, job_id,
"config")
get_url = utils.urljoin(self.TEST_ENDPOINT, self.client.jobs.base_path,
job_id)
mock_req.get(get_url, text=job_get_result_response)
mock_req.get(config_url, text=job_config_response)
jobObj = self.client.jobs.get(job_id)
job_config = jobObj.get_config()
self.assertIsInstance(job_config, job.JobConfig)
self.assertEqual("test_value", job_config["test_config"])
def test_job_instance_delete(self, mock_req):
delete_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.jobs.base_path, job_id)
status_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.jobs.base_path, job_id)
get_url = utils.urljoin(self.TEST_ENDPOINT, self.client.jobs.base_path,
job_id)
mock_req.get(get_url, text=job_get_result_response)
mock_req.get(status_url, text=job_status_response)
mock_req.delete(delete_url, text=job_delete_response)
test_job = self.client.jobs.get(job_id)
resp = test_job.delete()
self.assertEqual(200, resp.status_code)
resp = resp.json()
self.assertEqual("KILLED", resp["status"])
def test_delete(self, mock_req):
get_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.contexts.base_path,
context_name)
mock_req.delete(get_url, text=context_create_response)
self.client.contexts.delete(context_name)
def test_delete(self, mock_req):
delete_url = utils.urljoin(self.TEST_ENDPOINT, self.client.jobs.base_path, job_id)
mock_req.delete(delete_url, text=job_delete_response)
resp = self.client.jobs.delete(job_id)
self.assertEqual(200, resp.status_code)
resp = resp.json()
self.assertEqual("KILLED", resp["status"])
def test_get(self, mock_req):
get_url = utils.urljoin(self.TEST_ENDPOINT, self.client.apps.base_path)
mock_req.get(get_url, text=app_get_response)
test_app = self.client.apps.get(app_name)
self.assertAppFields(test_app)
def test_get_non_existing(self, mock_req):
get_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.contexts.base_path)
mock_req.get(get_url, text=context_get_response)
self.assertRaises(exceptions.NotFoundException,
self.client.contexts.get, 'does-not-exist')
def test_delete(self, mock_req):
"""Test context delete"""
get_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.contexts.base_path, CONTEXT_NAME)
mock_req.delete(get_url, text=CONTEXT_CREATE_RESPONSE)
self.client.contexts.delete(CONTEXT_NAME)
def test_create(self, mock_req):
post_url = utils.urljoin(self.TEST_ENDPOINT, self.client.jobs.base_path)
query = "?classPath=test.class.path&appName=test_app"
post_url = "{}{}".format(post_url, query)
mock_req.post(post_url, text=job_create_response)
test_job = self.client.jobs.create(FakeApp, "test.class.path")
self.assertEqual(context, test_job.context)
self.assertEqual(started, test_job.status)
self.assertEqual(job_id, test_job.jobId)
get_url = utils.urljoin(self.TEST_ENDPOINT, self.client.jobs.base_path, job_id)
mock_req.get(get_url, text=job_get_result_response)
test_job_result = self.client.jobs.get(test_job.jobId)
self.assertEqual(finished, test_job_result.status)
self.assertEqual([1, 2, 3], test_job_result.result)
def test_get(self, mock_req):
get_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.apps.base_path)
mock_req.get(get_url, text=app_get_response)
test_app = self.client.apps.get(app_name)
self.assertAppFields(test_app)
def test_get_non_existing(self, mock_req):
"""Test get non existing context"""
get_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.contexts.base_path)
mock_req.get(get_url, text=CONTEXT_GET_RESPONSE)
self.assertRaises(exceptions.NotFoundException,
self.client.contexts.get, 'does-not-exist')
def test_get(self, mock_req):
get_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.contexts.base_path)
mock_req.get(get_url, text=context_get_response)
test_ctx = self.client.contexts.get(context_name)
self.assertContextFields(test_ctx)
def test_list(self, mock_req):
get_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.contexts.base_path)
mock_req.get(get_url, text=context_get_response)
ctx_list = self.client.contexts.list()
self.assertContextFields(next(ctx_list))
def test_list(self, mock_req):
get_url = utils.urljoin(self.TEST_ENDPOINT, self.client.apps.base_path)
mock_req.get(get_url, text=app_get_response)
app_list = self.client.apps.list()
for tapp in app_list:
self.assertAppFields(tapp)
def test_get_non_existing(self, mock_req):
status_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.jobs.base_path,
"does-not-exist")
mock_req.get(status_url, text=job_get_status_not_found_response,
status_code=404)
self.assertRaises(exceptions.NotFoundException,
self.client.jobs.get, 'does-not-exist')
def test_delete(self, mock_req):
delete_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.jobs.base_path, job_id)
mock_req.delete(delete_url, text=job_delete_response)
resp = self.client.jobs.delete(job_id)
self.assertEqual(200, resp.status_code)
resp = resp.json()
self.assertEqual("KILLED", resp["status"])
def delete(self, job_id):
"""Delete a specific Job.
:param job_id: The jobId of the :class:`Job` to get.
"""
url = self.base_path
url = utils.urljoin(url, job_id)
resp = self.client._delete(url)
return resp
def test_list(self, mock_req):
"""Test list contexts"""
get_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.contexts.base_path)
mock_req.get(get_url, text=CONTEXT_GET_RESPONSE)
ctx_list = self.client.contexts.list()
self.assert_ctx_fields(next(ctx_list))
def test_get_non_existing(self, mock_req):
status_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.jobs.base_path,
"does-not-exist")
mock_req.get(status_url,
text=job_get_status_not_found_response,
status_code=404)
self.assertRaises(exceptions.NotFoundException, self.client.jobs.get,
'does-not-exist')
def test_get(self, mock_req):
"""Test get context"""
get_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.contexts.base_path)
mock_req.get(get_url, text=CONTEXT_GET_RESPONSE)
test_ctx = self.client.contexts.get(CONTEXT_NAME)
self.assert_ctx_fields(test_ctx)
def delete(self, job_id):
"""Delete a specific Job.
:param job_id: The jobId of the :class:`Job` to get.
"""
url = self.base_path
url = utils.urljoin(url, job_id)
resp = self.client._delete(url)
return resp
def test_list(self, mock_req):
get_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.apps.base_path)
mock_req.get(get_url, text=app_get_response)
app_list = self.client.apps.list()
for tapp in app_list:
self.assertAppFields(tapp)
def get(self, job_id):
"""Get a specific Job. This returns more information than create.
:param job_id: The jobId of the :class:`Job` to get.
:rtype: :class:`Job`
"""
url = utils.urljoin(self.base_path, job_id)
resp = self.client._get(url).json()
return self._create_resource(resp)
def get(self, job_id):
"""Get a specific Job. This returns more information than create.
:param job_id: The jobId of the :class:`Job` to get.
:rtype: :class:`Job`
"""
url = utils.urljoin(self.base_path, job_id)
resp = self.client._get(url).json()
return self._create_resource(resp)
def get_config(self, job_id):
"""Get job configuration.
:param job_id: The jobId of the :class:`Job` to get.
:rtype: :class:`JobConfig`
"""
url = utils.urljoin(self.base_path, job_id, "config")
resp = self.client._get(url).json()
return JobConfig(resp)
def test_create(self, mock_req):
post_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.jobs.base_path)
query = "?classPath=test.class.path&appName=test_app"
post_url = "{}{}".format(post_url, query)
mock_req.post(post_url, text=job_create_response)
test_job = self.client.jobs.create(FakeApp, "test.class.path")
self.assertEqual(context, test_job.context)
self.assertEqual(started, test_job.status)
self.assertEqual(job_id, test_job.jobId)
get_url = utils.urljoin(self.TEST_ENDPOINT, self.client.jobs.base_path,
job_id)
mock_req.get(get_url, text=job_get_result_response)
test_job_result = self.client.jobs.get(test_job.jobId)
self.assertEqual(finished, test_job_result.status)
self.assertEqual([1, 2, 3], test_job_result.result)
def delete(self, name):
"""Delete a specific App.
:param name: The name of the :class:`App` to delete.
"""
url = self.base_path
url = utils.urljoin(url, name)
resp = self.client._delete(url)
return resp
def get_config(self, job_id):
"""Get job configuration.
:param job_id: The jobId of the :class:`Job` to get.
:rtype: :class:`JobConfig`
"""
url = utils.urljoin(self.base_path, job_id, "config")
resp = self.client._get(url).json()
return JobConfig(resp)
def create(self, name, params=None):
"""Create a Spark context.
:param name: Descriptive name of context
:param params: Dictionary of context parameters
:rtype: :class:`Context`
"""
url = utils.urljoin(self.base_path, name)
self.client._post(url, params=params)
return self._create_resource(name)
def test_create(self, mock_req):
"""Test create context"""
post_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.contexts.base_path, CONTEXT_NAME)
mock_req.post(post_url, text=CONTEXT_CREATE_RESPONSE)
params = {'num-cpu-cores': '4', 'memory-per-node': '512m'}
test_context = self.client.contexts.create(CONTEXT_NAME, params)
self.assert_ctx_fields(test_context)
def test_create_with_ctx(self, mock_req):
post_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.jobs.base_path)
query = ("?classPath=test.class.path&appName=test_app&"
"context=test-context")
post_url = "{}{}".format(post_url, query)
get_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.jobs.base_path,
job_id)
mock_req.post(post_url, text=job_create_response)
mock_req.get(get_url, text=job_get_result_response)
test_job = self.client.jobs.create(FakeApp, "test.class.path",
ctx=FakeContext)
self.assertEqual(context, test_job.context)
self.assertEqual(status, test_job.status)
self.assertEqual(job_id, test_job.jobId)
self.assertEqual("STARTED", test_job.status)
self.assertEqual([1, 2, 3], test_job.result)
def test_create(self, mock_req):
post_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.contexts.base_path,
context_name)
mock_req.post(post_url, text=context_create_response)
params = {'num-cpu-cores': '4',
'memory-per-node': '512m'}
test_context = self.client.contexts.create(context_name, params)
self.assertContextFields(test_context)
def create(self, name, jar_blob):
"""Create an app.
:param name: Descriptive name of application
:param jar_blob: Jar binary
:rtype: :class:`App`
"""
url = self.base_path
url = utils.urljoin(url, name)
# Strange that it is not JSON
self.client._post(url, data=jar_blob)
return self.get(name)
def test_create_with_sync(self, mock_req):
post_url = utils.urljoin(self.TEST_ENDPOINT,
self.client.jobs.base_path)
query = "?classPath=test.class.path&appName=test_app"
post_url = "{}{}".format(post_url, query)
mock_req.post(post_url, text=job_create_with_sync_response)
test_job_result = self.client.jobs.create(FakeApp,
"test.class.path",
sync=True)
self.assertEqual(job_id, test_job_result.jobId)
self.assertEqual([1, 2, 3], test_job_result.result)
def create(self, name, app_binary, app_type=AppType.JAVA):
"""Create an app.
:param name: Descriptive name of application
:param app_binary: Application binary
:param app_type: App type, for example java or python, default: java
:rtype: :class:`App`
"""
headers = {'Content-Type': AppType.get_header(app_type)}
url = self.base_path
url = utils.urljoin(url, name)
# Strange that it is not JSON
self.client._post(url, data=app_binary, headers=headers)
time.sleep(1)
return self.get(name)
def _request(self, path, method, **kwargs):
url = utils.urljoin(self.endpoint, path)
http = requests.Session()
if self.auth:
kwargs['auth'] = self.auth
resp = http.request(method, url, **kwargs)
try:
resp.raise_for_status()
except requests.RequestException as e:
if resp.status_code == 404:
exc_type = exceptions.NotFoundException
else:
exc_type = exceptions.HttpException
raise exc_type(str(e),
details=self._parse_error_response(resp),
status_code=resp.status_code)
return resp
def test_get_non_existing(self, mock_req):
get_url = utils.urljoin(self.TEST_ENDPOINT, self.client.apps.base_path)
mock_req.get(get_url, text=app_get_response)
self.assertRaises(exceptions.NotFoundException, self.client.apps.get,
'does-not-exist')
def test_delete(self, mock_req):
get_url = utils.urljoin(self.TEST_ENDPOINT, self.client.apps.base_path,
app_name)
mock_req.delete(get_url, text=app_create_response)
self.client.apps.delete(app_name)