def _get_sjsclient():
sjs_url = os.getenv("TESTSJS_SPARKJOB_SERVER_URL")
sjs_user = os.getenv("TESTSJS_SPARKJOB_SERVER_USERNAME")
sjs_password = os.getenv("TESTSJS_SPARKJOB_SERVER_PASSWORD")
auth = None
if sjs_user or sjs_password:
auth = requests.auth.HTTPBasicAuth(sjs_user, sjs_password)
return client.Client(sjs_url, auth)
def test_parse_error_response(self):
http_client = client.Client(self.TEST_URL)
resp = mock.Mock()
resp.json = mock.Mock()
resp.json.return_value = {"status": "Not found"}
self.assertEqual("Not found", http_client._parse_error_response(resp))
resp = FakeRespone("Not found")
self.assertEqual("Not found", http_client._parse_error_response(resp))
def test_get(self, mock_req):
mock_req.get(self.TEST_URL, text=fake_response)
sclient = client.Client(self.TEST_ENDPOINT)
resp = sclient._get(self.TEST_PATH)
self.assertEqual("GET", mock_req.last_request.method)
self.assertResponseOK(resp, body=fake_response)
def test_not_found(self, mock_req):
sclient = client.Client(self.TEST_ENDPOINT)
status = 404
mock_req.get(self.TEST_URL, status_code=status)
exc = self.assertRaises(exceptions.NotFoundException, sclient._get,
self.TEST_PATH)
self.assertEqual(status, exc.status_code)
def test_server_error(self, mock_req):
sclient = client.Client(self.TEST_ENDPOINT)
status = 500
mock_req.get(self.TEST_URL, status_code=500)
exc = self.assertRaises(exceptions.HttpException, sclient._get,
self.TEST_PATH)
self.assertEqual(status, exc.status_code)
def test_put(self, mock_req):
mock_req.put(self.TEST_URL, text=fake_response)
sclient = client.Client(self.TEST_ENDPOINT)
resp = sclient._put(self.TEST_PATH, data=fake_request)
self.assertEqual("PUT", mock_req.last_request.method)
self.assertEqual(
fake_request,
mock_req.last_request.body,
)
self.assertResponseOK(resp, body=fake_response)
def test_post(self, mock_req):
mock_req.post(self.TEST_URL, text=fake_response)
sclient = client.Client(self.TEST_ENDPOINT)
resp = sclient._post(self.TEST_PATH, json=fake_record)
self.assertEqual("POST", mock_req.last_request.method)
self.assertEqual(
json.dumps(fake_record),
mock_req.last_request.body,
)
self.assertResponseOK(resp, body=fake_response)
def submit(self, url):
sjs = client.Client(url)
egg_file_path = "/home/dbalck/251/spark_blast/test/spark/dist/blast-0.0.0-py2.7.egg"
egg_blob = open(egg_file_path, 'rb').read()
wc_app = sjs.apps.create("ttc_wordcount", egg_blob,
sjsapp.AppType.PYTHON)
for app in sjs.apps.list():
print app.name
test_app = sjs.apps.get("ttc_wordcount")
class_path = "spark.wordcount.WordCountSparkJob"
# config = {"test_config": "test_config_value"}
job = sjs.jobs.create(test_app, class_path)
print("Job Status: ", job.status)
def submit_job(api_url, class_name):
sjs = client.Client(JobserverDetails.get_jobserver_url())
app = sjs.apps.get(JobserverDetails.get_app())
ctx = sjs.contexts.get(JobserverDetails.get_context())
class_path = JobserverDetails.get_class_path(class_name)
config = {'config_url': api_url}
job = sjs.jobs.create(app, class_path, ctx=ctx, conf=config)
# print
JobserverDetails.print_job_details(job)
def setUp(self):
super(TestJob, self).setUp()
self.client = client.Client(self.TEST_ENDPOINT)
import time
import logging
import json
from sjsclient import client
logging.basicConfig()
logger = logging.getLogger()
logger.setLevel(logging.INFO)
sjs = client.Client("http://jobserver.ml-test-blog.internal:8090")
myApp = sjs.apps.get("ml")
myContext = sjs.contexts.get("ml-context")
def testHandler(event, context):
logger.info('got event{}'.format(event))
s3DataLocProtocol = "\"s3://{}\"".format(event['s3DataLoc'])
s3ModelLocProtocol = "\"s3://{}\"".format(event['s3ModelLoc'])
conf = '{' + 's3DataLoc:{},s3ModelLoc:{}'.format(s3DataLocProtocol,
s3ModelLocProtocol) + '}'
class_path = "com.amazonaws.proserv.ml.TestParams"
myJob = sjs.jobs.create(myApp, class_path, conf=conf, ctx=myContext)
myId = myJob.jobId
while myJob.status != "FINISHED":
time.sleep(2)
myJob = sjs.jobs.get(myId)
return {'result': sjs.jobs.get(myId).result}
def loadHandler(event, context):
import json
from sjsclient import client
host = "localhost"
host_with_hdfs = "hdfs://" + host
sjs = client.Client("http://" + host + ":" + "8090")
test_app = sjs.apps.get("test_api_1")
test_ctx = sjs.contexts.get("pysql-context")
class_path = "bi.sparkjobs.filter_job"
config = {
'result_path':
'hdfs://localhost:9000/datasets/65/output/meta.json_Ecommerce_data0.csv_1',
'input_path':
'hdfs://localhost:9000/datasets/65/input/Ecommerce_data0.csv',
'measure_suggestions': [],
'dimension_filter': {
u'Buyer_Age': [u'45 to 54', u'18 to 24']
},
'measure_filter': {},
'consider_columns': ['Buyer_Age']
}
job = sjs.jobs.create(test_app,
class_path,
ctx=test_ctx,
conf=json.dumps(config))
job_url = "http://{0}:8090/jobs/{1}".format(host, job.jobId)
from sjsclient import client
from sjsclient import exceptions
from stm import settings as s
import json
import time
import temp_traffic_utils
sjs = client.Client(s.job_server_url)
ttu = temp_traffic_utils.TempTrafficUtils()
def load_model(model_name):
app = sjs.apps.get(s.app_name)
context = sjs.contexts.get(s.context)
job = sjs.jobs.create(app,
s.class_path,
ctx=context,
conf=json.dumps({
"task": "load_model",
"model": model_name
}))
while job.status != "FINISHED":
time.sleep(1)
job = sjs.jobs.get(job.jobId)
if job.status == "ERROR":
return json.dumps({
"status":
0,
"error":
"Job Fail -> see to JobServer interface for more details"
})
def setUp(self):
"""Test suite setup"""
super(TestContext, self).setUp()
self.client = client.Client(self.TEST_ENDPOINT)
