def test_zsession_spark_submit(self):
client_config = ClientConfig("http://localhost:8080")
session = ZSession(client_config,
'spark',
intp_properties={'spark.master': 'local[*]'})
session.login('user1', 'password2')
try:
session.start()
self.assertIsNotNone(session.session_info.session_id)
self.assertIsNotNone(session.session_info.weburl)
result = session.submit("sc.version")
result = session.wait_util_finished(result.statement_id)
self.assertEqual('FINISHED', result.status)
self.assertEqual(1, len(result.results))
self.assertEqual(0, len(result.jobUrls))
result = session.submit(
"df = spark.createDataFrame([(1,'a'),(2,'b')])\n" +
"df.registerTempTable('df')\n" + "df.show()",
sub_interpreter="pyspark")
result = session.wait_util_finished(result.statement_id)
self.assertEqual('FINISHED', result.status)
self.assertEqual(1, len(result.results))
self.assertTrue(len(result.jobUrls) > 0)
self.assertEqual('TEXT', result.results[0][0])
finally:
session.stop()
def test_execute_paragraph(self):
client_config = ClientConfig("http://localhost:8080")
client = ZeppelinClient(client_config)
client.login('user1', 'password2')
note_id = None
try:
note_id = client.create_note('/pyzeppelin/test/note_1')
paragraph_id = client.add_paragraph(note_id, 'shell example', "%sh echo 'hello world'")
paragraph_result = client.execute_paragraph(note_id, paragraph_id)
self.assertEqual('FINISHED', paragraph_result.status)
self.assertEqual(1, len(paragraph_result.results))
self.assertEqual('TEXT', paragraph_result.results[0][0])
self.assertEqual('hello world\n', paragraph_result.results[0][1])
# # dynamic forms
paragraph_id = client.add_paragraph(note_id, "dynamic form example", "%sh echo 'hello ${name=abc}'")
paragraph_result = client.execute_paragraph(note_id, paragraph_id)
self.assertEqual('FINISHED', paragraph_result.status)
self.assertEqual(1, len(paragraph_result.results))
self.assertEqual('TEXT', paragraph_result.results[0][0])
self.assertEqual('hello abc\n', paragraph_result.results[0][1])
# run paragraph with parameters
paragraph_result = client.execute_paragraph(note_id, paragraph_id, params = {'name': 'zeppelin'})
self.assertEqual('FINISHED', paragraph_result.status)
self.assertEqual(1, len(paragraph_result.results))
self.assertEqual('TEXT', paragraph_result.results[0][0])
self.assertEqual('hello zeppelin\n', paragraph_result.results[0][1])
finally:
if note_id:
client.delete_note(note_id)
def test_note_operation(self):
client_config = ClientConfig("http://localhost:8080")
client = ZeppelinClient(client_config)
#client.login('user1', 'password2')
client.get_version()
note_id = client.create_note('/pyzeppelin/test/note_1')
client.delete_note(note_id)
with self.assertRaises(Exception) as context:
client.delete_note('invalid_note_id')
self.assertTrue('No such note' in str(context.exception))
# note2_id = None
# try:
# note2_id = client.create_note('/pyzeppelin/test/note_2')
# with self.assertRaises(Exception) as context:
# client.create_note('/pyzeppelin/test/note_2')
# self.assertTrue('existed' in str(context.exception))
# finally:
# if note2_id:
# client.delete_note(note2_id)
note_id = client.create_note('/pyzeppelin/test/note_1')
client.add_paragraph(note_id, 'shell example', "%sh echo 'hello world'")
cloned_note_id = client.clone_note(note_id, '/pyzeppelin/test/cloned_note_1')
cloned_note = client.query_note_result(cloned_note_id)
self.assertEqual(1, len(cloned_note.paragraphs))
def test_zsession_sh_execute(self):
client_config = ClientConfig("http://localhost:8080")
session = ZSession(client_config, 'sh')
session.login('user1', 'password2')
try:
session.start()
self.assertIsNotNone(session.session_info.session_id)
result = session.execute("echo 'hello world'")
self.assertEqual('FINISHED', result.status)
self.assertEqual(1, len(result.results))
self.assertEqual('TEXT', result.results[0][0])
self.assertEqual('hello world\n', result.results[0][1])
result = session.execute("invalid_command")
self.assertEqual('ERROR', result.status)
finally:
session.stop()
def test_submit_note(self):
client_config = ClientConfig("http://localhost:8080")
client = ZeppelinClient(client_config)
client.login('user1', 'password2')
with self.assertRaises(Exception) as context:
client.submit_note('invalid_note_id')
self.assertTrue('No such note' in str(context.exception))
note_id = None
try:
note_id = client.create_note('/pyzeppelin/test/note_1')
paragraph_id = client.add_paragraph(note_id, 'shell example', "%sh echo 'hello world'")
note_result = client.submit_note(note_id)
note_result = client.wait_until_note_finished(note_id)
self.assertEqual(False, note_result.is_running)
self.assertEqual(1, len(note_result.paragraphs))
self.assertEqual('TEXT', note_result.paragraphs[0].results[0][0])
self.assertEqual('hello world\n', note_result.paragraphs[0].results[0][1])
# # dynamic forms
paragraph_id = client.update_paragraph(note_id, paragraph_id, "dynamic form example", "%sh echo 'hello ${name=abc}'")
note_result = client.submit_note(note_id)
note_result = client.wait_until_note_finished(note_id)
self.assertEqual(False, note_result.is_running)
self.assertEqual(1, len(note_result.paragraphs))
self.assertEqual('TEXT', note_result.paragraphs[0].results[0][0])
self.assertEqual('hello abc\n', note_result.paragraphs[0].results[0][1])
# run paragraph with parameters
note_result = client.submit_note(note_id, params = {'name' : 'zeppelin'})
note_result = client.wait_until_note_finished(note_id)
self.assertEqual(False, note_result.is_running)
self.assertEqual(1, len(note_result.paragraphs))
self.assertEqual('TEXT', note_result.paragraphs[0].results[0][0])
self.assertEqual('hello zeppelin\n', note_result.paragraphs[0].results[0][1])
finally:
if note_id:
client.delete_note(note_id)
self._check_response(resp)
return SessionInfo(resp.json()['body'])
def next_session_paragraph(self, note_id, max_statement):
resp = self.session.post(self.zeppelin_rest_url + "/api/notebook/" +
note_id + "/paragraph/next",
params={'maxParagraph': max_statement})
self._check_response(resp)
return resp.json()['message']
if __name__ == "__main__":
client_config = ClientConfig(
"https://knox.c-9181957fabf52f7e.cn-hangzhou.databricks.aliyuncs.com:8443/gateway/cluster-topo/zeppelin/"
)
client = ZeppelinClient(client_config)
client.login(
"zongze_ram",
"1234qwer",
knox_sso=
"https://knox.c-9181957fabf52f7e.cn-hangzhou.databricks.aliyuncs.com:8443/gateway/knoxsso/api/v1/websso"
)
print('version:' + client.get_version())
note_id = None
try:
note_id = client.create_note('/test/note_18', 'spark')
note_result = client.query_note_result(note_id)
print(note_result)
return ExecuteResult(paragraph_result)
def cancel(self, statement_id):
self.zeppelin_client.cancel(self.session_info.note_id, statement_id)
def query_statement(self, statement_id):
paragraph_result = self.zeppelin_client.query_paragraph_result(
self.session_info.note_id, statement_id)
return ExecuteResult(paragraph_result)
def session_id(self):
if self.session_info:
return self.session_info.note_id
else:
return None
if __name__ == "__main__":
client_config = ClientConfig("http://localhost:8080")
session = ZSession(client_config, "python")
session.login("user1", "password2")
try:
session.start()
print("session_id:" + session.session_id())
result = session.execute("1/0")
print("execute_result: " + str(result))
finally:
session.stop()