def test_create_sql_hive_context_pyspark(self):
kind = constants.SESSION_KIND_PYSPARK
http_client = MagicMock()
self.post_responses = [DummyResponse(201, self.session_create_json),
DummyResponse(201, self.post_statement_json),
DummyResponse(201, self.post_statement_json)]
http_client.post.side_effect = self._next_response_post
self.get_responses = [DummyResponse(200, self.ready_sessions_json),
DummyResponse(200, self.running_statement_json),
DummyResponse(200, self.ready_statement_json),
DummyResponse(200, self.ready_sessions_json),
DummyResponse(200, self.ready_statement_json)]
http_client.get.side_effect = self._next_response_get
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session(kind=kind, http_client=http_client)
conf.load()
session.start()
session.create_sql_context()
assert call("/sessions/0/statements", [201], {"code": "from pyspark.sql import HiveContext\n"
"sqlContext = HiveContext(sc)"}) \
in http_client.post.call_args_list
def test_start_emits_start_end_failed_session_when_wait_for_idle_throws(self):
http_client = MagicMock()
http_client.post_session.return_value = self.session_create_json
http_client.get_session.return_value = self.ready_sessions_json
spark_events = MagicMock()
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
kind = constants.SESSION_KIND_SPARK
session = self._create_session(kind=kind, http_client=http_client, spark_events=spark_events)
session.create_sql_context = MagicMock()
session.wait_for_idle = MagicMock(side_effect=ValueError)
try:
session.start()
assert False
except ValueError:
pass
conf.load()
spark_events.emit_session_creation_start_event.assert_called_once_with(session.guid, kind)
spark_events.emit_session_creation_end_event.assert_called_once_with(session.guid, kind, session.id,
session.status, False, "ValueError", "")
def test_execute_failure_post_statement_emits_event():
spark_events = MagicMock()
kind = SESSION_KIND_SPARK
http_client = MagicMock()
http_client.post_statement.side_effect = KeyError('Something bad happened here')
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = tls.TestLivySession._create_session(kind=kind, http_client=http_client)
session.wait_for_idle = MagicMock()
conf.load()
session.start(create_sql_context=False)
session.wait_for_idle = MagicMock()
command = Command("command", spark_events=spark_events)
try:
result = command.execute(session)
assert False
except KeyError as e:
spark_events.emit_statement_execution_start_event._assert_called_once_with(session.guid, session.kind,
session.id, command.guid)
spark_events.emit_statement_execution_start_event._assert_called_once_with(session.guid, session.kind,
session.id, command.guid,
-1, False, "KeyError",
"Something bad happened here")
assert_equals(e, http_client.post_statement.side_effect)
def test_execute_failure_wait_for_session_emits_event():
spark_events = MagicMock()
kind = SESSION_KIND_SPARK
http_client = MagicMock()
http_client.post_session.return_value = tls.TestLivySession.session_create_json
http_client.post_statement.return_value = tls.TestLivySession.post_statement_json
http_client.get_session.return_value = tls.TestLivySession.ready_sessions_json
http_client.get_statement.return_value = tls.TestLivySession.ready_statement_json
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = tls.TestLivySession._create_session(kind=kind, http_client=http_client)
conf.load()
session.start(create_sql_context=False)
session.wait_for_idle = MagicMock(side_effect=ValueError("yo"))
command = Command("command", spark_events=spark_events)
try:
result = command.execute(session)
assert False
except ValueError as e:
spark_events.emit_statement_execution_start_event._assert_called_once_with(session.guid, session.kind,
session.id, command.guid)
spark_events.emit_statement_execution_start_event._assert_called_once_with(session.guid, session.kind,
session.id, command.guid,
-1, False, "ValueError", "yo")
assert_equals(e, session.wait_for_idle.side_effect)
def test_delete_session_emits_start_failed_end_when_delete_throws(self):
self.http_client.delete_session.side_effect = ValueError
self.http_client.get_session.return_value = self.ready_sessions_json
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session()
conf.load()
session.start(create_sql_context=False)
session.id = 0
end_id = session.id
end_status = constants.BUSY_SESSION_STATUS
session.status = end_status
try:
session.delete()
assert False
except ValueError:
pass
self.spark_events.emit_session_deletion_start_event.assert_called_once_with(
session.guid, session.kind, end_id, end_status)
self.spark_events.emit_session_deletion_end_event.assert_called_once_with(
session.guid, session.kind, end_id, end_status, False, "ValueError", "")
def test_execute_failure_wait_for_session_emits_event():
spark_events = MagicMock()
kind = SESSION_KIND_SPARK
http_client = MagicMock()
http_client.post_session.return_value = tls.TestLivySession.session_create_json
http_client.post_statement.return_value = tls.TestLivySession.post_statement_json
http_client.get_session.return_value = tls.TestLivySession.ready_sessions_json
http_client.get_statement.return_value = tls.TestLivySession.ready_statement_json
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = tls.TestLivySession._create_session(kind=kind,
http_client=http_client)
conf.load()
session.start(create_sql_context=False)
session.wait_for_idle = MagicMock(side_effect=ValueError("yo"))
command = Command("command", spark_events=spark_events)
try:
result = command.execute(session)
assert False
except ValueError as e:
spark_events.emit_statement_execution_start_event._assert_called_once_with(
session.guid, session.kind, session.id, command.guid)
spark_events.emit_statement_execution_start_event._assert_called_once_with(
session.guid, session.kind, session.id, command.guid, -1, False,
"ValueError", "yo")
assert_equals(e, session.wait_for_idle.side_effect)
def test_create_sql_context_spark(self):
kind = constants.SESSION_KIND_SPARK
http_client = MagicMock()
http_client.post_session.return_value = self.session_create_json
self.post_statement_responses = [
self.post_statement_json, self.post_statement_json
]
http_client.post_statement.side_effect = self._next_statement_response_post
self.get_session_responses = [
self.ready_sessions_json, self.ready_sessions_json
]
http_client.get_session.side_effect = self._next_session_response_get
self.get_statement_responses = [
self.running_statement_json, self.ready_statement_json,
self.ready_statement_json
]
http_client.get_statement.side_effect = self._next_statement_response_get
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session(kind=kind, http_client=http_client)
conf.load()
session.start()
assert call(0, {"code": "val sqlContext = new org.apache.spark.sql.hive.HiveContext(sc)"}) \
in http_client.post_statement.call_args_list
def test_execute():
spark_events = MagicMock()
kind = SESSION_KIND_SPARK
http_client = MagicMock()
http_client.post_session.return_value = tls.TestLivySession.session_create_json
http_client.post_statement.return_value = tls.TestLivySession.post_statement_json
http_client.get_session.return_value = tls.TestLivySession.ready_sessions_json
http_client.get_statement.return_value = tls.TestLivySession.ready_statement_json
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = tls.TestLivySession._create_session(kind=kind,
http_client=http_client)
conf.load()
session.start(create_sql_context=False)
command = Command("command", spark_events=spark_events)
result = command.execute(session)
http_client.post_statement.assert_called_with(0, {"code": command.code})
http_client.get_statement.assert_called_with(0, 0)
assert result[0]
assert_equals(tls.TestLivySession.pi_result, result[1])
spark_events.emit_statement_execution_start_event._assert_called_once_with(
session.guid, session.kind, session.id, command.guid)
spark_events.emit_statement_execution_end_event._assert_called_once_with(
session.guid, session.kind, session.id, command.guid, 0, True, "", "")
def test_delete_session_emits_start_failed_end_when_delete_throws(self):
http_client = MagicMock()
http_client.delete_session.side_effect = ValueError
http_client.get_session.return_value = self.ready_sessions_json
spark_events = MagicMock()
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session(http_client=http_client,
spark_events=spark_events)
conf.load()
session.start(create_sql_context=False)
session.id = 0
end_id = session.id
end_status = constants.BUSY_SESSION_STATUS
session.status = end_status
try:
session.delete()
assert False
except ValueError:
pass
spark_events.emit_session_deletion_start_event.assert_called_once_with(
session.guid, session.kind, end_id, end_status)
spark_events.emit_session_deletion_end_event.assert_called_once_with(
session.guid, session.kind, end_id, end_status, False,
"ValueError", "")
def test_start_emits_start_end_failed_session_when_wait_for_idle_throws(
self):
http_client = MagicMock()
http_client.post_session.return_value = self.session_create_json
http_client.get_session.return_value = self.ready_sessions_json
spark_events = MagicMock()
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
kind = constants.SESSION_KIND_SPARK
session = self._create_session(kind=kind,
http_client=http_client,
spark_events=spark_events)
session.create_sql_context = MagicMock()
session.wait_for_idle = MagicMock(side_effect=ValueError)
try:
session.start()
assert False
except ValueError:
pass
conf.load()
spark_events.emit_session_creation_start_event.assert_called_once_with(
session.guid, kind)
spark_events.emit_session_creation_end_event.assert_called_once_with(
session.guid, kind, session.id, session.status, False,
"ValueError", "")
def test_delete_session_emits_start_failed_end_when_in_bad_state(self):
http_client = MagicMock()
http_client.get_session.return_value = self.ready_sessions_json
spark_events = MagicMock()
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session(http_client=http_client,
spark_events=spark_events)
conf.load()
session.start(create_sql_context=False)
session.id = 0
end_id = session.id
end_status = constants.DEAD_SESSION_STATUS
session.status = end_status
session.delete()
assert_equals(1, session.ipython_display.send_error.call_count)
spark_events.emit_session_deletion_start_event.assert_called_once_with(
session.guid, session.kind, end_id, end_status)
spark_events.emit_session_deletion_end_event.assert_called_once_with(
session.guid, session.kind, session.id,
constants.DEAD_SESSION_STATUS, True, "", "")
def test_execute(self):
kind = constants.SESSION_KIND_SPARK
http_client = MagicMock()
self.post_responses = [
DummyResponse(201, self.session_create_json),
DummyResponse(201, self.post_statement_json)
]
http_client.post.side_effect = self._next_response_post
self.get_responses = [
DummyResponse(200, self.running_statement_json),
DummyResponse(200, self.ready_statement_json)
]
http_client.get.side_effect = self._next_response_get
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session(kind=kind, http_client=http_client)
conf.load()
session.start()
command = "command"
result = session.execute(command)
http_client.post.assert_called_with("/sessions/0/statements", [201],
{"code": command})
http_client.get.assert_called_with("/sessions/0/statements", [200])
assert_equals(2, http_client.get.call_count)
assert result[0]
assert_equals(self.pi_result, result[1])
def test_create_sql_hive_context_pyspark(self):
kind = constants.SESSION_KIND_PYSPARK
http_client = MagicMock()
self.post_responses = [
DummyResponse(201, self.session_create_json),
DummyResponse(201, self.post_statement_json),
DummyResponse(201, self.post_statement_json)
]
http_client.post.side_effect = self._next_response_post
self.get_responses = [
DummyResponse(200, self.ready_sessions_json),
DummyResponse(200, self.running_statement_json),
DummyResponse(200, self.ready_statement_json),
DummyResponse(200, self.ready_sessions_json),
DummyResponse(200, self.ready_statement_json)
]
http_client.get.side_effect = self._next_response_get
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session(kind=kind, http_client=http_client)
conf.load()
session.start()
session.create_sql_context()
assert call("/sessions/0/statements", [201], {"code": "from pyspark.sql import HiveContext\n"
"sqlContext = HiveContext(sc)"}) \
in http_client.post.call_args_list
def test_delete_session_emits_start_failed_end_when_in_bad_state(self):
http_client = MagicMock()
http_client.get_session.return_value = self.ready_sessions_json
spark_events = MagicMock()
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session(http_client=http_client, spark_events=spark_events)
conf.load()
session.start(create_sql_context=False)
session.id = 0
end_id = session.id
end_status = constants.DEAD_SESSION_STATUS
session.status = end_status
session.delete()
assert_equals(1, session.ipython_display.send_error.call_count)
spark_events.emit_session_deletion_start_event.assert_called_once_with(session.guid, session.kind, end_id,
end_status)
spark_events.emit_session_deletion_end_event.assert_called_once_with(session.guid, session.kind, session.id,
constants.DEAD_SESSION_STATUS, True, "",
"")
def test_execute_failure_get_statement_output_emits_event():
spark_events = MagicMock()
kind = SESSION_KIND_SPARK
http_client = MagicMock()
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = tls.TestLivySession._create_session(kind=kind, http_client=http_client)
session.wait_for_idle = MagicMock()
conf.load()
session.start(create_sql_context=False)
session.wait_for_idle = MagicMock()
command = Command("command", spark_events=spark_events)
command._get_statement_output = MagicMock(side_effect=AttributeError('OHHHH'))
try:
result = command.execute(session)
assert False
except AttributeError as e:
spark_events.emit_statement_execution_start_event._assert_called_once_with(session.guid, session.kind,
session.id, command.guid)
spark_events.emit_statement_execution_start_event._assert_called_once_with(session.guid, session.kind,
session.id, command.guid,
-1, False, "AttributeError",
"OHHHH")
assert_equals(e, command._get_statement_output.side_effect)
def test_create_sql_hive_context_happens_once(self):
kind = constants.SESSION_KIND_SPARK
http_client = MagicMock()
ipython_display = MagicMock()
http_client.post_session.return_value = self.session_create_json
self.post_statement_responses = [self.post_statement_json,
self.post_statement_json]
http_client.post_statement.side_effect = self._next_statement_response_post
http_client.get_session.return_value = self.ready_sessions_json
self.get_statement_responses = [self.running_statement_json,
self.ready_statement_json,
self.ready_statement_json]
http_client.get_statement.side_effect = self._next_statement_response_get
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session(kind=kind, http_client=http_client)
session.ipython_display = ipython_display
conf.load()
session.start(create_sql_context=False)
# Reset the mock so that post called count is accurate
http_client.reset_mock()
session.create_sql_context()
assert ipython_display.writeln.call_count == 2
# Second call should not issue a post request
session.create_sql_context()
assert call(0, {"code": "val sqlContext = new org.apache.spark.sql.hive.HiveContext(sc)"}) \
in http_client.post_statement.call_args_list
assert len(http_client.post_statement.call_args_list) == 1
def test_execute():
spark_events = MagicMock()
kind = SESSION_KIND_SPARK
http_client = MagicMock()
http_client.post_session.return_value = tls.TestLivySession.session_create_json
http_client.post_statement.return_value = tls.TestLivySession.post_statement_json
http_client.get_session.return_value = tls.TestLivySession.ready_sessions_json
http_client.get_statement.return_value = tls.TestLivySession.ready_statement_json
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = tls.TestLivySession._create_session(kind=kind, http_client=http_client)
conf.load()
session.start(create_sql_context=False)
command = Command("command", spark_events=spark_events)
result = command.execute(session)
http_client.post_statement.assert_called_with(0, {"code": command.code})
http_client.get_statement.assert_called_with(0, 0)
assert result[0]
assert_equals(tls.TestLivySession.pi_result, result[1])
spark_events.emit_statement_execution_start_event._assert_called_once_with(session.guid, session.kind,
session.id, command.guid)
spark_events.emit_statement_execution_end_event._assert_called_once_with(session.guid, session.kind,
session.id, command.guid,
0, True, "", "")
def test_execute_failure_post_statement_emits_event():
spark_events = MagicMock()
kind = SESSION_KIND_SPARK
http_client = MagicMock()
http_client.post_statement.side_effect = KeyError(
'Something bad happened here')
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = tls.TestLivySession._create_session(kind=kind,
http_client=http_client)
session.wait_for_idle = MagicMock()
conf.load()
session.start(create_sql_context=False)
session.wait_for_idle = MagicMock()
command = Command("command", spark_events=spark_events)
try:
result = command.execute(session)
assert False
except KeyError as e:
spark_events.emit_statement_execution_start_event._assert_called_once_with(
session.guid, session.kind, session.id, command.guid)
spark_events.emit_statement_execution_start_event._assert_called_once_with(
session.guid, session.kind, session.id, command.guid, -1, False,
"KeyError", "Something bad happened here")
assert_equals(e, http_client.post_statement.side_effect)
def test_execute(self):
kind = Constants.session_kind_spark
http_client = MagicMock()
self.post_responses = [
DummyResponse(201, self.session_create_json),
DummyResponse(201, self.post_statement_json),
]
http_client.post.side_effect = self._next_response_post
self.get_responses = [
DummyResponse(200, self.running_statement_json),
DummyResponse(200, self.ready_statement_json),
]
http_client.get.side_effect = self._next_response_get
conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01})
session = self._create_session(kind=kind, http_client=http_client)
conf.load()
session.start()
command = "command"
result = session.execute(command)
http_client.post.assert_called_with("/sessions/0/statements", [201], {"code": command})
http_client.get.assert_called_with("/sessions/0/statements", [200])
assert_equals(2, http_client.get.call_count)
assert result[0]
assert_equals(self.pi_result, result[1])
def test_execute_failure_get_statement_output_emits_event():
spark_events = MagicMock()
kind = SESSION_KIND_SPARK
http_client = MagicMock()
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = tls.TestLivySession._create_session(kind=kind,
http_client=http_client)
session.wait_for_idle = MagicMock()
conf.load()
session.start(create_sql_context=False)
session.wait_for_idle = MagicMock()
command = Command("command", spark_events=spark_events)
command._get_statement_output = MagicMock(
side_effect=AttributeError('OHHHH'))
try:
result = command.execute(session)
assert False
except AttributeError as e:
spark_events.emit_statement_execution_start_event._assert_called_once_with(
session.guid, session.kind, session.id, command.guid)
spark_events.emit_statement_execution_start_event._assert_called_once_with(
session.guid, session.kind, session.id, command.guid, -1, False,
"AttributeError", "OHHHH")
assert_equals(e, command._get_statement_output.side_effect)
def test_constructor_throws_statement_sleep_seconds(self):
conf.override_all({
"status_sleep_seconds": 3,
"statement_sleep_seconds": 0,
"create_sql_context_timeout_seconds": 60
})
self._create_session()
conf.load()
def test_constructor_throws_invalid_session_sql_combo(self):
conf.override_all({
"status_sleep_seconds": 2,
"statement_sleep_seconds": 2,
"create_sql_context_timeout_seconds": 60
})
self._create_session(sql_created=True)
conf.load()
def test_constructor_throws_invalid_session_sql_combo(self):
conf.override_all({
"status_sleep_seconds": 2,
"statement_sleep_seconds": 2,
"create_sql_context_timeout_seconds": 60
})
self._create_session(sql_created=True)
conf.load()
def test_constructor_throws_sql_create_timeout_seconds(self):
conf.override_all({
"status_sleep_seconds": 4,
"statement_sleep_seconds": 2,
"create_sql_context_timeout_seconds": 0
})
self._create_session()
conf.load()
def test_constructor_starts_with_no_session(self):
conf.override_all(
{"status_sleep_seconds": 4, "statement_sleep_seconds": 2, "create_sql_context_timeout_seconds": 60}
)
session = self._create_session()
conf.load()
assert session.id == "-1"
assert not session.started_sql_context
def test_delete_session_when_dead_throws(self):
http_client = MagicMock()
http_client.post.return_value = DummyResponse(201, self.session_create_json)
conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01})
session = self._create_session(http_client=http_client)
conf.load()
session._status = "dead"
session.delete()
def test_constructor_starts_with_existing_session(self):
conf.override_all(
{"status_sleep_seconds": 4, "statement_sleep_seconds": 2, "create_sql_context_timeout_seconds": 60}
)
session_id = "1"
session = self._create_session(session_id=session_id, sql_created=True)
conf.load()
assert session.id == session_id
assert session.started_sql_context
def test_delete_session_when_not_started(self):
http_client = MagicMock()
http_client.post.return_value = DummyResponse(201, self.session_create_json)
conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01})
session = self._create_session(http_client=http_client)
conf.load()
session.delete()
assert_equals("dead", session._status)
assert_equals("-1", session.id)
def test_is_final_status(self):
conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01})
session = self._create_session()
conf.load()
assert not session.is_final_status("idle")
assert not session.is_final_status("starting")
assert not session.is_final_status("busy")
assert session.is_final_status("dead")
assert session.is_final_status("error")
def test_serialize_periodically():
conf.override_all({conf.serialize_period_seconds.__name__: 0.1,
conf.serialize_periodically.__name__: True})
serializer = MagicMock()
ClientManager(serializer)
time.sleep(0.5)
assert serializer.serialize_state.call_count >= 1
conf.load()
def test_constructor_starts_with_no_session(self):
conf.override_all({
"status_sleep_seconds": 4,
"statement_sleep_seconds": 2,
"create_sql_context_timeout_seconds": 60
})
session = self._create_session()
conf.load()
assert session.id == "-1"
assert not session.started_sql_context
def test_delete_session_when_not_started(self):
self.http_client.post_session.return_value = self.session_create_json
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session()
conf.load()
session.delete()
assert_equals(session.ipython_display.send_error.call_count, 1)
def test_constructor_starts_with_existing_session(self):
conf.override_all({
"status_sleep_seconds": 4,
"statement_sleep_seconds": 2,
"create_sql_context_timeout_seconds": 60
})
session_id = "1"
session = self._create_session(session_id=session_id, sql_created=True)
conf.load()
assert session.id == session_id
assert session.started_sql_context
def test_configuration_load():
i = 1000
config = {conf.statement_sleep_seconds.__name__: i}
fsrw = MagicMock()
fsrw.path = ""
read_lines = MagicMock(return_value=[json.dumps(config)])
fsrw.read_lines = read_lines
fsrw_class = MagicMock(return_value=fsrw)
conf.load(fsrw_class)
assert conf._overrides is not None
assert_equals(conf._overrides, config)
assert_equals(conf.statement_sleep_seconds(), i)
def test_configuration_load():
i = 1000
config = { conf.statement_sleep_seconds.__name__: i }
fsrw = MagicMock()
fsrw.path = ""
read_lines = MagicMock(return_value=[json.dumps(config)])
fsrw.read_lines = read_lines
fsrw_class = MagicMock(return_value=fsrw)
conf.load(fsrw_class)
assert conf._overrides is not None
assert_equals(conf._overrides, config)
assert_equals(conf.statement_sleep_seconds(), i)
def test_configuration_load_not_lazy():
a = "whoops"
config = {conf.default_chart_type.__name__: a}
fsrw = MagicMock()
fsrw.path = ""
read_lines = MagicMock(return_value=[json.dumps(config)])
fsrw.read_lines = read_lines
fsrw_class = MagicMock(return_value=fsrw)
conf.override_all({conf.default_chart_type.__name__: "bar"})
conf.load(fsrw_class)
assert conf._overrides is not None
assert_equals(conf._overrides, config)
assert_equals(conf.default_chart_type(), "whoops")
def test_configuration_load_not_lazy():
a = "whoops"
config = { conf.events_handler_class.__name__: a }
fsrw = MagicMock()
fsrw.path = ""
read_lines = MagicMock(return_value=[json.dumps(config)])
fsrw.read_lines = read_lines
fsrw_class = MagicMock(return_value=fsrw)
conf.override_all({conf.events_handler_class.__name__: "bar"})
conf.load(fsrw_class)
assert conf._overrides is not None
assert_equals(conf._overrides, config)
assert_equals(conf.events_handler_class(), a)
def test_delete_session_when_not_started(self):
http_client = MagicMock()
http_client.post_session.return_value = self.session_create_json
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session(http_client=http_client)
conf.load()
session.delete()
assert_equals(session.ipython_display.send_error.call_count, 1)
def test_logs_gets_latest_logs(self):
http_client = MagicMock()
http_client.post.return_value = DummyResponse(201, self.session_create_json)
http_client.get.return_value = DummyResponse(200, self.log_json)
conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01})
session = self._create_session(http_client=http_client)
conf.load()
session.start()
logs = session.logs
assert_equals("hi\nhi", logs)
http_client.get.assert_called_with("/sessions/0/log?from=0", [200])
def test_delete_session_when_dead_throws(self):
http_client = MagicMock()
http_client.post.return_value = DummyResponse(201,
self.session_create_json)
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session(http_client=http_client)
conf.load()
session.status = "dead"
session.delete()
def test_create_sql_hive_context_happens_once(self):
kind = Constants.session_kind_spark
http_client = MagicMock()
ipython_display = MagicMock()
self.post_responses = [
DummyResponse(201, self.session_create_json),
DummyResponse(201, self.post_statement_json),
DummyResponse(201, self.post_statement_json),
]
http_client.post.side_effect = self._next_response_post
self.get_responses = [
DummyResponse(200, self.ready_sessions_json),
DummyResponse(200, self.running_statement_json),
DummyResponse(200, self.ready_statement_json),
DummyResponse(200, self.ready_sessions_json),
DummyResponse(200, self.ready_statement_json),
]
http_client.get.side_effect = self._next_response_get
conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01})
session = self._create_session(kind=kind, http_client=http_client)
session.ipython_display = ipython_display
conf.load()
session.start()
# Reset the mock so that post called count is accurate
http_client.reset_mock()
session.create_sql_context()
assert ipython_display.writeln.call_count == 3
# Second call should not issue a post request
session.create_sql_context()
assert (
call(
"/sessions/0/statements",
[201],
{"code": "val sqlContext = new org.apache.spark.sql.SQLContext" "(sc)\nimport sqlContext.implicits._"},
)
in http_client.post.call_args_list
)
assert (
call(
"/sessions/0/statements",
[201],
{"code": "val hiveContext = new org.apache.spark.sql.hive.Hive" "Context(sc)"},
)
in http_client.post.call_args_list
)
assert len(http_client.post.call_args_list) == 2
def test_start_passes_in_all_properties(self):
http_client = MagicMock()
http_client.post.return_value = DummyResponse(201, self.session_create_json)
conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01})
kind = Constants.session_kind_spark
properties = {"kind": kind, "extra": 1}
ipython_display = MagicMock()
session = LivySession(ipython_display, http_client, "-1", False, properties)
session.start()
conf.load()
http_client.post.assert_called_with("/sessions", [201], properties)
def test_status_gets_latest_status(self):
http_client = MagicMock()
http_client.post.return_value = DummyResponse(201, self.session_create_json)
http_client.get.return_value = DummyResponse(200, self.ready_sessions_json)
conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01})
session = self._create_session(http_client=http_client)
conf.load()
session.start()
session._refresh_status()
state = session._status
assert_equals("idle", state)
http_client.get.assert_called_with("/sessions/0", [200])
def test_delete_session_when_active(self):
self.http_client.post_session.return_value = self.session_create_json
self.http_client.get_session.return_value = self.ready_sessions_json
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session()
conf.load()
session.start(create_sql_context=False)
session.delete()
assert_equals("dead", session.status)
def test_delete_session_when_dead_throws(self):
http_client = MagicMock()
http_client.post.return_value = self.session_create_json
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session(http_client=http_client)
conf.load()
session.status = "dead"
session.delete()
assert_equals(session.ipython_display.send_error.call_count, 1)
def test_start_python_starts_session(self):
http_client = MagicMock()
http_client.post.return_value = DummyResponse(201, self.session_create_json)
conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01})
kind = Constants.session_kind_pyspark
session = self._create_session(kind=kind, http_client=http_client)
session.start()
conf.load()
assert_equals(kind, session.kind)
assert_equals("starting", session._status)
assert_equals("0", session.id)
http_client.post.assert_called_with("/sessions", [201], {"kind": "pyspark"})
def test_create_sql_hive_context_unknown_throws(self):
kind = "unknown"
self.http_client.post_session.return_value = self.session_create_json
self.http_client.post_statement.return_value = self.post_statement_json
self.http_client.get_session.return_value = self.ready_sessions_json
self.get_statement_responses = [self.running_statement_json, self.ready_statement_json]
self.http_client.get_statement.side_effect = self._next_statement_response_get
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session(kind=kind)
conf.load()
session.start()
def test_is_final_status(self):
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session()
conf.load()
assert not session.is_final_status("idle")
assert not session.is_final_status("starting")
assert not session.is_final_status("busy")
assert session.is_final_status("dead")
assert session.is_final_status("error")
def test_get_config():
usr = "******"
pwd = "p"
url = "url"
config = {conf.kernel_python_credentials.__name__: {user_ev: usr, pass_ev: pwd, url_ev: url}}
conf.override_all(config)
u, p, r = kernel._get_configuration()
assert u == usr
assert p == pwd
assert r == url
conf.load()
def test_delete_session_when_active(self):
http_client = MagicMock()
http_client.post_session.return_value = self.session_create_json
http_client.get_session.return_value = self.ready_sessions_json
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session(http_client=http_client)
conf.load()
session.start(create_sql_context=False)
session.delete()
assert_equals("dead", session.status)
def test_delete_session_when_not_started(self):
http_client = MagicMock()
http_client.post.return_value = DummyResponse(201,
self.session_create_json)
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session(http_client=http_client)
conf.load()
session.delete()
assert_equals("dead", session.status)
assert_equals("-1", session.id)
def test_status_gets_latest_status(self):
http_client = MagicMock()
http_client.post_session.return_value = self.session_create_json
http_client.get_session.return_value = self.ready_sessions_json
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session(http_client=http_client)
conf.load()
session.start(create_sql_context=False)
session.refresh_status()
state = session.status
assert_equals("idle", state)
http_client.get_session.assert_called_with(0)
def test_logs_gets_latest_logs(self):
http_client = MagicMock()
http_client.post_session.return_value = self.session_create_json
http_client.get_session.return_value = self.ready_sessions_json
http_client.get_all_session_logs.return_value = self.log_json
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session(http_client=http_client)
conf.load()
session.start(create_sql_context=False)
logs = session.get_logs()
assert_equals("hi\nhi", logs)
http_client.get_all_session_logs.assert_called_with(0)
def test_logs_gets_latest_logs(self):
http_client = MagicMock()
http_client.post.return_value = DummyResponse(201,
self.session_create_json)
http_client.get.return_value = DummyResponse(200, self.log_json)
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session(http_client=http_client)
conf.load()
session.start()
logs = session.get_logs()
assert_equals("hi\nhi", logs)
http_client.get.assert_called_with("/sessions/0/log?from=0", [200])
def test_create_sql_hive_context_happens_once(self):
kind = constants.SESSION_KIND_SPARK
http_client = MagicMock()
ipython_display = MagicMock()
http_client.post_session.return_value = self.session_create_json
self.post_statement_responses = [
self.post_statement_json, self.post_statement_json
]
http_client.post_statement.side_effect = self._next_statement_response_post
http_client.get_session.return_value = self.ready_sessions_json
self.get_statement_responses = [
self.running_statement_json, self.ready_statement_json,
self.ready_statement_json
]
http_client.get_statement.side_effect = self._next_statement_response_get
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session(kind=kind, http_client=http_client)
session.ipython_display = ipython_display
conf.load()
session.start(create_sql_context=False)
# Reset the mock so that post called count is accurate
http_client.reset_mock()
session.create_sql_context()
assert ipython_display.writeln.call_count == 2
assert session.created_sql_context
# Second call should not issue a post request
session.create_sql_context()
assert call(0, {"code": "val sqlContext = new org.apache.spark.sql.hive.HiveContext(sc)"}) \
in http_client.post_statement.call_args_list
assert len(http_client.post_statement.call_args_list) == 1
session._get_sql_context_creation_command = MagicMock()
session._get_sql_context_creation_command.return_value.execute.return_value = (
True, "Success")
session.created_sql_context = None
session.create_sql_context()
assert session.created_sql_context
def test_create_sql_hive_context_unknown_throws(self):
kind = "unknown"
http_client = MagicMock()
http_client.post_session.return_value = self.session_create_json
http_client.post_statement.return_value = self.post_statement_json
http_client.get_session.return_value = self.ready_sessions_json
self.get_statement_responses = [
self.running_statement_json, self.ready_statement_json
]
http_client.get_statement.side_effect = self._next_statement_response_get
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session(kind=kind, http_client=http_client)
conf.load()
session.start()
def test_start_passes_in_all_properties(self):
http_client = MagicMock()
http_client.post_session.return_value = self.session_create_json
http_client.get_session.return_value = self.ready_sessions_json
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
kind = constants.SESSION_KIND_SPARK
properties = {"kind": kind, "extra": 1}
ipython_display = MagicMock()
session = LivySession(http_client, properties, ipython_display)
session.start(create_sql_context=False)
conf.load()
http_client.post_session.assert_called_with(properties)
def test_start_passes_in_all_properties(self):
http_client = MagicMock()
http_client.post.return_value = DummyResponse(201,
self.session_create_json)
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
kind = constants.SESSION_KIND_SPARK
properties = {"kind": kind, "extra": 1}
ipython_display = MagicMock()
session = LivySession(http_client, properties, ipython_display)
session.start()
conf.load()
http_client.post.assert_called_with("/sessions", [201], properties)
def test_create_sql_hive_context_happens_once(self):
kind = constants.SESSION_KIND_SPARK
http_client = MagicMock()
ipython_display = MagicMock()
self.post_responses = [
DummyResponse(201, self.session_create_json),
DummyResponse(201, self.post_statement_json),
DummyResponse(201, self.post_statement_json)
]
http_client.post.side_effect = self._next_response_post
self.get_responses = [
DummyResponse(200, self.ready_sessions_json),
DummyResponse(200, self.running_statement_json),
DummyResponse(200, self.ready_statement_json),
DummyResponse(200, self.ready_sessions_json),
DummyResponse(200, self.ready_statement_json)
]
http_client.get.side_effect = self._next_response_get
conf.override_all({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = self._create_session(kind=kind, http_client=http_client)
session.ipython_display = ipython_display
conf.load()
session.start()
# Reset the mock so that post called count is accurate
http_client.reset_mock()
session.create_sql_context()
assert ipython_display.writeln.call_count == 2
# Second call should not issue a post request
session.create_sql_context()
assert call(
"/sessions/0/statements", [201], {
"code":
"val sqlContext = new org.apache.spark.sql.hive.Hive"
"Context(sc)"
}) in http_client.post.call_args_list
assert len(http_client.post.call_args_list) == 1