def test_create_sql_hive_context_pyspark(self):
kind = "python"
http_client = MagicMock()
self.post_responses = [DummyResponse(201, self.session_create_json),
DummyResponse(201, self.post_statement_json),
DummyResponse(201, self.post_statement_json)]
http_client.post.side_effect = self._next_response_post
self.get_responses = [DummyResponse(200, self.ready_sessions_json),
DummyResponse(200, self.running_statement_json),
DummyResponse(200, self.ready_statement_json),
DummyResponse(200, self.ready_sessions_json),
DummyResponse(200, self.ready_statement_json)]
http_client.get.side_effect = self._next_response_get
_t_config_hook({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = LivySession(http_client, kind, "-1", False)
_t_config_hook({})
session.start()
session.create_sql_context()
assert call("/sessions/0/statements", [201], {"code": "from pyspark.sql import SQLContext\n"
"from pyspark.sql.types import *\n"
"sqlContext = SQLContext(sc)"}) \
in http_client.post.call_args_list
assert call("/sessions/0/statements", [201], {"code": "from pyspark.sql import HiveContext\n"
"hiveContext = HiveContext(sc)"}) \
in http_client.post.call_args_list
def test_create_sql_context_spark(self):
kind = "scala"
http_client = MagicMock()
self.post_responses = [DummyResponse(201, self.session_create_json),
DummyResponse(201, self.post_statement_json),
DummyResponse(201, self.post_statement_json)]
http_client.post.side_effect = self._next_response_post
self.get_responses = [DummyResponse(200, self.ready_sessions_json),
DummyResponse(200, self.running_statement_json),
DummyResponse(200, self.ready_statement_json),
DummyResponse(200, self.ready_sessions_json),
DummyResponse(200, self.ready_statement_json)]
http_client.get.side_effect = self._next_response_get
_t_config_hook({
"status_sleep_seconds": 0.01,
"statement_sleep_seconds": 0.01
})
session = LivySession(http_client, kind, "-1", False)
_t_config_hook({})
session.start()
session.create_sql_context()
assert call("/sessions/0/statements", [201], {"code": "val sqlContext = new org.apache.spark.sql.SQLContext"
"(sc)\nimport sqlContext.implicits._"}) \
in http_client.post.call_args_list
assert call("/sessions/0/statements", [201], {"code": "val hiveContext = new org.apache.spark.sql.hive.Hive"
"Context(sc)"}) \
in http_client.post.call_args_list
def test_create_sql_context_unknown_throws(self):
kind = "unknown"
http_client = MagicMock()
self.post_responses = [DummyResponse(201, self.session_create_json),
DummyResponse(201, self.post_statement_json)]
http_client.post.side_effect = self._next_response_post
self.get_responses = [DummyResponse(200, self.ready_sessions_json),
DummyResponse(200, self.running_statement_json),
DummyResponse(200, self.ready_statement_json)]
http_client.get.side_effect = self._next_response_get
session = LivySession(http_client, kind, state_sleep_seconds=0.01, statement_sleep_seconds=0.01)
session.start()
session.create_sql_context()
def test_create_sql_context_spark(self):
kind = "scala"
http_client = MagicMock()
self.post_responses = [DummyResponse(201, self.session_create_json),
DummyResponse(201, self.post_statement_json)]
http_client.post.side_effect = self._next_response_post
self.get_responses = [DummyResponse(200, self.ready_sessions_json),
DummyResponse(200, self.running_statement_json),
DummyResponse(200, self.ready_statement_json)]
http_client.get.side_effect = self._next_response_get
session = LivySession(http_client, kind, state_sleep_seconds=0.01, statement_sleep_seconds=0.01)
session.start()
session.create_sql_context()
http_client.post.assert_called_with("/sessions/0/statements", [201],
{"code": "val sqlContext = new org.apache.spark.sql.SQLContext(sc)\n"
"import sqlContext.implicits._"})
def test_create_sql_context_pyspark(self):
kind = "python"
http_client = MagicMock()
self.post_responses = [DummyResponse(201, self.session_create_json),
DummyResponse(201, self.post_statement_json)]
http_client.post.side_effect = self._next_response_post
self.get_responses = [DummyResponse(200, self.ready_sessions_json),
DummyResponse(200, self.running_statement_json),
DummyResponse(200, self.ready_statement_json)]
http_client.get.side_effect = self._next_response_get
session = LivySession(http_client, kind, state_sleep_seconds=0.01, statement_sleep_seconds=0.01)
session.start()
session.create_sql_context()
http_client.post.assert_called_with("/sessions/0/statements", [201],
{"code": "from pyspark.sql import SQLContext\n"
"from pyspark.sql.types import *\n"
"sqlContext = SQLContext(sc)"})
def test_create_sql_context_happens_once(self):
kind = "scala"
http_client = MagicMock()
self.post_responses = [DummyResponse(201, self.session_create_json),
DummyResponse(201, self.post_statement_json)]
http_client.post.side_effect = self._next_response_post
self.get_responses = [DummyResponse(200, self.ready_sessions_json),
DummyResponse(200, self.running_statement_json),
DummyResponse(200, self.ready_statement_json)]
http_client.get.side_effect = self._next_response_get
session = LivySession(http_client, kind, state_sleep_seconds=0.01, statement_sleep_seconds=0.01)
session.start()
# Reset the mock so that post called count is accurate
http_client.reset_mock()
session.create_sql_context()
# Second call should not issue a post request
session.create_sql_context()
http_client.post.assert_called_once_with("/sessions/0/statements", [201],
{"code": "val sqlContext = new org.apache.spark.sql.SQLContext(sc)\n"
"import sqlContext.implicits._"})
