def test_create_sql_hive_context_happens_once(self): kind = constants.SESSION_KIND_SPARK http_client = MagicMock() ipython_display = MagicMock() http_client.post_session.return_value = self.session_create_json self.post_statement_responses = [self.post_statement_json, self.post_statement_json] http_client.post_statement.side_effect = self._next_statement_response_post http_client.get_session.return_value = self.ready_sessions_json self.get_statement_responses = [self.running_statement_json, self.ready_statement_json, self.ready_statement_json] http_client.get_statement.side_effect = self._next_statement_response_get conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = self._create_session(kind=kind, http_client=http_client) session.ipython_display = ipython_display conf.load() session.start(create_sql_context=False) # Reset the mock so that post called count is accurate http_client.reset_mock() session.create_sql_context() assert ipython_display.writeln.call_count == 2 # Second call should not issue a post request session.create_sql_context() assert call(0, {"code": "val sqlContext = new org.apache.spark.sql.hive.HiveContext(sc)"}) \ in http_client.post_statement.call_args_list assert len(http_client.post_statement.call_args_list) == 1
def test_create_sql_hive_context_pyspark(self): kind = constants.SESSION_KIND_PYSPARK http_client = MagicMock() self.post_responses = [DummyResponse(201, self.session_create_json), DummyResponse(201, self.post_statement_json), DummyResponse(201, self.post_statement_json)] http_client.post.side_effect = self._next_response_post self.get_responses = [DummyResponse(200, self.ready_sessions_json), DummyResponse(200, self.running_statement_json), DummyResponse(200, self.ready_statement_json), DummyResponse(200, self.ready_sessions_json), DummyResponse(200, self.ready_statement_json)] http_client.get.side_effect = self._next_response_get conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = self._create_session(kind=kind, http_client=http_client) conf.load() session.start() session.create_sql_context() assert call("/sessions/0/statements", [201], {"code": "from pyspark.sql import HiveContext\n" "sqlContext = HiveContext(sc)"}) \ in http_client.post.call_args_list
def test_create_sql_context_spark(self): kind = constants.SESSION_KIND_SPARK http_client = MagicMock() http_client.post_session.return_value = self.session_create_json self.post_statement_responses = [ self.post_statement_json, self.post_statement_json ] http_client.post_statement.side_effect = self._next_statement_response_post self.get_session_responses = [ self.ready_sessions_json, self.ready_sessions_json ] http_client.get_session.side_effect = self._next_session_response_get self.get_statement_responses = [ self.running_statement_json, self.ready_statement_json, self.ready_statement_json ] http_client.get_statement.side_effect = self._next_statement_response_get conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = self._create_session(kind=kind, http_client=http_client) conf.load() session.start() assert call(0, {"code": "val sqlContext = new org.apache.spark.sql.hive.HiveContext(sc)"}) \ in http_client.post_statement.call_args_list
def test_execute_failure_wait_for_session_emits_event(): spark_events = MagicMock() kind = SESSION_KIND_SPARK http_client = MagicMock() http_client.post_session.return_value = tls.TestLivySession.session_create_json http_client.post_statement.return_value = tls.TestLivySession.post_statement_json http_client.get_session.return_value = tls.TestLivySession.ready_sessions_json http_client.get_statement.return_value = tls.TestLivySession.ready_statement_json conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = tls.TestLivySession._create_session(kind=kind, http_client=http_client) conf.load() session.start(create_sql_context=False) session.wait_for_idle = MagicMock(side_effect=ValueError("yo")) command = Command("command", spark_events=spark_events) try: result = command.execute(session) assert False except ValueError as e: spark_events.emit_statement_execution_start_event._assert_called_once_with(session.guid, session.kind, session.id, command.guid) spark_events.emit_statement_execution_start_event._assert_called_once_with(session.guid, session.kind, session.id, command.guid, -1, False, "ValueError", "yo") assert_equals(e, session.wait_for_idle.side_effect)
def test_execute(self): kind = Constants.session_kind_spark http_client = MagicMock() self.post_responses = [ DummyResponse(201, self.session_create_json), DummyResponse(201, self.post_statement_json), ] http_client.post.side_effect = self._next_response_post self.get_responses = [ DummyResponse(200, self.running_statement_json), DummyResponse(200, self.ready_statement_json), ] http_client.get.side_effect = self._next_response_get conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01}) session = self._create_session(kind=kind, http_client=http_client) conf.load() session.start() command = "command" result = session.execute(command) http_client.post.assert_called_with("/sessions/0/statements", [201], {"code": command}) http_client.get.assert_called_with("/sessions/0/statements", [200]) assert_equals(2, http_client.get.call_count) assert result[0] assert_equals(self.pi_result, result[1])
def test_delete_session_emits_start_failed_end_when_delete_throws(self): http_client = MagicMock() http_client.delete_session.side_effect = ValueError http_client.get_session.return_value = self.ready_sessions_json spark_events = MagicMock() conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = self._create_session(http_client=http_client, spark_events=spark_events) conf.load() session.start(create_sql_context=False) session.id = 0 end_id = session.id end_status = constants.BUSY_SESSION_STATUS session.status = end_status try: session.delete() assert False except ValueError: pass spark_events.emit_session_deletion_start_event.assert_called_once_with( session.guid, session.kind, end_id, end_status) spark_events.emit_session_deletion_end_event.assert_called_once_with( session.guid, session.kind, end_id, end_status, False, "ValueError", "")
def test_execute_failure_post_statement_emits_event(): spark_events = MagicMock() kind = SESSION_KIND_SPARK http_client = MagicMock() http_client.post_statement.side_effect = KeyError( 'Something bad happened here') conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = tls.TestLivySession._create_session(kind=kind, http_client=http_client) session.wait_for_idle = MagicMock() conf.load() session.start(create_sql_context=False) session.wait_for_idle = MagicMock() command = Command("command", spark_events=spark_events) try: result = command.execute(session) assert False except KeyError as e: spark_events.emit_statement_execution_start_event._assert_called_once_with( session.guid, session.kind, session.id, command.guid) spark_events.emit_statement_execution_start_event._assert_called_once_with( session.guid, session.kind, session.id, command.guid, -1, False, "KeyError", "Something bad happened here") assert_equals(e, http_client.post_statement.side_effect)
def test_execute_failure_get_statement_output_emits_event(): spark_events = MagicMock() kind = SESSION_KIND_SPARK http_client = MagicMock() conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = tls.TestLivySession._create_session(kind=kind, http_client=http_client) session.wait_for_idle = MagicMock() conf.load() session.start(create_sql_context=False) session.wait_for_idle = MagicMock() command = Command("command", spark_events=spark_events) command._get_statement_output = MagicMock(side_effect=AttributeError('OHHHH')) try: result = command.execute(session) assert False except AttributeError as e: spark_events.emit_statement_execution_start_event._assert_called_once_with(session.guid, session.kind, session.id, command.guid) spark_events.emit_statement_execution_start_event._assert_called_once_with(session.guid, session.kind, session.id, command.guid, -1, False, "AttributeError", "OHHHH") assert_equals(e, command._get_statement_output.side_effect)
def test_execute_failure_get_statement_output_emits_event(): spark_events = MagicMock() kind = SESSION_KIND_SPARK http_client = MagicMock() conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = tls.TestLivySession._create_session(kind=kind, http_client=http_client) session.wait_for_idle = MagicMock() conf.load() session.start(create_sql_context=False) session.wait_for_idle = MagicMock() command = Command("command", spark_events=spark_events) command._get_statement_output = MagicMock( side_effect=AttributeError('OHHHH')) try: result = command.execute(session) assert False except AttributeError as e: spark_events.emit_statement_execution_start_event._assert_called_once_with( session.guid, session.kind, session.id, command.guid) spark_events.emit_statement_execution_start_event._assert_called_once_with( session.guid, session.kind, session.id, command.guid, -1, False, "AttributeError", "OHHHH") assert_equals(e, command._get_statement_output.side_effect)
def test_execute_failure_wait_for_session_emits_event(): spark_events = MagicMock() kind = SESSION_KIND_SPARK http_client = MagicMock() http_client.post_session.return_value = tls.TestLivySession.session_create_json http_client.post_statement.return_value = tls.TestLivySession.post_statement_json http_client.get_session.return_value = tls.TestLivySession.ready_sessions_json http_client.get_statement.return_value = tls.TestLivySession.ready_statement_json conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = tls.TestLivySession._create_session(kind=kind, http_client=http_client) conf.load() session.start(create_sql_context=False) session.wait_for_idle = MagicMock(side_effect=ValueError("yo")) command = Command("command", spark_events=spark_events) try: result = command.execute(session) assert False except ValueError as e: spark_events.emit_statement_execution_start_event._assert_called_once_with( session.guid, session.kind, session.id, command.guid) spark_events.emit_statement_execution_start_event._assert_called_once_with( session.guid, session.kind, session.id, command.guid, -1, False, "ValueError", "yo") assert_equals(e, session.wait_for_idle.side_effect)
def test_configuration_decorator(): def test_f(): return 0 conf.override_all({test_f.__name__: -1}) test_f_decorated = conf._override(test_f) assert_not_equals(test_f_decorated(), test_f()) assert_equals(test_f_decorated(), -1)
def test_execute(): spark_events = MagicMock() kind = SESSION_KIND_SPARK http_client = MagicMock() http_client.post_session.return_value = tls.TestLivySession.session_create_json http_client.post_statement.return_value = tls.TestLivySession.post_statement_json http_client.get_session.return_value = tls.TestLivySession.ready_sessions_json http_client.get_statement.return_value = tls.TestLivySession.ready_statement_json conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = tls.TestLivySession._create_session(kind=kind, http_client=http_client) conf.load() session.start(create_sql_context=False) command = Command("command", spark_events=spark_events) result = command.execute(session) http_client.post_statement.assert_called_with(0, {"code": command.code}) http_client.get_statement.assert_called_with(0, 0) assert result[0] assert_equals(tls.TestLivySession.pi_result, result[1]) spark_events.emit_statement_execution_start_event._assert_called_once_with(session.guid, session.kind, session.id, command.guid) spark_events.emit_statement_execution_end_event._assert_called_once_with(session.guid, session.kind, session.id, command.guid, 0, True, "", "")
def test_start_emits_start_end_failed_session_when_wait_for_idle_throws( self): http_client = MagicMock() http_client.post_session.return_value = self.session_create_json http_client.get_session.return_value = self.ready_sessions_json spark_events = MagicMock() conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) kind = constants.SESSION_KIND_SPARK session = self._create_session(kind=kind, http_client=http_client, spark_events=spark_events) session.create_sql_context = MagicMock() session.wait_for_idle = MagicMock(side_effect=ValueError) try: session.start() assert False except ValueError: pass conf.load() spark_events.emit_session_creation_start_event.assert_called_once_with( session.guid, kind) spark_events.emit_session_creation_end_event.assert_called_once_with( session.guid, kind, session.id, session.status, False, "ValueError", "")
def test_execute_failure_post_statement_emits_event(): spark_events = MagicMock() kind = SESSION_KIND_SPARK http_client = MagicMock() http_client.post_statement.side_effect = KeyError('Something bad happened here') conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = tls.TestLivySession._create_session(kind=kind, http_client=http_client) session.wait_for_idle = MagicMock() conf.load() session.start(create_sql_context=False) session.wait_for_idle = MagicMock() command = Command("command", spark_events=spark_events) try: result = command.execute(session) assert False except KeyError as e: spark_events.emit_statement_execution_start_event._assert_called_once_with(session.guid, session.kind, session.id, command.guid) spark_events.emit_statement_execution_start_event._assert_called_once_with(session.guid, session.kind, session.id, command.guid, -1, False, "KeyError", "Something bad happened here") assert_equals(e, http_client.post_statement.side_effect)
def test_delete_session_emits_start_failed_end_when_delete_throws(self): self.http_client.delete_session.side_effect = ValueError self.http_client.get_session.return_value = self.ready_sessions_json conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = self._create_session() conf.load() session.start(create_sql_context=False) session.id = 0 end_id = session.id end_status = constants.BUSY_SESSION_STATUS session.status = end_status try: session.delete() assert False except ValueError: pass self.spark_events.emit_session_deletion_start_event.assert_called_once_with( session.guid, session.kind, end_id, end_status) self.spark_events.emit_session_deletion_end_event.assert_called_once_with( session.guid, session.kind, end_id, end_status, False, "ValueError", "")
def test_delete_session_emits_start_failed_end_when_in_bad_state(self): http_client = MagicMock() http_client.get_session.return_value = self.ready_sessions_json spark_events = MagicMock() conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = self._create_session(http_client=http_client, spark_events=spark_events) conf.load() session.start(create_sql_context=False) session.id = 0 end_id = session.id end_status = constants.DEAD_SESSION_STATUS session.status = end_status session.delete() assert_equals(1, session.ipython_display.send_error.call_count) spark_events.emit_session_deletion_start_event.assert_called_once_with( session.guid, session.kind, end_id, end_status) spark_events.emit_session_deletion_end_event.assert_called_once_with( session.guid, session.kind, session.id, constants.DEAD_SESSION_STATUS, True, "", "")
def test_execute(): spark_events = MagicMock() kind = SESSION_KIND_SPARK http_client = MagicMock() http_client.post_session.return_value = tls.TestLivySession.session_create_json http_client.post_statement.return_value = tls.TestLivySession.post_statement_json http_client.get_session.return_value = tls.TestLivySession.ready_sessions_json http_client.get_statement.return_value = tls.TestLivySession.ready_statement_json conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = tls.TestLivySession._create_session(kind=kind, http_client=http_client) conf.load() session.start(create_sql_context=False) command = Command("command", spark_events=spark_events) result = command.execute(session) http_client.post_statement.assert_called_with(0, {"code": command.code}) http_client.get_statement.assert_called_with(0, 0) assert result[0] assert_equals(tls.TestLivySession.pi_result, result[1]) spark_events.emit_statement_execution_start_event._assert_called_once_with( session.guid, session.kind, session.id, command.guid) spark_events.emit_statement_execution_end_event._assert_called_once_with( session.guid, session.kind, session.id, command.guid, 0, True, "", "")
def test_execute(self): kind = constants.SESSION_KIND_SPARK http_client = MagicMock() self.post_responses = [ DummyResponse(201, self.session_create_json), DummyResponse(201, self.post_statement_json) ] http_client.post.side_effect = self._next_response_post self.get_responses = [ DummyResponse(200, self.running_statement_json), DummyResponse(200, self.ready_statement_json) ] http_client.get.side_effect = self._next_response_get conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = self._create_session(kind=kind, http_client=http_client) conf.load() session.start() command = "command" result = session.execute(command) http_client.post.assert_called_with("/sessions/0/statements", [201], {"code": command}) http_client.get.assert_called_with("/sessions/0/statements", [200]) assert_equals(2, http_client.get.call_count) assert result[0] assert_equals(self.pi_result, result[1])
def test_delete_session_emits_start_failed_end_when_in_bad_state(self): http_client = MagicMock() http_client.get_session.return_value = self.ready_sessions_json spark_events = MagicMock() conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = self._create_session(http_client=http_client, spark_events=spark_events) conf.load() session.start(create_sql_context=False) session.id = 0 end_id = session.id end_status = constants.DEAD_SESSION_STATUS session.status = end_status session.delete() assert_equals(1, session.ipython_display.send_error.call_count) spark_events.emit_session_deletion_start_event.assert_called_once_with(session.guid, session.kind, end_id, end_status) spark_events.emit_session_deletion_end_event.assert_called_once_with(session.guid, session.kind, session.id, constants.DEAD_SESSION_STATUS, True, "", "")
def test_start_emits_start_end_failed_session_when_wait_for_idle_throws(self): http_client = MagicMock() http_client.post_session.return_value = self.session_create_json http_client.get_session.return_value = self.ready_sessions_json spark_events = MagicMock() conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) kind = constants.SESSION_KIND_SPARK session = self._create_session(kind=kind, http_client=http_client, spark_events=spark_events) session.create_sql_context = MagicMock() session.wait_for_idle = MagicMock(side_effect=ValueError) try: session.start() assert False except ValueError: pass conf.load() spark_events.emit_session_creation_start_event.assert_called_once_with(session.guid, kind) spark_events.emit_session_creation_end_event.assert_called_once_with(session.guid, kind, session.id, session.status, False, "ValueError", "")
def test_create_sql_hive_context_pyspark(self): kind = constants.SESSION_KIND_PYSPARK http_client = MagicMock() self.post_responses = [ DummyResponse(201, self.session_create_json), DummyResponse(201, self.post_statement_json), DummyResponse(201, self.post_statement_json) ] http_client.post.side_effect = self._next_response_post self.get_responses = [ DummyResponse(200, self.ready_sessions_json), DummyResponse(200, self.running_statement_json), DummyResponse(200, self.ready_statement_json), DummyResponse(200, self.ready_sessions_json), DummyResponse(200, self.ready_statement_json) ] http_client.get.side_effect = self._next_response_get conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = self._create_session(kind=kind, http_client=http_client) conf.load() session.start() session.create_sql_context() assert call("/sessions/0/statements", [201], {"code": "from pyspark.sql import HiveContext\n" "sqlContext = HiveContext(sc)"}) \ in http_client.post.call_args_list
def test_constructor_throws_invalid_session_sql_combo(self): conf.override_all({ "status_sleep_seconds": 2, "statement_sleep_seconds": 2, "create_sql_context_timeout_seconds": 60 }) self._create_session(sql_created=True) conf.load()
def test_constructor_throws_sql_create_timeout_seconds(self): conf.override_all({ "status_sleep_seconds": 4, "statement_sleep_seconds": 2, "create_sql_context_timeout_seconds": 0 }) self._create_session() conf.load()
def test_constructor_throws_statement_sleep_seconds(self): conf.override_all({ "status_sleep_seconds": 3, "statement_sleep_seconds": 0, "create_sql_context_timeout_seconds": 60 }) self._create_session() conf.load()
def test_constructor_starts_with_no_session(self): conf.override_all( {"status_sleep_seconds": 4, "statement_sleep_seconds": 2, "create_sql_context_timeout_seconds": 60} ) session = self._create_session() conf.load() assert session.id == "-1" assert not session.started_sql_context
def test_delete_session_when_dead_throws(self): http_client = MagicMock() http_client.post.return_value = DummyResponse(201, self.session_create_json) conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01}) session = self._create_session(http_client=http_client) conf.load() session._status = "dead" session.delete()
def _setup(): global magic, spark_controller, shell, ipython_display conf.override_all({}) magic = RemoteSparkMagics(shell=None, widget=MagicMock()) magic.shell = shell = MagicMock() magic.ipython_display = ipython_display = MagicMock() magic.spark_controller = spark_controller = MagicMock()
def test_configuration_override(): kpc = { 'username': '******', 'password': '******', 'url': 'L' } overrides = { conf.kernel_python_credentials.__name__: kpc } conf.override_all(overrides) conf.override(conf.status_sleep_seconds.__name__, 1) assert_equals(conf._overrides, { conf.kernel_python_credentials.__name__: kpc, conf.status_sleep_seconds.__name__: 1 }) assert_equals(conf.status_sleep_seconds(), 1) assert_equals(conf.kernel_python_credentials(), kpc)
def _setup(): global magic, spark_controller, shell, ipython_display conf.override_all({}) magic = TestKernelMagics(shell=None) magic.shell = shell = MagicMock() magic.ipython_display = ipython_display = MagicMock() magic.spark_controller = spark_controller = MagicMock()
def test_constructor_starts_with_existing_session(self): conf.override_all( {"status_sleep_seconds": 4, "statement_sleep_seconds": 2, "create_sql_context_timeout_seconds": 60} ) session_id = "1" session = self._create_session(session_id=session_id, sql_created=True) conf.load() assert session.id == session_id assert session.started_sql_context
def test_is_final_status(self): conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01}) session = self._create_session() conf.load() assert not session.is_final_status("idle") assert not session.is_final_status("starting") assert not session.is_final_status("busy") assert session.is_final_status("dead") assert session.is_final_status("error")
def test_serialize_periodically(): conf.override_all({conf.serialize_period_seconds.__name__: 0.1, conf.serialize_periodically.__name__: True}) serializer = MagicMock() ClientManager(serializer) time.sleep(0.5) assert serializer.serialize_state.call_count >= 1 conf.load()
def test_constructor_starts_with_no_session(self): conf.override_all({ "status_sleep_seconds": 4, "statement_sleep_seconds": 2, "create_sql_context_timeout_seconds": 60 }) session = self._create_session() conf.load() assert session.id == "-1" assert not session.started_sql_context
def _setup(): global magic, spark_controller, shell, ipython_display, spark_events conf.override_all({}) spark_events = MagicMock() magic = TestKernelMagics(shell=None, spark_events=spark_events) magic.shell = shell = MagicMock() magic.ipython_display = ipython_display = MagicMock() magic.spark_controller = spark_controller = MagicMock() magic._generate_uuid = MagicMock(return_value='0000')
def test_delete_session_when_not_started(self): http_client = MagicMock() http_client.post.return_value = DummyResponse(201, self.session_create_json) conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01}) session = self._create_session(http_client=http_client) conf.load() session.delete() assert_equals("dead", session._status) assert_equals("-1", session.id)
def test_constructor_starts_with_existing_session(self): conf.override_all({ "status_sleep_seconds": 4, "statement_sleep_seconds": 2, "create_sql_context_timeout_seconds": 60 }) session_id = "1" session = self._create_session(session_id=session_id, sql_created=True) conf.load() assert session.id == session_id assert session.started_sql_context
def test_delete_session_when_not_started(self): self.http_client.post_session.return_value = self.session_create_json conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = self._create_session() conf.load() session.delete() assert_equals(session.ipython_display.send_error.call_count, 1)
def test_logs_gets_latest_logs(self): http_client = MagicMock() http_client.post.return_value = DummyResponse(201, self.session_create_json) http_client.get.return_value = DummyResponse(200, self.log_json) conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01}) session = self._create_session(http_client=http_client) conf.load() session.start() logs = session.logs assert_equals("hi\nhi", logs) http_client.get.assert_called_with("/sessions/0/log?from=0", [200])
def test_configuration_load_not_lazy(): a = "whoops" config = { conf.events_handler_class.__name__: a } fsrw = MagicMock() fsrw.path = "" read_lines = MagicMock(return_value=[json.dumps(config)]) fsrw.read_lines = read_lines fsrw_class = MagicMock(return_value=fsrw) conf.override_all({conf.events_handler_class.__name__: "bar"}) conf.load(fsrw_class) assert conf._overrides is not None assert_equals(conf._overrides, config) assert_equals(conf.events_handler_class(), a)
def test_delete_session_when_dead_throws(self): http_client = MagicMock() http_client.post.return_value = DummyResponse(201, self.session_create_json) conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = self._create_session(http_client=http_client) conf.load() session.status = "dead" session.delete()
def test_delete_session_when_not_started(self): http_client = MagicMock() http_client.post_session.return_value = self.session_create_json conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = self._create_session(http_client=http_client) conf.load() session.delete() assert_equals(session.ipython_display.send_error.call_count, 1)
def test_sqlquery_loads_defaults(): defaults = { conf.default_samplemethod.__name__: "sample", conf.default_maxrows.__name__: 419, conf.default_samplefraction.__name__: 0.99, } conf.override_all(defaults) query = "DROP TABLE USERS;" sqlquery = SQLQuery(query) assert_equals(sqlquery.query, query) assert_equals(sqlquery.samplemethod, defaults[conf.default_samplemethod.__name__]) assert_equals(sqlquery.maxrows, defaults[conf.default_maxrows.__name__]) assert_equals(sqlquery.samplefraction, defaults[conf.default_samplefraction.__name__])
def test_configuration_load_not_lazy(): a = "whoops" config = {conf.default_chart_type.__name__: a} fsrw = MagicMock() fsrw.path = "" read_lines = MagicMock(return_value=[json.dumps(config)]) fsrw.read_lines = read_lines fsrw_class = MagicMock(return_value=fsrw) conf.override_all({conf.default_chart_type.__name__: "bar"}) conf.load(fsrw_class) assert conf._overrides is not None assert_equals(conf._overrides, config) assert_equals(conf.default_chart_type(), "whoops")
def test_create_sql_hive_context_happens_once(self): kind = Constants.session_kind_spark http_client = MagicMock() ipython_display = MagicMock() self.post_responses = [ DummyResponse(201, self.session_create_json), DummyResponse(201, self.post_statement_json), DummyResponse(201, self.post_statement_json), ] http_client.post.side_effect = self._next_response_post self.get_responses = [ DummyResponse(200, self.ready_sessions_json), DummyResponse(200, self.running_statement_json), DummyResponse(200, self.ready_statement_json), DummyResponse(200, self.ready_sessions_json), DummyResponse(200, self.ready_statement_json), ] http_client.get.side_effect = self._next_response_get conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01}) session = self._create_session(kind=kind, http_client=http_client) session.ipython_display = ipython_display conf.load() session.start() # Reset the mock so that post called count is accurate http_client.reset_mock() session.create_sql_context() assert ipython_display.writeln.call_count == 3 # Second call should not issue a post request session.create_sql_context() assert ( call( "/sessions/0/statements", [201], {"code": "val sqlContext = new org.apache.spark.sql.SQLContext" "(sc)\nimport sqlContext.implicits._"}, ) in http_client.post.call_args_list ) assert ( call( "/sessions/0/statements", [201], {"code": "val hiveContext = new org.apache.spark.sql.hive.Hive" "Context(sc)"}, ) in http_client.post.call_args_list ) assert len(http_client.post.call_args_list) == 2
def test_is_final_status(self): conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = self._create_session() conf.load() assert not session.is_final_status("idle") assert not session.is_final_status("starting") assert not session.is_final_status("busy") assert session.is_final_status("dead") assert session.is_final_status("error")
def test_delete_session_when_active(self): self.http_client.post_session.return_value = self.session_create_json self.http_client.get_session.return_value = self.ready_sessions_json conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = self._create_session() conf.load() session.start(create_sql_context=False) session.delete() assert_equals("dead", session.status)
def test_create_sql_hive_context_unknown_throws(self): kind = "unknown" self.http_client.post_session.return_value = self.session_create_json self.http_client.post_statement.return_value = self.post_statement_json self.http_client.get_session.return_value = self.ready_sessions_json self.get_statement_responses = [self.running_statement_json, self.ready_statement_json] self.http_client.get_statement.side_effect = self._next_statement_response_get conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = self._create_session(kind=kind) conf.load() session.start()
def test_delete_session_when_active(self): http_client = MagicMock() http_client.post_session.return_value = self.session_create_json http_client.get_session.return_value = self.ready_sessions_json conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = self._create_session(http_client=http_client) conf.load() session.start(create_sql_context=False) session.delete() assert_equals("dead", session.status)
def test_delete_session_when_not_started(self): http_client = MagicMock() http_client.post.return_value = DummyResponse(201, self.session_create_json) conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = self._create_session(http_client=http_client) conf.load() session.delete() assert_equals("dead", session.status) assert_equals("-1", session.id)
def test_logs_gets_latest_logs(self): http_client = MagicMock() http_client.post.return_value = DummyResponse(201, self.session_create_json) http_client.get.return_value = DummyResponse(200, self.log_json) conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = self._create_session(http_client=http_client) conf.load() session.start() logs = session.get_logs() assert_equals("hi\nhi", logs) http_client.get.assert_called_with("/sessions/0/log?from=0", [200])
def test_create_sql_hive_context_happens_once(self): kind = constants.SESSION_KIND_SPARK http_client = MagicMock() ipython_display = MagicMock() http_client.post_session.return_value = self.session_create_json self.post_statement_responses = [ self.post_statement_json, self.post_statement_json ] http_client.post_statement.side_effect = self._next_statement_response_post http_client.get_session.return_value = self.ready_sessions_json self.get_statement_responses = [ self.running_statement_json, self.ready_statement_json, self.ready_statement_json ] http_client.get_statement.side_effect = self._next_statement_response_get conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = self._create_session(kind=kind, http_client=http_client) session.ipython_display = ipython_display conf.load() session.start(create_sql_context=False) # Reset the mock so that post called count is accurate http_client.reset_mock() session.create_sql_context() assert ipython_display.writeln.call_count == 2 assert session.created_sql_context # Second call should not issue a post request session.create_sql_context() assert call(0, {"code": "val sqlContext = new org.apache.spark.sql.hive.HiveContext(sc)"}) \ in http_client.post_statement.call_args_list assert len(http_client.post_statement.call_args_list) == 1 session._get_sql_context_creation_command = MagicMock() session._get_sql_context_creation_command.return_value.execute.return_value = ( True, "Success") session.created_sql_context = None session.create_sql_context() assert session.created_sql_context
def test_create_sql_hive_context_unknown_throws(self): kind = "unknown" http_client = MagicMock() http_client.post_session.return_value = self.session_create_json http_client.post_statement.return_value = self.post_statement_json http_client.get_session.return_value = self.ready_sessions_json self.get_statement_responses = [ self.running_statement_json, self.ready_statement_json ] http_client.get_statement.side_effect = self._next_statement_response_get conf.override_all({ "status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01 }) session = self._create_session(kind=kind, http_client=http_client) conf.load() session.start()