Exemplo n.º 1
0
    def test_create_sql_hive_context_pyspark(self):
        kind = constants.SESSION_KIND_PYSPARK
        http_client = MagicMock()
        self.post_responses = [DummyResponse(201, self.session_create_json),
                               DummyResponse(201, self.post_statement_json),
                               DummyResponse(201, self.post_statement_json)]
        http_client.post.side_effect = self._next_response_post
        self.get_responses = [DummyResponse(200, self.ready_sessions_json),
                              DummyResponse(200, self.running_statement_json),
                              DummyResponse(200, self.ready_statement_json),
                              DummyResponse(200, self.ready_sessions_json),
                              DummyResponse(200, self.ready_statement_json)]
        http_client.get.side_effect = self._next_response_get
        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        session = self._create_session(kind=kind, http_client=http_client)
        conf.load()
        session.start()

        session.create_sql_context()

        assert call("/sessions/0/statements", [201], {"code": "from pyspark.sql import HiveContext\n"
                                                              "sqlContext = HiveContext(sc)"}) \
               in http_client.post.call_args_list
Exemplo n.º 2
0
    def test_start_emits_start_end_failed_session_when_wait_for_idle_throws(self):
        http_client = MagicMock()
        http_client.post_session.return_value = self.session_create_json
        http_client.get_session.return_value = self.ready_sessions_json

        spark_events = MagicMock()

        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        kind = constants.SESSION_KIND_SPARK
        session = self._create_session(kind=kind, http_client=http_client, spark_events=spark_events)
        session.create_sql_context = MagicMock()
        session.wait_for_idle = MagicMock(side_effect=ValueError)

        try:
            session.start()
            assert False
        except ValueError:
            pass

        conf.load()

        spark_events.emit_session_creation_start_event.assert_called_once_with(session.guid, kind)
        spark_events.emit_session_creation_end_event.assert_called_once_with(session.guid, kind, session.id,
                                                                             session.status, False, "ValueError", "")
Exemplo n.º 3
0
def test_execute_failure_post_statement_emits_event():
    spark_events = MagicMock()
    kind = SESSION_KIND_SPARK
    http_client = MagicMock()
    http_client.post_statement.side_effect = KeyError('Something bad happened here')
    conf.override_all({
        "status_sleep_seconds": 0.01,
        "statement_sleep_seconds": 0.01
    })
    session = tls.TestLivySession._create_session(kind=kind, http_client=http_client)
    session.wait_for_idle = MagicMock()
    conf.load()
    session.start(create_sql_context=False)
    session.wait_for_idle = MagicMock()
    command = Command("command", spark_events=spark_events)

    try:
        result = command.execute(session)
        assert False
    except KeyError as e:
        spark_events.emit_statement_execution_start_event._assert_called_once_with(session.guid, session.kind,
                                                                                   session.id, command.guid)
        spark_events.emit_statement_execution_start_event._assert_called_once_with(session.guid, session.kind,
                                                                                   session.id, command.guid,
                                                                                   -1, False, "KeyError",
                                                                                   "Something bad happened here")
        assert_equals(e, http_client.post_statement.side_effect)
Exemplo n.º 4
0
def test_execute_failure_wait_for_session_emits_event():
    spark_events = MagicMock()
    kind = SESSION_KIND_SPARK
    http_client = MagicMock()
    http_client.post_session.return_value = tls.TestLivySession.session_create_json
    http_client.post_statement.return_value = tls.TestLivySession.post_statement_json
    http_client.get_session.return_value = tls.TestLivySession.ready_sessions_json
    http_client.get_statement.return_value = tls.TestLivySession.ready_statement_json
    conf.override_all({
        "status_sleep_seconds": 0.01,
        "statement_sleep_seconds": 0.01
    })
    session = tls.TestLivySession._create_session(kind=kind, http_client=http_client)
    conf.load()
    session.start(create_sql_context=False)
    session.wait_for_idle = MagicMock(side_effect=ValueError("yo"))
    command = Command("command", spark_events=spark_events)

    try:
        result = command.execute(session)
        assert False
    except ValueError as e:
        spark_events.emit_statement_execution_start_event._assert_called_once_with(session.guid, session.kind,
                                                                                   session.id, command.guid)
        spark_events.emit_statement_execution_start_event._assert_called_once_with(session.guid, session.kind,
                                                                                   session.id, command.guid,
                                                                                   -1, False, "ValueError", "yo")
        assert_equals(e, session.wait_for_idle.side_effect)
Exemplo n.º 5
0
    def test_delete_session_emits_start_failed_end_when_delete_throws(self):
        self.http_client.delete_session.side_effect = ValueError
        self.http_client.get_session.return_value = self.ready_sessions_json

        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        session = self._create_session()
        conf.load()
        session.start(create_sql_context=False)
        session.id = 0
        end_id = session.id
        end_status = constants.BUSY_SESSION_STATUS
        session.status = end_status

        try:
            session.delete()
            assert False
        except ValueError:
            pass

        self.spark_events.emit_session_deletion_start_event.assert_called_once_with(
            session.guid, session.kind, end_id, end_status)
        self.spark_events.emit_session_deletion_end_event.assert_called_once_with(
            session.guid, session.kind, end_id, end_status, False, "ValueError", "")
Exemplo n.º 6
0
def test_execute_failure_wait_for_session_emits_event():
    spark_events = MagicMock()
    kind = SESSION_KIND_SPARK
    http_client = MagicMock()
    http_client.post_session.return_value = tls.TestLivySession.session_create_json
    http_client.post_statement.return_value = tls.TestLivySession.post_statement_json
    http_client.get_session.return_value = tls.TestLivySession.ready_sessions_json
    http_client.get_statement.return_value = tls.TestLivySession.ready_statement_json
    conf.override_all({
        "status_sleep_seconds": 0.01,
        "statement_sleep_seconds": 0.01
    })
    session = tls.TestLivySession._create_session(kind=kind,
                                                  http_client=http_client)
    conf.load()
    session.start(create_sql_context=False)
    session.wait_for_idle = MagicMock(side_effect=ValueError("yo"))
    command = Command("command", spark_events=spark_events)

    try:
        result = command.execute(session)
        assert False
    except ValueError as e:
        spark_events.emit_statement_execution_start_event._assert_called_once_with(
            session.guid, session.kind, session.id, command.guid)
        spark_events.emit_statement_execution_start_event._assert_called_once_with(
            session.guid, session.kind, session.id, command.guid, -1, False,
            "ValueError", "yo")
        assert_equals(e, session.wait_for_idle.side_effect)
Exemplo n.º 7
0
    def test_create_sql_context_spark(self):
        kind = constants.SESSION_KIND_SPARK
        http_client = MagicMock()
        http_client.post_session.return_value = self.session_create_json
        self.post_statement_responses = [
            self.post_statement_json, self.post_statement_json
        ]
        http_client.post_statement.side_effect = self._next_statement_response_post
        self.get_session_responses = [
            self.ready_sessions_json, self.ready_sessions_json
        ]
        http_client.get_session.side_effect = self._next_session_response_get
        self.get_statement_responses = [
            self.running_statement_json, self.ready_statement_json,
            self.ready_statement_json
        ]
        http_client.get_statement.side_effect = self._next_statement_response_get
        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        session = self._create_session(kind=kind, http_client=http_client)
        conf.load()
        session.start()

        assert call(0, {"code": "val sqlContext = new org.apache.spark.sql.hive.HiveContext(sc)"}) \
               in http_client.post_statement.call_args_list
Exemplo n.º 8
0
def test_execute():
    spark_events = MagicMock()
    kind = SESSION_KIND_SPARK
    http_client = MagicMock()
    http_client.post_session.return_value = tls.TestLivySession.session_create_json
    http_client.post_statement.return_value = tls.TestLivySession.post_statement_json
    http_client.get_session.return_value = tls.TestLivySession.ready_sessions_json
    http_client.get_statement.return_value = tls.TestLivySession.ready_statement_json
    conf.override_all({
        "status_sleep_seconds": 0.01,
        "statement_sleep_seconds": 0.01
    })
    session = tls.TestLivySession._create_session(kind=kind,
                                                  http_client=http_client)
    conf.load()
    session.start(create_sql_context=False)
    command = Command("command", spark_events=spark_events)

    result = command.execute(session)

    http_client.post_statement.assert_called_with(0, {"code": command.code})
    http_client.get_statement.assert_called_with(0, 0)
    assert result[0]
    assert_equals(tls.TestLivySession.pi_result, result[1])
    spark_events.emit_statement_execution_start_event._assert_called_once_with(
        session.guid, session.kind, session.id, command.guid)
    spark_events.emit_statement_execution_end_event._assert_called_once_with(
        session.guid, session.kind, session.id, command.guid, 0, True, "", "")
Exemplo n.º 9
0
    def test_delete_session_emits_start_failed_end_when_delete_throws(self):
        http_client = MagicMock()
        http_client.delete_session.side_effect = ValueError
        http_client.get_session.return_value = self.ready_sessions_json

        spark_events = MagicMock()

        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        session = self._create_session(http_client=http_client,
                                       spark_events=spark_events)
        conf.load()
        session.start(create_sql_context=False)
        session.id = 0
        end_id = session.id
        end_status = constants.BUSY_SESSION_STATUS
        session.status = end_status

        try:
            session.delete()
            assert False
        except ValueError:
            pass

        spark_events.emit_session_deletion_start_event.assert_called_once_with(
            session.guid, session.kind, end_id, end_status)
        spark_events.emit_session_deletion_end_event.assert_called_once_with(
            session.guid, session.kind, end_id, end_status, False,
            "ValueError", "")
Exemplo n.º 10
0
    def test_start_emits_start_end_failed_session_when_wait_for_idle_throws(
            self):
        http_client = MagicMock()
        http_client.post_session.return_value = self.session_create_json
        http_client.get_session.return_value = self.ready_sessions_json

        spark_events = MagicMock()

        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        kind = constants.SESSION_KIND_SPARK
        session = self._create_session(kind=kind,
                                       http_client=http_client,
                                       spark_events=spark_events)
        session.create_sql_context = MagicMock()
        session.wait_for_idle = MagicMock(side_effect=ValueError)

        try:
            session.start()
            assert False
        except ValueError:
            pass

        conf.load()

        spark_events.emit_session_creation_start_event.assert_called_once_with(
            session.guid, kind)
        spark_events.emit_session_creation_end_event.assert_called_once_with(
            session.guid, kind, session.id, session.status, False,
            "ValueError", "")
Exemplo n.º 11
0
    def test_delete_session_emits_start_failed_end_when_in_bad_state(self):
        http_client = MagicMock()
        http_client.get_session.return_value = self.ready_sessions_json

        spark_events = MagicMock()

        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        session = self._create_session(http_client=http_client,
                                       spark_events=spark_events)
        conf.load()
        session.start(create_sql_context=False)
        session.id = 0
        end_id = session.id
        end_status = constants.DEAD_SESSION_STATUS
        session.status = end_status

        session.delete()

        assert_equals(1, session.ipython_display.send_error.call_count)
        spark_events.emit_session_deletion_start_event.assert_called_once_with(
            session.guid, session.kind, end_id, end_status)
        spark_events.emit_session_deletion_end_event.assert_called_once_with(
            session.guid, session.kind, session.id,
            constants.DEAD_SESSION_STATUS, True, "", "")
Exemplo n.º 12
0
    def test_execute(self):
        kind = constants.SESSION_KIND_SPARK
        http_client = MagicMock()
        self.post_responses = [
            DummyResponse(201, self.session_create_json),
            DummyResponse(201, self.post_statement_json)
        ]
        http_client.post.side_effect = self._next_response_post
        self.get_responses = [
            DummyResponse(200, self.running_statement_json),
            DummyResponse(200, self.ready_statement_json)
        ]
        http_client.get.side_effect = self._next_response_get
        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        session = self._create_session(kind=kind, http_client=http_client)
        conf.load()
        session.start()
        command = "command"

        result = session.execute(command)

        http_client.post.assert_called_with("/sessions/0/statements", [201],
                                            {"code": command})
        http_client.get.assert_called_with("/sessions/0/statements", [200])
        assert_equals(2, http_client.get.call_count)
        assert result[0]
        assert_equals(self.pi_result, result[1])
Exemplo n.º 13
0
    def test_create_sql_hive_context_pyspark(self):
        kind = constants.SESSION_KIND_PYSPARK
        http_client = MagicMock()
        self.post_responses = [
            DummyResponse(201, self.session_create_json),
            DummyResponse(201, self.post_statement_json),
            DummyResponse(201, self.post_statement_json)
        ]
        http_client.post.side_effect = self._next_response_post
        self.get_responses = [
            DummyResponse(200, self.ready_sessions_json),
            DummyResponse(200, self.running_statement_json),
            DummyResponse(200, self.ready_statement_json),
            DummyResponse(200, self.ready_sessions_json),
            DummyResponse(200, self.ready_statement_json)
        ]
        http_client.get.side_effect = self._next_response_get
        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        session = self._create_session(kind=kind, http_client=http_client)
        conf.load()
        session.start()

        session.create_sql_context()

        assert call("/sessions/0/statements", [201], {"code": "from pyspark.sql import HiveContext\n"
                                                              "sqlContext = HiveContext(sc)"}) \
               in http_client.post.call_args_list
Exemplo n.º 14
0
    def test_delete_session_emits_start_failed_end_when_in_bad_state(self):
        http_client = MagicMock()
        http_client.get_session.return_value = self.ready_sessions_json

        spark_events = MagicMock()

        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        session = self._create_session(http_client=http_client, spark_events=spark_events)
        conf.load()
        session.start(create_sql_context=False)
        session.id = 0
        end_id = session.id
        end_status = constants.DEAD_SESSION_STATUS
        session.status = end_status

        session.delete()

        assert_equals(1, session.ipython_display.send_error.call_count)
        spark_events.emit_session_deletion_start_event.assert_called_once_with(session.guid, session.kind, end_id,
                                                                               end_status)
        spark_events.emit_session_deletion_end_event.assert_called_once_with(session.guid, session.kind, session.id,
                                                                             constants.DEAD_SESSION_STATUS, True, "",
                                                                             "")
Exemplo n.º 15
0
def test_execute_failure_get_statement_output_emits_event():
    spark_events = MagicMock()
    kind = SESSION_KIND_SPARK
    http_client = MagicMock()
    conf.override_all({
        "status_sleep_seconds": 0.01,
        "statement_sleep_seconds": 0.01
    })
    session = tls.TestLivySession._create_session(kind=kind, http_client=http_client)
    session.wait_for_idle = MagicMock()
    conf.load()
    session.start(create_sql_context=False)
    session.wait_for_idle = MagicMock()
    command = Command("command", spark_events=spark_events)
    command._get_statement_output = MagicMock(side_effect=AttributeError('OHHHH'))

    try:
        result = command.execute(session)
        assert False
    except AttributeError as e:
        spark_events.emit_statement_execution_start_event._assert_called_once_with(session.guid, session.kind,
                                                                                   session.id, command.guid)
        spark_events.emit_statement_execution_start_event._assert_called_once_with(session.guid, session.kind,
                                                                                   session.id, command.guid,
                                                                                   -1, False, "AttributeError",
                                                                                   "OHHHH")
        assert_equals(e, command._get_statement_output.side_effect)
Exemplo n.º 16
0
    def test_create_sql_hive_context_happens_once(self):
        kind = constants.SESSION_KIND_SPARK
        http_client = MagicMock()
        ipython_display = MagicMock()

        http_client.post_session.return_value = self.session_create_json
        self.post_statement_responses = [self.post_statement_json,
                                         self.post_statement_json]
        http_client.post_statement.side_effect = self._next_statement_response_post
        http_client.get_session.return_value = self.ready_sessions_json
        self.get_statement_responses = [self.running_statement_json,
                                        self.ready_statement_json,
                                        self.ready_statement_json]
        http_client.get_statement.side_effect = self._next_statement_response_get
        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        session = self._create_session(kind=kind, http_client=http_client)
        session.ipython_display = ipython_display
        conf.load()
        session.start(create_sql_context=False)

        # Reset the mock so that post called count is accurate
        http_client.reset_mock()

        session.create_sql_context()
        assert ipython_display.writeln.call_count == 2

        # Second call should not issue a post request
        session.create_sql_context()

        assert call(0, {"code": "val sqlContext = new org.apache.spark.sql.hive.HiveContext(sc)"}) \
               in http_client.post_statement.call_args_list
        assert len(http_client.post_statement.call_args_list) == 1
Exemplo n.º 17
0
def test_execute():
    spark_events = MagicMock()
    kind = SESSION_KIND_SPARK
    http_client = MagicMock()
    http_client.post_session.return_value = tls.TestLivySession.session_create_json
    http_client.post_statement.return_value = tls.TestLivySession.post_statement_json
    http_client.get_session.return_value = tls.TestLivySession.ready_sessions_json
    http_client.get_statement.return_value = tls.TestLivySession.ready_statement_json
    conf.override_all({
        "status_sleep_seconds": 0.01,
        "statement_sleep_seconds": 0.01
    })
    session = tls.TestLivySession._create_session(kind=kind, http_client=http_client)
    conf.load()
    session.start(create_sql_context=False)
    command = Command("command", spark_events=spark_events)

    result = command.execute(session)

    http_client.post_statement.assert_called_with(0, {"code": command.code})
    http_client.get_statement.assert_called_with(0, 0)
    assert result[0]
    assert_equals(tls.TestLivySession.pi_result, result[1])
    spark_events.emit_statement_execution_start_event._assert_called_once_with(session.guid, session.kind,
                                                                                        session.id, command.guid)
    spark_events.emit_statement_execution_end_event._assert_called_once_with(session.guid, session.kind,
                                                                                      session.id, command.guid,
                                                                                      0, True, "", "")
Exemplo n.º 18
0
def test_execute_failure_post_statement_emits_event():
    spark_events = MagicMock()
    kind = SESSION_KIND_SPARK
    http_client = MagicMock()
    http_client.post_statement.side_effect = KeyError(
        'Something bad happened here')
    conf.override_all({
        "status_sleep_seconds": 0.01,
        "statement_sleep_seconds": 0.01
    })
    session = tls.TestLivySession._create_session(kind=kind,
                                                  http_client=http_client)
    session.wait_for_idle = MagicMock()
    conf.load()
    session.start(create_sql_context=False)
    session.wait_for_idle = MagicMock()
    command = Command("command", spark_events=spark_events)

    try:
        result = command.execute(session)
        assert False
    except KeyError as e:
        spark_events.emit_statement_execution_start_event._assert_called_once_with(
            session.guid, session.kind, session.id, command.guid)
        spark_events.emit_statement_execution_start_event._assert_called_once_with(
            session.guid, session.kind, session.id, command.guid, -1, False,
            "KeyError", "Something bad happened here")
        assert_equals(e, http_client.post_statement.side_effect)
Exemplo n.º 19
0
    def test_execute(self):
        kind = Constants.session_kind_spark
        http_client = MagicMock()
        self.post_responses = [
            DummyResponse(201, self.session_create_json),
            DummyResponse(201, self.post_statement_json),
        ]
        http_client.post.side_effect = self._next_response_post
        self.get_responses = [
            DummyResponse(200, self.running_statement_json),
            DummyResponse(200, self.ready_statement_json),
        ]
        http_client.get.side_effect = self._next_response_get
        conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01})
        session = self._create_session(kind=kind, http_client=http_client)
        conf.load()
        session.start()
        command = "command"

        result = session.execute(command)

        http_client.post.assert_called_with("/sessions/0/statements", [201], {"code": command})
        http_client.get.assert_called_with("/sessions/0/statements", [200])
        assert_equals(2, http_client.get.call_count)
        assert result[0]
        assert_equals(self.pi_result, result[1])
Exemplo n.º 20
0
def test_execute_failure_get_statement_output_emits_event():
    spark_events = MagicMock()
    kind = SESSION_KIND_SPARK
    http_client = MagicMock()
    conf.override_all({
        "status_sleep_seconds": 0.01,
        "statement_sleep_seconds": 0.01
    })
    session = tls.TestLivySession._create_session(kind=kind,
                                                  http_client=http_client)
    session.wait_for_idle = MagicMock()
    conf.load()
    session.start(create_sql_context=False)
    session.wait_for_idle = MagicMock()
    command = Command("command", spark_events=spark_events)
    command._get_statement_output = MagicMock(
        side_effect=AttributeError('OHHHH'))

    try:
        result = command.execute(session)
        assert False
    except AttributeError as e:
        spark_events.emit_statement_execution_start_event._assert_called_once_with(
            session.guid, session.kind, session.id, command.guid)
        spark_events.emit_statement_execution_start_event._assert_called_once_with(
            session.guid, session.kind, session.id, command.guid, -1, False,
            "AttributeError", "OHHHH")
        assert_equals(e, command._get_statement_output.side_effect)
Exemplo n.º 21
0
 def test_constructor_throws_statement_sleep_seconds(self):
     conf.override_all({
         "status_sleep_seconds": 3,
         "statement_sleep_seconds": 0,
         "create_sql_context_timeout_seconds": 60
     })
     self._create_session()
     conf.load()
Exemplo n.º 22
0
 def test_constructor_throws_invalid_session_sql_combo(self):
     conf.override_all({
         "status_sleep_seconds": 2,
         "statement_sleep_seconds": 2,
         "create_sql_context_timeout_seconds": 60
     })
     self._create_session(sql_created=True)
     conf.load()
Exemplo n.º 23
0
 def test_constructor_throws_invalid_session_sql_combo(self):
     conf.override_all({
         "status_sleep_seconds": 2,
         "statement_sleep_seconds": 2,
         "create_sql_context_timeout_seconds": 60
     })
     self._create_session(sql_created=True)
     conf.load()
Exemplo n.º 24
0
 def test_constructor_throws_sql_create_timeout_seconds(self):
     conf.override_all({
         "status_sleep_seconds": 4,
         "statement_sleep_seconds": 2,
         "create_sql_context_timeout_seconds": 0
     })
     self._create_session()
     conf.load()
Exemplo n.º 25
0
    def test_constructor_starts_with_no_session(self):
        conf.override_all(
            {"status_sleep_seconds": 4, "statement_sleep_seconds": 2, "create_sql_context_timeout_seconds": 60}
        )
        session = self._create_session()
        conf.load()

        assert session.id == "-1"
        assert not session.started_sql_context
Exemplo n.º 26
0
    def test_delete_session_when_dead_throws(self):
        http_client = MagicMock()
        http_client.post.return_value = DummyResponse(201, self.session_create_json)
        conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01})
        session = self._create_session(http_client=http_client)
        conf.load()
        session._status = "dead"

        session.delete()
Exemplo n.º 27
0
    def test_constructor_starts_with_existing_session(self):
        conf.override_all(
            {"status_sleep_seconds": 4, "statement_sleep_seconds": 2, "create_sql_context_timeout_seconds": 60}
        )
        session_id = "1"
        session = self._create_session(session_id=session_id, sql_created=True)
        conf.load()

        assert session.id == session_id
        assert session.started_sql_context
Exemplo n.º 28
0
    def test_delete_session_when_not_started(self):
        http_client = MagicMock()
        http_client.post.return_value = DummyResponse(201, self.session_create_json)
        conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01})
        session = self._create_session(http_client=http_client)
        conf.load()

        session.delete()

        assert_equals("dead", session._status)
        assert_equals("-1", session.id)
Exemplo n.º 29
0
    def test_is_final_status(self):
        conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01})
        session = self._create_session()
        conf.load()

        assert not session.is_final_status("idle")
        assert not session.is_final_status("starting")
        assert not session.is_final_status("busy")

        assert session.is_final_status("dead")
        assert session.is_final_status("error")
Exemplo n.º 30
0
def test_serialize_periodically():
    conf.override_all({conf.serialize_period_seconds.__name__: 0.1,
                       conf.serialize_periodically.__name__: True})
    serializer = MagicMock()
    ClientManager(serializer)

    time.sleep(0.5)

    assert serializer.serialize_state.call_count >= 1

    conf.load()
Exemplo n.º 31
0
    def test_constructor_starts_with_no_session(self):
        conf.override_all({
            "status_sleep_seconds": 4,
            "statement_sleep_seconds": 2,
            "create_sql_context_timeout_seconds": 60
        })
        session = self._create_session()
        conf.load()

        assert session.id == "-1"
        assert not session.started_sql_context
Exemplo n.º 32
0
    def test_delete_session_when_not_started(self):
        self.http_client.post_session.return_value = self.session_create_json
        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        session = self._create_session()
        conf.load()

        session.delete()

        assert_equals(session.ipython_display.send_error.call_count, 1)
Exemplo n.º 33
0
    def test_constructor_starts_with_existing_session(self):
        conf.override_all({
            "status_sleep_seconds": 4,
            "statement_sleep_seconds": 2,
            "create_sql_context_timeout_seconds": 60
        })
        session_id = "1"
        session = self._create_session(session_id=session_id, sql_created=True)
        conf.load()

        assert session.id == session_id
        assert session.started_sql_context
Exemplo n.º 34
0
def test_configuration_load():
    i = 1000
    config = {conf.statement_sleep_seconds.__name__: i}
    fsrw = MagicMock()
    fsrw.path = ""
    read_lines = MagicMock(return_value=[json.dumps(config)])
    fsrw.read_lines = read_lines
    fsrw_class = MagicMock(return_value=fsrw)
    conf.load(fsrw_class)
    assert conf._overrides is not None
    assert_equals(conf._overrides, config)
    assert_equals(conf.statement_sleep_seconds(), i)
Exemplo n.º 35
0
def test_configuration_load():
    i = 1000
    config = { conf.statement_sleep_seconds.__name__: i }
    fsrw = MagicMock()
    fsrw.path = ""
    read_lines = MagicMock(return_value=[json.dumps(config)])
    fsrw.read_lines = read_lines
    fsrw_class = MagicMock(return_value=fsrw)
    conf.load(fsrw_class)
    assert conf._overrides is not None
    assert_equals(conf._overrides, config)
    assert_equals(conf.statement_sleep_seconds(), i)
Exemplo n.º 36
0
def test_configuration_load_not_lazy():
    a = "whoops"
    config = {conf.default_chart_type.__name__: a}
    fsrw = MagicMock()
    fsrw.path = ""
    read_lines = MagicMock(return_value=[json.dumps(config)])
    fsrw.read_lines = read_lines
    fsrw_class = MagicMock(return_value=fsrw)
    conf.override_all({conf.default_chart_type.__name__: "bar"})
    conf.load(fsrw_class)
    assert conf._overrides is not None
    assert_equals(conf._overrides, config)
    assert_equals(conf.default_chart_type(), "whoops")
Exemplo n.º 37
0
def test_configuration_load_not_lazy():
    a = "whoops"
    config = { conf.events_handler_class.__name__: a }
    fsrw = MagicMock()
    fsrw.path = ""
    read_lines = MagicMock(return_value=[json.dumps(config)])
    fsrw.read_lines = read_lines
    fsrw_class = MagicMock(return_value=fsrw)
    conf.override_all({conf.events_handler_class.__name__: "bar"})
    conf.load(fsrw_class)
    assert conf._overrides is not None
    assert_equals(conf._overrides, config)
    assert_equals(conf.events_handler_class(), a)
Exemplo n.º 38
0
    def test_delete_session_when_not_started(self):
        http_client = MagicMock()
        http_client.post_session.return_value = self.session_create_json
        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        session = self._create_session(http_client=http_client)
        conf.load()

        session.delete()

        assert_equals(session.ipython_display.send_error.call_count, 1)
Exemplo n.º 39
0
    def test_logs_gets_latest_logs(self):
        http_client = MagicMock()
        http_client.post.return_value = DummyResponse(201, self.session_create_json)
        http_client.get.return_value = DummyResponse(200, self.log_json)
        conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01})
        session = self._create_session(http_client=http_client)
        conf.load()
        session.start()

        logs = session.logs

        assert_equals("hi\nhi", logs)
        http_client.get.assert_called_with("/sessions/0/log?from=0", [200])
Exemplo n.º 40
0
    def test_delete_session_when_dead_throws(self):
        http_client = MagicMock()
        http_client.post.return_value = DummyResponse(201,
                                                      self.session_create_json)
        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        session = self._create_session(http_client=http_client)
        conf.load()
        session.status = "dead"

        session.delete()
Exemplo n.º 41
0
    def test_create_sql_hive_context_happens_once(self):
        kind = Constants.session_kind_spark
        http_client = MagicMock()
        ipython_display = MagicMock()

        self.post_responses = [
            DummyResponse(201, self.session_create_json),
            DummyResponse(201, self.post_statement_json),
            DummyResponse(201, self.post_statement_json),
        ]
        http_client.post.side_effect = self._next_response_post
        self.get_responses = [
            DummyResponse(200, self.ready_sessions_json),
            DummyResponse(200, self.running_statement_json),
            DummyResponse(200, self.ready_statement_json),
            DummyResponse(200, self.ready_sessions_json),
            DummyResponse(200, self.ready_statement_json),
        ]
        http_client.get.side_effect = self._next_response_get
        conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01})
        session = self._create_session(kind=kind, http_client=http_client)
        session.ipython_display = ipython_display
        conf.load()
        session.start()

        # Reset the mock so that post called count is accurate
        http_client.reset_mock()

        session.create_sql_context()
        assert ipython_display.writeln.call_count == 3

        # Second call should not issue a post request
        session.create_sql_context()

        assert (
            call(
                "/sessions/0/statements",
                [201],
                {"code": "val sqlContext = new org.apache.spark.sql.SQLContext" "(sc)\nimport sqlContext.implicits._"},
            )
            in http_client.post.call_args_list
        )
        assert (
            call(
                "/sessions/0/statements",
                [201],
                {"code": "val hiveContext = new org.apache.spark.sql.hive.Hive" "Context(sc)"},
            )
            in http_client.post.call_args_list
        )
        assert len(http_client.post.call_args_list) == 2
Exemplo n.º 42
0
    def test_start_passes_in_all_properties(self):
        http_client = MagicMock()
        http_client.post.return_value = DummyResponse(201, self.session_create_json)

        conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01})
        kind = Constants.session_kind_spark
        properties = {"kind": kind, "extra": 1}

        ipython_display = MagicMock()
        session = LivySession(ipython_display, http_client, "-1", False, properties)
        session.start()
        conf.load()

        http_client.post.assert_called_with("/sessions", [201], properties)
Exemplo n.º 43
0
    def test_status_gets_latest_status(self):
        http_client = MagicMock()
        http_client.post.return_value = DummyResponse(201, self.session_create_json)
        http_client.get.return_value = DummyResponse(200, self.ready_sessions_json)
        conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01})
        session = self._create_session(http_client=http_client)
        conf.load()
        session.start()

        session._refresh_status()
        state = session._status

        assert_equals("idle", state)
        http_client.get.assert_called_with("/sessions/0", [200])
Exemplo n.º 44
0
    def test_delete_session_when_active(self):
        self.http_client.post_session.return_value = self.session_create_json
        self.http_client.get_session.return_value = self.ready_sessions_json
        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        session = self._create_session()
        conf.load()
        session.start(create_sql_context=False)

        session.delete()

        assert_equals("dead", session.status)
Exemplo n.º 45
0
    def test_delete_session_when_dead_throws(self):
        http_client = MagicMock()
        http_client.post.return_value = self.session_create_json
        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        session = self._create_session(http_client=http_client)
        conf.load()
        session.status = "dead"

        session.delete()

        assert_equals(session.ipython_display.send_error.call_count, 1)
Exemplo n.º 46
0
    def test_start_python_starts_session(self):
        http_client = MagicMock()
        http_client.post.return_value = DummyResponse(201, self.session_create_json)

        conf.override_all({"status_sleep_seconds": 0.01, "statement_sleep_seconds": 0.01})
        kind = Constants.session_kind_pyspark
        session = self._create_session(kind=kind, http_client=http_client)
        session.start()
        conf.load()

        assert_equals(kind, session.kind)
        assert_equals("starting", session._status)
        assert_equals("0", session.id)
        http_client.post.assert_called_with("/sessions", [201], {"kind": "pyspark"})
Exemplo n.º 47
0
 def test_create_sql_hive_context_unknown_throws(self):
     kind = "unknown"
     self.http_client.post_session.return_value = self.session_create_json
     self.http_client.post_statement.return_value = self.post_statement_json
     self.http_client.get_session.return_value = self.ready_sessions_json
     self.get_statement_responses = [self.running_statement_json, self.ready_statement_json]
     self.http_client.get_statement.side_effect = self._next_statement_response_get
     conf.override_all({
         "status_sleep_seconds": 0.01,
         "statement_sleep_seconds": 0.01
     })
     session = self._create_session(kind=kind)
     conf.load()
     session.start()
Exemplo n.º 48
0
    def test_is_final_status(self):
        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        session = self._create_session()
        conf.load()

        assert not session.is_final_status("idle")
        assert not session.is_final_status("starting")
        assert not session.is_final_status("busy")

        assert session.is_final_status("dead")
        assert session.is_final_status("error")
def test_get_config():
    usr = "******"
    pwd = "p"
    url = "url"

    config = {conf.kernel_python_credentials.__name__: {user_ev: usr, pass_ev: pwd, url_ev: url}}
    conf.override_all(config)

    u, p, r = kernel._get_configuration()

    assert u == usr
    assert p == pwd
    assert r == url

    conf.load()
Exemplo n.º 50
0
    def test_delete_session_when_active(self):
        http_client = MagicMock()
        http_client.post_session.return_value = self.session_create_json
        http_client.get_session.return_value = self.ready_sessions_json
        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        session = self._create_session(http_client=http_client)
        conf.load()
        session.start(create_sql_context=False)

        session.delete()

        assert_equals("dead", session.status)
Exemplo n.º 51
0
    def test_delete_session_when_not_started(self):
        http_client = MagicMock()
        http_client.post.return_value = DummyResponse(201,
                                                      self.session_create_json)
        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        session = self._create_session(http_client=http_client)
        conf.load()

        session.delete()

        assert_equals("dead", session.status)
        assert_equals("-1", session.id)
Exemplo n.º 52
0
    def test_status_gets_latest_status(self):
        http_client = MagicMock()
        http_client.post_session.return_value = self.session_create_json
        http_client.get_session.return_value = self.ready_sessions_json
        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        session = self._create_session(http_client=http_client)
        conf.load()
        session.start(create_sql_context=False)

        session.refresh_status()
        state = session.status

        assert_equals("idle", state)
        http_client.get_session.assert_called_with(0)
Exemplo n.º 53
0
    def test_logs_gets_latest_logs(self):
        http_client = MagicMock()
        http_client.post_session.return_value = self.session_create_json
        http_client.get_session.return_value = self.ready_sessions_json
        http_client.get_all_session_logs.return_value = self.log_json
        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        session = self._create_session(http_client=http_client)
        conf.load()
        session.start(create_sql_context=False)

        logs = session.get_logs()

        assert_equals("hi\nhi", logs)
        http_client.get_all_session_logs.assert_called_with(0)
Exemplo n.º 54
0
    def test_logs_gets_latest_logs(self):
        http_client = MagicMock()
        http_client.post.return_value = DummyResponse(201,
                                                      self.session_create_json)
        http_client.get.return_value = DummyResponse(200, self.log_json)
        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        session = self._create_session(http_client=http_client)
        conf.load()
        session.start()

        logs = session.get_logs()

        assert_equals("hi\nhi", logs)
        http_client.get.assert_called_with("/sessions/0/log?from=0", [200])
Exemplo n.º 55
0
    def test_create_sql_hive_context_happens_once(self):
        kind = constants.SESSION_KIND_SPARK
        http_client = MagicMock()
        ipython_display = MagicMock()

        http_client.post_session.return_value = self.session_create_json
        self.post_statement_responses = [
            self.post_statement_json, self.post_statement_json
        ]
        http_client.post_statement.side_effect = self._next_statement_response_post
        http_client.get_session.return_value = self.ready_sessions_json
        self.get_statement_responses = [
            self.running_statement_json, self.ready_statement_json,
            self.ready_statement_json
        ]
        http_client.get_statement.side_effect = self._next_statement_response_get
        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        session = self._create_session(kind=kind, http_client=http_client)
        session.ipython_display = ipython_display
        conf.load()
        session.start(create_sql_context=False)

        # Reset the mock so that post called count is accurate
        http_client.reset_mock()

        session.create_sql_context()
        assert ipython_display.writeln.call_count == 2
        assert session.created_sql_context

        # Second call should not issue a post request
        session.create_sql_context()

        assert call(0, {"code": "val sqlContext = new org.apache.spark.sql.hive.HiveContext(sc)"}) \
               in http_client.post_statement.call_args_list
        assert len(http_client.post_statement.call_args_list) == 1

        session._get_sql_context_creation_command = MagicMock()

        session._get_sql_context_creation_command.return_value.execute.return_value = (
            True, "Success")
        session.created_sql_context = None
        session.create_sql_context()
        assert session.created_sql_context
Exemplo n.º 56
0
 def test_create_sql_hive_context_unknown_throws(self):
     kind = "unknown"
     http_client = MagicMock()
     http_client.post_session.return_value = self.session_create_json
     http_client.post_statement.return_value = self.post_statement_json
     http_client.get_session.return_value = self.ready_sessions_json
     self.get_statement_responses = [
         self.running_statement_json, self.ready_statement_json
     ]
     http_client.get_statement.side_effect = self._next_statement_response_get
     conf.override_all({
         "status_sleep_seconds": 0.01,
         "statement_sleep_seconds": 0.01
     })
     session = self._create_session(kind=kind, http_client=http_client)
     conf.load()
     session.start()
Exemplo n.º 57
0
    def test_start_passes_in_all_properties(self):
        http_client = MagicMock()
        http_client.post_session.return_value = self.session_create_json
        http_client.get_session.return_value = self.ready_sessions_json

        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        kind = constants.SESSION_KIND_SPARK
        properties = {"kind": kind, "extra": 1}

        ipython_display = MagicMock()
        session = LivySession(http_client, properties, ipython_display)
        session.start(create_sql_context=False)
        conf.load()

        http_client.post_session.assert_called_with(properties)
Exemplo n.º 58
0
    def test_start_passes_in_all_properties(self):
        http_client = MagicMock()
        http_client.post.return_value = DummyResponse(201,
                                                      self.session_create_json)

        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        kind = constants.SESSION_KIND_SPARK
        properties = {"kind": kind, "extra": 1}

        ipython_display = MagicMock()
        session = LivySession(http_client, properties, ipython_display)
        session.start()
        conf.load()

        http_client.post.assert_called_with("/sessions", [201], properties)
Exemplo n.º 59
0
    def test_create_sql_hive_context_happens_once(self):
        kind = constants.SESSION_KIND_SPARK
        http_client = MagicMock()
        ipython_display = MagicMock()

        self.post_responses = [
            DummyResponse(201, self.session_create_json),
            DummyResponse(201, self.post_statement_json),
            DummyResponse(201, self.post_statement_json)
        ]
        http_client.post.side_effect = self._next_response_post
        self.get_responses = [
            DummyResponse(200, self.ready_sessions_json),
            DummyResponse(200, self.running_statement_json),
            DummyResponse(200, self.ready_statement_json),
            DummyResponse(200, self.ready_sessions_json),
            DummyResponse(200, self.ready_statement_json)
        ]
        http_client.get.side_effect = self._next_response_get
        conf.override_all({
            "status_sleep_seconds": 0.01,
            "statement_sleep_seconds": 0.01
        })
        session = self._create_session(kind=kind, http_client=http_client)
        session.ipython_display = ipython_display
        conf.load()
        session.start()

        # Reset the mock so that post called count is accurate
        http_client.reset_mock()

        session.create_sql_context()
        assert ipython_display.writeln.call_count == 2

        # Second call should not issue a post request
        session.create_sql_context()

        assert call(
            "/sessions/0/statements", [201], {
                "code":
                "val sqlContext = new org.apache.spark.sql.hive.Hive"
                "Context(sc)"
            }) in http_client.post.call_args_list
        assert len(http_client.post.call_args_list) == 1