def test_that_retrain_calls_eval_method_correctly(mocker):
mock_classifier = mocker.Mock()
mock_classifier.fit == mocker.Mock()
mock_classifier.predict_proba == mocker.Mock()
mock_eval_method = mocker.Mock(
return_value={"test_score": np.array([0.8])}
)
test_array = np.array([[1, 2, 3], [1, 2, 3], [1, 2, 3]])
widget = SemiSupervisor(
features=test_array,
classifier=mock_classifier,
eval_method=mock_eval_method,
)
widget._annotation_loop.send({"source": "", "value": "dummy label 1"})
widget._annotation_loop.send({"source": "", "value": "dummy label 2"})
widget.retrain()
assert mock_eval_method.call_count == 1
call_arguments = mock_eval_method.call_args[0]
assert call_arguments[0] is mock_classifier
assert (call_arguments[1] == test_array[:2, :]).all()
assert pytest.helpers.same_elements(
call_arguments[2], ["dummy label 1", "dummy label 2"]
)
assert widget.model_performance.value == "Score: 0.80"
def test_that_orchestration_sleeps_enough(mocker):
widget = SemiSupervisor()
# patch methods not to be used:
mocker.patch.object(widget, "retrain")
widget.model_performance = mocker.MagicMock(
**{"model_performance.value": "nice"}
)
t0 = time()
widget.orchestrate(max_runs=2, interval_seconds=0.01, interval_n_labels=0)
t1 = time()
assert t1 - t0 > 0.02
def test_that_run_orchestration_with_none_runs_orchestrator_once(mocker):
widget = SemiSupervisor()
mocker.patch.object(widget, "_run_orchestration", return_value=1)
dummy_html_widget = namedtuple("dummy_html_widget", ["value"])
widget.model_performance = dummy_html_widget(
value="test model performance"
)
# t0 = time()
widget.orchestrate(interval_seconds=0.1, max_runs=2)
assert widget._run_orchestration.call_count == 2
def test_that_eval_method_is_set_correctly(mocker):
mock_eval_method = mocker.Mock()
# test the default
widget = SemiSupervisor()
assert widget.eval_method.func is cross_validate
# test the normal method
widget = SemiSupervisor(eval_method=mock_eval_method)
assert widget.eval_method is mock_eval_method
# test the unhappy case
with pytest.raises(ValueError):
widget = SemiSupervisor(eval_method="not a callable")
def test_that_run_orchestration_doesnt_rerun_if_not_enough_new_labels(mocker):
widget = SemiSupervisor()
# patch the nr of provided labels:
mocker.patch.object(widget, "queue")
widget.queue.configure_mock(**{"_labelled_count.side_effect": [0, 5, 10]})
# patch methods not to be used:
mocker.patch.object(widget, "retrain")
widget.model_performance = mocker.MagicMock(
**{"model_performance.value": "nice"}
)
# print([widget.queue._labelled_count() for i in range(3)])
# raise ValueError
widget.orchestrate(max_runs=2, interval_seconds=0, interval_n_labels=10)
assert widget.queue._labelled_count.call_count == 3
assert widget.retrain.call_count == 2
def test_that_supplied_features_are_passed_to_queue(mocker):
test_array = np.array([[1, 2, 3], [1, 2, 3]])
mock_queue = mocker.patch(
"superintendent.distributed.semisupervisor.DatabaseQueue.enqueue_many"
)
widget = SemiSupervisor(features=test_array, labels=None) # noqa
mock_queue.assert_called_once_with(test_array, labels=None)
def test_that_supplied_labels_are_passed_to_queue(mocker):
test_array = np.array([[1, 2, 3], [1, 2, 3]])
test_labels = np.array(["hi", "hello"])
mock_queue = mocker.patch(
"superintendent.distributed.semisupervisor.DatabaseQueue.enqueue_many"
)
widget = SemiSupervisor(features=test_array, labels=test_labels) # noqa
args, kwargs = mock_queue.call_args
assert pytest.helpers.exact_element_match(test_array, args[0])
assert pytest.helpers.exact_element_match(test_labels, kwargs["labels"])
def test_that_sending_labels_into_iterator_submits_them_to_queue(mocker):
mock_submit = mocker.patch(
"superintendent.distributed.semisupervisor.DatabaseQueue.submit"
)
test_array = np.array([[1, 2, 3], [1, 2, 3]])
widget = SemiSupervisor(features=test_array)
widget._annotation_loop.send({"source": "", "value": "dummy label"})
mock_submit.assert_called_once_with(1, "dummy label")
def test_that_get_worker_id_sets_correct_children():
widget = SemiSupervisor(worker_id=True)
assert any(
isinstance(child, ipywidgets.Text)
for child in pytest.helpers.recursively_list_widget_children(
widget.layout
)
)
worker_id_text_field = next(
child
for child in pytest.helpers.recursively_list_widget_children(
widget.layout
)
if isinstance(child, ipywidgets.Text)
)
worker_id_text_field.value = "test worker id"
widget._set_worker_id(worker_id_text_field)
assert widget.queue.worker_id == "test worker id"
def test_that_sending_undo_into_iterator_calls_undo_on_queue(mocker):
mock_undo = mocker.patch(
"superintendent.distributed.semisupervisor.DatabaseQueue.undo"
)
test_array = np.array([[1, 2, 3], [1, 2, 3]])
widget = SemiSupervisor(features=test_array)
mock_undo.reset_mock()
widget._annotation_loop.send({"source": "__undo__", "value": None})
assert mock_undo.call_count == 2
def test_that_reorder_is_set_correctly(mocker):
mock_reorder_method = mocker.Mock()
# test the default
widget = SemiSupervisor()
assert widget.reorder is None
# test passing a string
widget = SemiSupervisor(reorder="entropy")
assert widget.reorder is superintendent.prioritisation.entropy
# test a function
widget = SemiSupervisor(reorder=mock_reorder_method)
assert widget.reorder is mock_reorder_method
# test the unhappy case
with pytest.raises(NotImplementedError):
widget = SemiSupervisor(reorder="dummy function name")
with pytest.raises(ValueError):
widget = SemiSupervisor(reorder=1)
def test_that_classifier_is_set_correctly(mocker):
mock_classifier = mocker.Mock()
mock_classifier.fit == "dummy fit"
mock_classifier.predict_proba == "dummy predict"
mock_on_click = mocker.patch("ipywidgets.Button.on_click")
widget = SemiSupervisor(classifier=mock_classifier)
assert widget.classifier is mock_classifier
assert hasattr(widget, "retrain_button")
assert isinstance(widget.retrain_button, ipywidgets.Button)
assert ((widget.retrain,),) in mock_on_click.call_args_list
def test_that_run_orchestration_calls_retrain_and_prints(mocker):
mock_value = mocker.patch.object(
ipywidgets.HTML, "value", value="test model performance"
)
mock_print = mocker.patch(
"superintendent.distributed.semisupervisor.print"
)
mock_sleep = mocker.patch("time.sleep")
widget = SemiSupervisor(worker_id=True)
dummy_html_widget = namedtuple("dummy_html_widget", ["value"])
widget.model_performance = dummy_html_widget(
value="test model performance"
)
mock_retrain = mocker.patch.object(widget, "retrain")
widget._run_orchestration()
mock_retrain.assert_called_once_with()
mock_print.assert_called_once_with("test model performance")
mock_sleep.assert_called_once_with(60)
def test_that_retrain_calls_reorder_correctly(mocker):
test_probabilities = np.array([[0.2, 0.3], [0.1, 0.4]])
mock_eval_method = mocker.Mock(
return_value={"test_score": np.array([0.8])}
)
mock_reordering = mocker.Mock(return_value=[0, 1])
test_array = np.array([[1, 2, 3], [1, 2, 3], [1, 2, 3], [1, 2, 3]])
widget = SemiSupervisor(
features=test_array,
classifier=LogisticRegression(),
eval_method=mock_eval_method,
reorder=mock_reordering,
shuffle_prop=0.2,
)
mocker.patch.object(
widget.classifier, "fit", return_value=LogisticRegression()
)
mocker.patch.object(
widget.classifier, "predict_proba", return_value=test_probabilities
)
widget._annotation_loop.send({"source": "", "value": "dummy label 1"})
widget._annotation_loop.send({"source": "", "value": "dummy label 2"})
widget.retrain()
assert mock_reordering.call_count == 1
call_args, call_kwargs = mock_reordering.call_args_list[0]
assert (call_args[0] == test_probabilities).all()
assert call_kwargs["shuffle_prop"] == 0.2
def test_that_progressbar_value_is_updated_and_render_finished_called(mocker):
mock_render_finished = mocker.patch(
"superintendent.distributed.semisupervisor"
".SemiSupervisor._render_finished"
)
test_array = np.array([[1, 2, 3], [1, 2, 3]])
widget = SemiSupervisor(features=test_array)
assert widget.progressbar.value == 0
widget._annotation_loop.send({"source": "", "value": "dummy label"})
assert widget.progressbar.value == 0.5
widget._annotation_loop.send({"source": "", "value": "dummy label"})
assert widget.progressbar.value == 1
assert mock_render_finished.call_count == 1
def test_that_sending_skip_calls_no_queue_method(mocker):
mock_undo = mocker.patch(
"superintendent.distributed.semisupervisor.DatabaseQueue.undo"
)
mock_submit = mocker.patch(
"superintendent.queueing.SimpleLabellingQueue.submit"
)
test_array = np.array([[1, 2, 3], [1, 2, 3]])
widget = SemiSupervisor(features=test_array)
mock_undo.reset_mock()
mock_submit.reset_mock()
widget._annotation_loop.send({"source": "__skip__", "value": None})
assert mock_undo.call_count == 0
assert mock_submit.call_count == 0
def test_that_retrain_with_no_labels_sets_warnings(mocker):
mock_classifier = mocker.Mock()
mock_classifier.fit == mocker.Mock()
mock_classifier.predict_proba == mocker.Mock()
mock_eval_method = mocker.Mock(
return_value={"test_score": np.array([0.8])}
)
test_array = np.array([[1, 2, 3], [1, 2, 3], [1, 2, 3]])
widget = SemiSupervisor(
features=test_array,
classifier=mock_classifier,
eval_method=mock_eval_method,
)
widget.retrain()
assert (
widget.model_performance.value
== "Score: Not enough labels to retrain."
)
widget._annotation_loop.send({"source": "", "value": "dummy label 1"})
widget.retrain()
assert (
widget.model_performance.value
== "Score: Not enough labels to retrain."
)
widget._annotation_loop.send({"source": "", "value": "dummy label 2"})
widget.retrain()
assert mock_eval_method.call_count == 1
assert widget.model_performance.value == "Score: 0.80"
def test_that_shuffle_prop_is_set_correctly(shuffle_prop):
widget = SemiSupervisor(shuffle_prop=shuffle_prop)
assert widget.shuffle_prop == shuffle_prop
def test_that_creating_a_widget_works():
widget = SemiSupervisor() # noqa
def test_that_calling_retrain_without_classifier_breaks():
with pytest.raises(ValueError):
test_array = np.array([[1, 2, 3], [1, 2, 3], [1, 2, 3]])
widget = SemiSupervisor(features=test_array)
widget.retrain()
