def test_adjudicate_combined_remove_unadj(self):
"""
Test combining adjudication switching with un-adjudication.
"""
iqrs = IqrSession()
# Set initial state
p0 = DescriptorMemoryElement('', 0).set_vector([0])
p1 = DescriptorMemoryElement('', 1).set_vector([1])
p2 = DescriptorMemoryElement('', 2).set_vector([2])
n3 = DescriptorMemoryElement('', 3).set_vector([3])
n4 = DescriptorMemoryElement('', 4).set_vector([4])
# Set initial state
iqrs.positive_descriptors = {p0, p1, p2}
iqrs.negative_descriptors = {n3, n4}
# Add p5, switch p1 to negative, unadj p2
p5 = DescriptorMemoryElement('', 5).set_vector([5])
iqrs.adjudicate(new_positives=[p5],
new_negatives=[p1],
un_positives=[p2])
assert iqrs.positive_descriptors == {p0, p5}
assert iqrs.negative_descriptors == {n3, n4, p1}
# Add n6, switch n4 to positive, unadj n3
n6 = DescriptorMemoryElement('', 6).set_vector([6])
iqrs.adjudicate(new_positives=[n4],
new_negatives=[n6],
un_negatives=[n3])
assert iqrs.positive_descriptors == {p0, p5, n4}
assert iqrs.negative_descriptors == {p1, n6}
def test_adjudicate_unadj_noeffect(self):
"""
Test that an empty call, or un-adjudicating a descriptor that is not
currently marked as a positive or negative, causes no state change.
"""
iqrs = IqrSession()
# Set initial state
p0 = DescriptorMemoryElement('', 0).set_vector([0])
p1 = DescriptorMemoryElement('', 1).set_vector([1])
p2 = DescriptorMemoryElement('', 2).set_vector([2])
n3 = DescriptorMemoryElement('', 3).set_vector([3])
n4 = DescriptorMemoryElement('', 4).set_vector([4])
# Set initial state
iqrs.positive_descriptors = {p0, p1, p2}
iqrs.negative_descriptors = {n3, n4}
# Empty adjudication
iqrs.adjudicate()
assert iqrs.positive_descriptors == {p0, p1, p2}
assert iqrs.negative_descriptors == {n3, n4}
# Attempt un-adjudication of a non-adjudicated element.
e = DescriptorMemoryElement('', 5).set_vector([5])
iqrs.adjudicate(un_positives=[e], un_negatives=[e])
assert iqrs.positive_descriptors == {p0, p1, p2}
assert iqrs.negative_descriptors == {n3, n4}
def test_adjudicate_cache_resetting_negative(self):
"""
Test results view cache resetting functionality on adjudicating certain
ways.
"""
e = DescriptorMemoryElement('', 0).set_vector([0])
iqrs = IqrSession()
iqrs._ordered_pos = True
iqrs._ordered_neg = True
iqrs._ordered_non_adj = True
# Check that adding a positive adjudication resets the positive and
# non-adjudicated result caches.
iqrs.adjudicate(new_negatives=[e])
assert iqrs._ordered_pos is True # NOT reset
assert iqrs._ordered_neg is None # reset
assert iqrs._ordered_non_adj is None # reset
def test_adjudication_cache_not_reset(self):
"""
Test that pos/neg/non-adj result caches are NOT reset when no state
change occurs under different circumstances
"""
# setup initial IQR session state.
p0 = DescriptorMemoryElement('', 0).set_vector([0])
p1 = DescriptorMemoryElement('', 1).set_vector([1])
p2 = DescriptorMemoryElement('', 2).set_vector([2])
n3 = DescriptorMemoryElement('', 3).set_vector([3])
n4 = DescriptorMemoryElement('', 4).set_vector([4])
iqrs = IqrSession()
iqrs.positive_descriptors = {p0, p1, p2}
iqrs.negative_descriptors = {n3, n4}
iqrs._ordered_pos = iqrs._ordered_neg = iqrs._ordered_non_adj = True
# Empty adjudication
iqrs.adjudicate()
assert iqrs._ordered_pos is True
assert iqrs._ordered_neg is True
assert iqrs._ordered_non_adj is True
# Repeat positive/negative adjudication
iqrs.adjudicate(new_positives=[p0])
assert iqrs._ordered_pos is True
assert iqrs._ordered_neg is True
assert iqrs._ordered_non_adj is True
iqrs.adjudicate(new_negatives=[n3])
assert iqrs._ordered_pos is True
assert iqrs._ordered_neg is True
assert iqrs._ordered_non_adj is True
iqrs.adjudicate(new_positives=[p1], new_negatives=[n4])
assert iqrs._ordered_pos is True
assert iqrs._ordered_neg is True
assert iqrs._ordered_non_adj is True
# No-op un-adjudication
e = DescriptorMemoryElement('', 5).set_vector([5])
iqrs.adjudicate(un_positives=[e], un_negatives=[e])
assert iqrs._ordered_pos is True
assert iqrs._ordered_neg is True
assert iqrs._ordered_non_adj is True
def test_adjudication_switch(self):
"""
Test providing positives and negatives on top of an existing state such
that the descriptor adjudications are reversed. (what was once positive
is now negative, etc.)
"""
iqrs = IqrSession()
p0 = DescriptorMemoryElement('', 0).set_vector([0])
p1 = DescriptorMemoryElement('', 1).set_vector([1])
p2 = DescriptorMemoryElement('', 2).set_vector([2])
n3 = DescriptorMemoryElement('', 3).set_vector([3])
n4 = DescriptorMemoryElement('', 4).set_vector([4])
# Set initial state
iqrs.positive_descriptors = {p0, p1, p2}
iqrs.negative_descriptors = {n3, n4}
# Adjudicate, partially swapping adjudications individually
iqrs.adjudicate(new_positives=[n3])
assert iqrs.positive_descriptors == {p0, p1, p2, n3}
assert iqrs.negative_descriptors == {n4}
iqrs.adjudicate(new_negatives=[p1])
assert iqrs.positive_descriptors == {p0, p2, n3}
assert iqrs.negative_descriptors == {n4, p1}
# Adjudicate swapping remaining at the same time
iqrs.adjudicate(new_positives=[n4], new_negatives=[p0, p2])
assert iqrs.positive_descriptors == {n3, n4}
assert iqrs.negative_descriptors == {p0, p1, p2}
def test_adjudicate_add_duplicates(self):
"""
Test that adding duplicate descriptors as positive or negative
adjudications has no effect as the behavior of sets should be observed.
"""
iqrs = IqrSession()
p0 = DescriptorMemoryElement('', 0).set_vector([0])
p2 = DescriptorMemoryElement('', 2).set_vector([2])
n1 = DescriptorMemoryElement('', 1).set_vector([1])
p3 = DescriptorMemoryElement('', 3).set_vector([3])
n4 = DescriptorMemoryElement('', 4).set_vector([4])
# Partially add the above descriptors
iqrs.adjudicate(new_positives=[p0], new_negatives=[n1])
assert iqrs.positive_descriptors == {p0}
assert iqrs.negative_descriptors == {n1}
# Add all descriptors, observing that that already added descriptors
# are ignored.
iqrs.adjudicate(new_positives=[p0, p2, p3], new_negatives=[n1, n4])
assert iqrs.positive_descriptors == {p0, p2, p3}
assert iqrs.negative_descriptors == {n1, n4}
# Duplicate previous call so no new descriptors are added. No change or
# issue should be observed.
iqrs.adjudicate(new_positives=[p0, p2, p3], new_negatives=[n1, n4])
assert iqrs.positive_descriptors == {p0, p2, p3}
assert iqrs.negative_descriptors == {n1, n4}
def test_adjudicate_remove_pos_neg(self):
"""
Test that we can remove positive and negative adjudications using
"un_*" parameters.
"""
iqrs = IqrSession()
# Set initial state
p0 = DescriptorMemoryElement('', 0).set_vector([0])
p1 = DescriptorMemoryElement('', 1).set_vector([1])
p2 = DescriptorMemoryElement('', 2).set_vector([2])
n3 = DescriptorMemoryElement('', 3).set_vector([3])
n4 = DescriptorMemoryElement('', 4).set_vector([4])
# Set initial state
iqrs.positive_descriptors = {p0, p1, p2}
iqrs.negative_descriptors = {n3, n4}
# "Un-Adjudicate" descriptors individually
iqrs.adjudicate(un_positives=[p1])
assert iqrs.positive_descriptors == {p0, p2}
assert iqrs.negative_descriptors == {n3, n4}
iqrs.adjudicate(un_negatives=[n3])
assert iqrs.positive_descriptors == {p0, p2}
assert iqrs.negative_descriptors == {n4}
# "Un-Adjudicate" collectively
iqrs.adjudicate(un_positives=[p0, p2], un_negatives=[n4])
assert iqrs.positive_descriptors == set()
assert iqrs.negative_descriptors == set()
def test_add_iqr_state_classifier_simple(self):
"""
Test calling IQR classifier add endpoint with a simple IQR Session
serialization.
"""
# Make a simple session with dummy adjudication descriptor elements
iqrs = IqrSession(session_uid=str("0"))
iqr_p1 = DescriptorMemoryElement('test', 0).set_vector([0])
iqr_n1 = DescriptorMemoryElement('test', 1).set_vector([1])
iqrs.adjudicate(new_positives=[iqr_p1], new_negatives=[iqr_n1])
test_iqrs_b64 = base64.b64encode(iqrs.get_state_bytes())
test_label = 'test-label-08976azsdv'
with mock.patch(STUB_CLASSIFIER_MOD_PATH +
".DummySupervisedClassifier._train") as m_cfier_train:
with self.app.test_client() as cli:
rv = cli.post('/iqr_classifier',
data={
'bytes_b64': test_iqrs_b64,
'label': test_label,
})
self.assertStatus(rv, 201)
self.assertResponseMessageRegex(
rv, "Finished training "
"IQR-session-based "
"classifier for label "
"'%s'." % test_label)
m_cfier_train.assert_called_once_with({
'positive': {iqr_p1},
'negative': {iqr_n1}
})
# Collection should include initial dummy classifier and new iqr
# classifier.
self.assertEqual(len(self.app.classifier_collection.labels()), 2)
self.assertIn(test_label, self.app.classifier_collection.labels())
def test_adjudicate_both_labels(self):
"""
Test that providing a descriptor element as both a positive AND
negative adjudication causes no state change..
"""
iqrs = IqrSession()
# Set initial state
p0 = DescriptorMemoryElement('', 0).set_vector([0])
p1 = DescriptorMemoryElement('', 1).set_vector([1])
p2 = DescriptorMemoryElement('', 2).set_vector([2])
n3 = DescriptorMemoryElement('', 3).set_vector([3])
n4 = DescriptorMemoryElement('', 4).set_vector([4])
# Set initial state
iqrs.positive_descriptors = {p0, p1, p2}
iqrs.negative_descriptors = {n3, n4}
# Attempt adjudicating a new element as both postive AND negative
e = DescriptorMemoryElement('', 5).set_vector([5])
iqrs.adjudicate(new_positives=[e], new_negatives=[e])
assert iqrs.positive_descriptors == {p0, p1, p2}
assert iqrs.negative_descriptors == {n3, n4}
def test_add_iqr_state_classifier_simple(self):
"""
Test calling IQR classifier add endpoint with a simple IQR Session
serialization.
"""
# Make a simple session with dummy adjudication descriptor elements
iqrs = IqrSession(session_uid=str("0"))
iqr_p1 = DescriptorMemoryElement('test', 0).set_vector([0])
iqr_n1 = DescriptorMemoryElement('test', 1).set_vector([1])
iqrs.adjudicate(
new_positives=[iqr_p1], new_negatives=[iqr_n1]
)
test_iqrs_b64 = base64.b64encode(iqrs.get_state_bytes())
test_label = 'test-label-08976azsdv'
with mock.patch(STUB_CLASSIFIER_MOD_PATH +
".DummySupervisedClassifier._train") as m_cfier_train:
with self.app.test_client() as cli:
rv = cli.post('/iqr_classifier', data={
'bytes_b64': test_iqrs_b64,
'label': test_label,
})
self.assertStatus(rv, 201)
self.assertResponseMessageRegex(rv, "Finished training "
"IQR-session-based "
"classifier for label "
"'%s'." % test_label)
m_cfier_train.assert_called_once_with(
{'positive': {iqr_p1}, 'negative': {iqr_n1}}
)
# Collection should include initial dummy classifier and new iqr
# classifier.
self.assertEqual(len(self.app.classifier_collection.labels()), 2)
self.assertIn(test_label, self.app.classifier_collection.labels())
def test_adjudicate_new_pos_neg(self):
"""
Test that providing iterables to ``new_positives`` and
``new_negatives`` parameters result in additions to the positive and
negative sets respectively.
"""
iqrs = IqrSession()
p0 = DescriptorMemoryElement('', 0).set_vector([0])
iqrs.adjudicate(new_positives=[p0])
assert iqrs.positive_descriptors == {p0}
assert iqrs.negative_descriptors == set()
n1 = DescriptorMemoryElement('', 1).set_vector([1])
iqrs.adjudicate(new_negatives=[n1])
assert iqrs.positive_descriptors == {p0}
assert iqrs.negative_descriptors == {n1}
p2 = DescriptorMemoryElement('', 2).set_vector([2])
p3 = DescriptorMemoryElement('', 3).set_vector([3])
n4 = DescriptorMemoryElement('', 4).set_vector([4])
iqrs.adjudicate(new_positives=[p2, p3], new_negatives=[n4])
assert iqrs.positive_descriptors == {p0, p2, p3}
assert iqrs.negative_descriptors == {n1, n4}
def test_refine_with_prev_results(self):
"""
Test that the results of RelevancyIndex ranking are directly reflected
in an existing results dictionary of probability values.
"""
# IqrSession instance. No config for rel_index because we will mock
# that.
iqrs = IqrSession()
# Mock relevancy index in order to check how its called and mock return
# value.
iqrs.rel_index = mock.MagicMock(spec=RelevancyIndex)
# Mock length to be non-zero to simulate it having contents
iqrs.rel_index.__len__.return_value = 1
test_in_pos_elem = DescriptorMemoryElement('t', 0).set_vector([0])
test_in_neg_elem = DescriptorMemoryElement('t', 1).set_vector([1])
test_ex_pos_elem = DescriptorMemoryElement('t', 2).set_vector([2])
test_ex_neg_elem = DescriptorMemoryElement('t', 3).set_vector([3])
test_other_elem = DescriptorMemoryElement('t', 4).set_vector([4])
# Mock return dictionary, probabilities don't matter much other than
# they are not 1.0 or 0.0.
iqrs.rel_index.rank.return_value = \
{e: 0.5 for e in [test_in_pos_elem, test_in_neg_elem,
test_other_elem]}
# Create a "previous state" of the results dictionary containing
# results from our "working set" of descriptor elements.
iqrs.results = {
test_in_pos_elem: 0.2,
test_in_neg_elem: 0.2,
test_other_elem: 0.2,
# ``refine`` replaces the previous dict, so disjoint keys are
# NOT retained.
'something else': 0.3,
}
# Prepare IQR state for refinement
# - set dummy internal/external positive negatives.
iqrs.external_descriptors(positive=[test_ex_pos_elem],
negative=[test_ex_neg_elem])
iqrs.adjudicate(new_positives=[test_in_pos_elem],
new_negatives=[test_in_neg_elem])
# Test calling refine method
iqrs.refine()
# We test that:
# - ``rel_index.rank`` called with the combination of
# external/adjudicated descriptor elements.
# - ``results`` attribute now has an dict value
# - value of ``results`` attribute is what we expect.
iqrs.rel_index.rank.assert_called_once_with(
{test_in_pos_elem, test_ex_pos_elem},
{test_in_neg_elem, test_ex_neg_elem},
)
assert iqrs.results is not None
assert len(iqrs.results) == 3
assert test_other_elem in iqrs.results
assert test_in_pos_elem in iqrs.results
assert test_in_neg_elem in iqrs.results
assert 'something else' not in iqrs.results
assert iqrs.results[test_other_elem] == 0.5
assert iqrs.results[test_in_pos_elem] == 0.5
assert iqrs.results[test_in_neg_elem] == 0.5
def test_refine_no_prev_results(self):
"""
Test that the results of RelevancyIndex ranking are directly reflected
in a new results dictionary of probability values, even for elements
that were also used in adjudication.
This test is useful because a previous state of the IQR Session
structure would force return probabilities for some descriptor elements
to certain values if they were also present in the positive or negative
adjudicate (internal or external) sets.
"""
# IqrSession instance. No config for rel_index because we will mock
# that.
iqrs = IqrSession()
# Mock relevancy index in order to check how its called and mock return
# value.
iqrs.rel_index = mock.MagicMock(spec=RelevancyIndex)
# Mock length to be non-zero to simulate it having contents
iqrs.rel_index.__len__.return_value = 1
test_in_pos_elem = DescriptorMemoryElement('t', 0).set_vector([0])
test_in_neg_elem = DescriptorMemoryElement('t', 1).set_vector([1])
test_ex_pos_elem = DescriptorMemoryElement('t', 2).set_vector([2])
test_ex_neg_elem = DescriptorMemoryElement('t', 3).set_vector([3])
test_other_elem = DescriptorMemoryElement('t', 4).set_vector([4])
# Mock return dictionary, probabilities don't matter much other than
# they are not 1.0 or 0.0.
iqrs.rel_index.rank.return_value = \
{e: 0.5 for e in [test_in_pos_elem, test_in_neg_elem,
test_other_elem]}
# Asserting expected pre-condition where there are no results yet.
assert iqrs.results is None
# Prepare IQR state for refinement
# - set dummy internal/external positive negatives.
iqrs.external_descriptors(positive=[test_ex_pos_elem],
negative=[test_ex_neg_elem])
iqrs.adjudicate(new_positives=[test_in_pos_elem],
new_negatives=[test_in_neg_elem])
# Test calling refine method
iqrs.refine()
# We test that:
# - ``rel_index.rank`` called with the combination of
# external/adjudicated descriptor elements.
# - ``results`` attribute now has a dict value
# - value of ``results`` attribute is what we expect.
iqrs.rel_index.rank.assert_called_once_with(
{test_in_pos_elem, test_ex_pos_elem},
{test_in_neg_elem, test_ex_neg_elem},
)
assert iqrs.results is not None
assert len(iqrs.results) == 3
assert test_other_elem in iqrs.results
assert test_in_pos_elem in iqrs.results
assert test_in_neg_elem in iqrs.results
assert iqrs.results[test_other_elem] == 0.5
assert iqrs.results[test_in_pos_elem] == 0.5
assert iqrs.results[test_in_neg_elem] == 0.5
