def test_create_or_update_submission(self):
r1 = Round.create(num=0)
team = Team.create(name=Team.OUR_NAME)
cs = ChallengeSet.create(name="foo")
cbn1 = ChallengeBinaryNode.create(name="foo", cs=cs, blob="BLOB")
cbn2 = ChallengeBinaryNode.create(name="foo", cs=cs, blob="BLOB2")
assert_equals(len(cs.fieldings), 0)
# Submit 2 patches at once
ChallengeSetFielding.create_or_update_submission(team=team,
cbns=[cbn1, cbn2],
round=r1)
assert_equals(len(cs.fieldings), 1)
assert_equals(len(cs.fieldings.get().cbns), 2)
assert_items_equal(cs.fieldings.get().cbns, [cbn1, cbn2])
assert_equals(cs.fieldings.get().submission_round,
Round.current_round())
assert_is_none(cs.fieldings.get().available_round)
# Submit again fails
cbn3 = ChallengeBinaryNode.create(name="foo3", cs=cs, blob="BLOB3")
ChallengeSetFielding.create_or_update_submission(team=team,
cbns=[cbn3],
round=r1)
assert_equals(len(cs.fieldings), 1)
assert_items_equal(cs.fieldings.get().cbns, [cbn3])
def test_esimated_cb_score(self):
r6 = Round.create(num=6)
team = Team.get_our()
cs = ChallengeSet.create(name="foo3")
cbn1 = ChallengeBinaryNode.create(name="foo_3",
cs=cs,
sha256="sum3",
blob="asdf")
pt = PatchType.create(name="patch1",
functionality_risk=0.0,
exploitability=0.2)
# PatchType exists, but no patched binary
assert_is_none(cbn1.estimated_cb_score)
# Patched
cbn2 = ChallengeBinaryNode.create(name="foo_3",
cs=cs,
sha256="sum4",
blob="asdf",
patch_type=pt)
pf = PollFeedback.create(cs=cs,
round=r6,
success=1.0,
timeout=0,
connect=0,
function=0,
time_overhead=0.0,
memory_overhead=0.0)
perf_score = {
'score': {
'rep': {
'task_clock': 4,
'rss': 3,
'flt': 1,
'file_size': 5
},
'ref': {
'task_clock': 4,
'rss': 3,
'flt': 2,
'file_size': 6
}
}
}
ps = PatchScore.create(cs=cs,
round=r6,
num_polls=1,
perf_score=perf_score,
patch_type=pt)
csf_orig = ChallengeSetFielding.create(cs=cs,
cbns=[cbn2],
team=team,
available_round=r6,
submission_round=r6,
poll_feedback=pf)
assert_almost_equals(cbn2.estimated_cb_score, 1.8)
def test_create(self):
r0 = Round.create(num=0)
r1 = Round.create(num=1)
team = Team.create(name=Team.OUR_NAME)
cs = ChallengeSet.create(name="foo")
cbn1 = ChallengeBinaryNode.create(name="foo_1", cs=cs, blob="aaa1")
cbn2 = ChallengeBinaryNode.create(name="foo_2", cs=cs, blob="aaa2")
csf = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team,
available_round=r0)
assert_equals(
csf.sha256,
"04de190c8dbd04bdb5768118c2cd745c7918f6858eddd765354819fc59c6d46e")
csf2 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1, cbn2],
team=team,
available_round=r1)
assert_equals(
csf2.sha256,
"277b0b746f1937a8f54797e2698e54f8646f0413ad353da19d93522c05817e73")
# insert duplicate team+cs+round fails
assert_raises(IntegrityError,
ChallengeSetFielding.create,
cs=cs,
cbns=[cbn1],
team=team,
available_round=r0)
def test_add_cbns_if_missing(self):
r0 = Round.create(num=0)
team = Team.create(name=Team.OUR_NAME)
cs = ChallengeSet.create(name="foo")
cbn1 = ChallengeBinaryNode.create(name="foo_1", cs=cs, blob="aaa1")
cbn2 = ChallengeBinaryNode.create(name="foo_2", cs=cs, blob="aaa2")
cbn3 = ChallengeBinaryNode.create(name="foo_3", cs=cs, blob="aaa3")
csf = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team,
available_round=r0)
assert_equals(len(csf.cbns), 1)
assert_equals(
csf.sha256,
"04de190c8dbd04bdb5768118c2cd745c7918f6858eddd765354819fc59c6d46e")
csf.add_cbns_if_missing(cbn2, cbn3)
assert_equals(len(csf.cbns), 3)
assert_equals(
csf.sha256,
"85b75a70a55b3e7606d1a1cc19ee1a16f1ad510dfb4dc84648a9df3ec6f83fe0")
csf.add_cbns_if_missing(cbn2, cbn3)
assert_equals(len(csf.cbns), 3)
assert_equals(
csf.sha256,
"85b75a70a55b3e7606d1a1cc19ee1a16f1ad510dfb4dc84648a9df3ec6f83fe0")
def test_latest(self):
team1 = Team.create(name=Team.OUR_NAME)
team2 = Team.create(name="other_team")
cs = ChallengeSet.create(name="foo")
cbn1 = ChallengeBinaryNode.create(name="foo_1",
cs=cs,
sha256="sum1",
blob="blob1")
cbn2 = ChallengeBinaryNode.create(name="foo_2",
cs=cs,
sha256="sum2",
blob="blob2")
cbn3 = ChallengeBinaryNode.create(name="foo_3",
cs=cs,
sha256="sum3",
blob="blob3")
r1 = Round.create(num=1)
csf11 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team1,
available_round=r1)
csf12 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team2,
available_round=r1)
assert_equals(csf11, ChallengeSetFielding.latest(cs, team1))
assert_equals(csf12, ChallengeSetFielding.latest(cs, team2))
r2 = Round.create(num=2)
csf21 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team1,
available_round=r2)
csf22 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn2],
team=team2,
available_round=r2)
assert_equals(csf21, ChallengeSetFielding.latest(cs, team1))
assert_equals(csf22, ChallengeSetFielding.latest(cs, team2))
r3 = Round.create(num=3)
csf31 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn3],
team=team1,
available_round=r3)
csf32 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team2,
available_round=r3)
assert_equals(csf31, ChallengeSetFielding.latest(cs, team1))
assert_equals(csf32, ChallengeSetFielding.latest(cs, team2))
def test_create_or_update_available(self):
r0 = Round.create(num=0)
team = Team.create(name=Team.OUR_NAME)
cs = ChallengeSet.create(name="foo")
cbn1 = ChallengeBinaryNode.create(name="foo_1", cs=cs, blob="aaa1")
cbn2 = ChallengeBinaryNode.create(name="foo_2", cs=cs, blob="aaa2")
csf1 = ChallengeSetFielding.create_or_update_available(team=team,
round=r0,
cbn=cbn1)
assert_in(cbn1, csf1.cbns)
assert_equals(len(csf1.cbns), 1)
csf2 = ChallengeSetFielding.create_or_update_available(team=team,
round=r0,
cbn=cbn2)
assert_equals(csf1, csf2)
assert_items_equal(csf1.cbns, [cbn1, cbn2])
def test_size_overhead(self):
ra = Round.create(num=6)
rb = Round.create(num=7)
cs = ChallengeSet.create(name="foo3")
team = Team.get_our()
cbn1 = ChallengeBinaryNode.create(name="foo_3",
cs=cs,
sha256="sum3",
blob="asdf")
cbn2 = ChallengeBinaryNode.create(name="foo_3a",
cs=cs,
sha256="sum4",
blob="asdf11")
pf = PollFeedback.create(cs=cs,
round=ra,
success=1.0,
timeout=0,
connect=0,
function=0,
time_overhead=0.0,
memory_overhead=0.0)
csf_orig = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team,
available_round=ra,
submission_round=ra,
poll_feedback=pf)
pf = PollFeedback.create(cs=cs,
round=ra,
success=1.0,
timeout=0,
connect=0,
function=0,
time_overhead=0.0,
memory_overhead=0.0)
csf_patched = ChallengeSetFielding.create(cs=cs,
cbns=[cbn2],
team=team,
available_round=rb,
poll_feedback=pf)
assert_less(cbn2.min_cb_score, 1.0)
def test_best(self):
team = Team.create(name=Team.OUR_NAME)
cs = ChallengeSet.create(name="foo")
r1 = Round.create(num=1)
ids1 = IDSRule.create(cs=cs, rules="ids1", sha256="sum1")
idsrf1 = IDSRuleFielding.create(cs=cs,
ids_rule=ids1,
team=team,
available_round=r1)
cbn1 = ChallengeBinaryNode.create(name="foo_3",
cs=cs,
sha256="sum3",
blob="blob3")
csf1 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team,
available_round=r1)
r2 = Round.create(num=2)
cbn2 = ChallengeBinaryNode.create(name="foo_3",
cs=cs,
sha256="sum4",
blob="blob4")
csf2 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn2],
team=team,
available_round=r2)
job = AFLJob.create(cbn=cbn1, cs=cs)
exploit = Exploit.create(job=job,
cs=cs,
blob="exploit",
pov_type='type1',
c_code="c_code")
assert_is_none(PovTestResult.best(csf1, idsrf1))
pov1 = PovTestResult.create(exploit=exploit,
cs_fielding=csf1,
num_success=2)
assert_equals(pov1, PovTestResult.best(csf1, None))
assert_is_none(PovTestResult.best(csf1, idsrf1))
assert_is_none(PovTestResult.best(csf2, idsrf1))
pov2 = PovTestResult.create(exploit=exploit,
cs_fielding=csf1,
ids_fielding=idsrf1,
num_success=2)
assert_equals(pov2, PovTestResult.best(csf1, idsrf1))
assert_is_none(PovTestResult.best(csf2, idsrf1))
pov3 = PovTestResult.create(exploit=exploit,
cs_fielding=csf1,
ids_fielding=idsrf1,
num_success=1)
assert_equals(pov2, PovTestResult.best(csf1, idsrf1))
assert_is_none(PovTestResult.best(csf2, idsrf1))
pov4 = PovTestResult.create(exploit=exploit,
cs_fielding=csf1,
ids_fielding=idsrf1,
num_success=4)
pov5 = PovTestResult.create(exploit=exploit,
cs_fielding=csf2,
ids_fielding=idsrf1,
num_success=8)
assert_equals(pov4, PovTestResult.best(csf1, idsrf1))
assert_equals(pov5, PovTestResult.best(csf2, idsrf1))
def test_cbn_to_polls(self):
r3 = Round.create(num=3)
r4 = Round.create(num=4)
r5 = Round.create(num=5)
cs = ChallengeSet.create(name="foo2")
team = Team.get_our()
cbn1 = ChallengeBinaryNode.create(name="foo_2",
cs=cs,
sha256="sum2",
blob="asdf")
assert_is_none(cbn1.avg_cb_score)
assert_is_none(cbn1.min_cb_score)
pf = PollFeedback.create(cs=cs,
round=r3,
success=0.0,
timeout=0,
connect=0,
function=0,
time_overhead=0.0,
memory_overhead=0.0)
csf3 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team,
available_round=r3,
submission_round=r3,
poll_feedback=pf)
pf = PollFeedback.create(cs=cs,
round=r4,
success=0.99,
timeout=0,
connect=0,
function=0,
time_overhead=0.0,
memory_overhead=0.0)
csf4 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team,
available_round=r4,
poll_feedback=pf)
pf = PollFeedback.create(cs=cs,
round=r5,
success=0.99,
timeout=0,
connect=0,
function=0,
time_overhead=0.0,
memory_overhead=0.2)
csf5 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team,
available_round=r5,
poll_feedback=pf)
assert_equal(len(cbn1.poll_feedbacks), 2)
assert_almost_equal(cbn1.avg_cb_score,
((0.9609803444828162 + 0.6830134553650711) / 2))
assert_almost_equal(cbn1.min_cb_score, 0.6830134553650711)
def test_poll_feedback(self):
r0 = Round.create(num=0)
cs = ChallengeSet.create(name="foo")
team = Team.get_our()
cbn1 = ChallengeBinaryNode.create(name="foo_1",
cs=cs,
sha256="sum1",
blob="asdf")
csf = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team,
available_round=r0)
# Check full score
p = PollFeedback.create(cs=cs,
round=r0,
success=1.0,
timeout=0,
connect=0,
function=0,
time_overhead=0.0,
memory_overhead=0.0)
csf.poll_feedback = p
csf.save()
assert_equal(p.cqe_performance_score, 1)
assert_equal(p.cqe_functionality_score, 1)
assert_equal(p.availability, 1)
assert_equal(p.cb_score, 1)
# Check minimum overhead
p = PollFeedback.create(cs=cs,
round=r0,
success=1.0,
timeout=0,
connect=0,
function=0,
time_overhead=0.0,
memory_overhead=0.09)
csf.poll_feedback = p
csf.save()
assert_equal(p.memory_overhead, 0.09)
assert_equal(p.availability, 1)
assert_equal(p.cb_score, 1)
p = PollFeedback.create(cs=cs,
round=r0,
success=1.0,
timeout=0,
connect=0,
function=0,
time_overhead=0.03,
memory_overhead=0.03)
csf.poll_feedback = p
csf.save()
assert_equal(p.availability, 1)
assert_equal(p.cb_score, 1)
# Check failed polls
p = PollFeedback.create(
cs=cs,
round=r0,
success=0.99,
timeout=0,
connect=0,
function=0,
time_overhead=0.03,
memory_overhead=0.03,
)
csf.poll_feedback = p
csf.save()
assert_almost_equal(p.availability, 0.9609803444828162)
assert_almost_equal(p.cb_score, 0.9609803444828162)
# Check more failures
p = PollFeedback.create(cs=cs,
round=r0,
success=0.97,
timeout=0,
connect=0,
function=0,
time_overhead=0.0,
memory_overhead=0.0)
csf.poll_feedback = p
csf.save()
assert_almost_equal(p.availability, 0.8884870479156888)
assert_almost_equal(p.cb_score, 0.8884870479156888)
# Check high failures
p = PollFeedback.create(cs=cs,
round=r0,
success=0.01,
timeout=0,
connect=0,
function=0,
time_overhead=0.0,
memory_overhead=0.0)
csf.poll_feedback = p
csf.save()
assert_equal(p.availability, 0.00381)
assert_equal(p.cb_score, 0.00381)
# Check high overhead
p = PollFeedback.create(cs=cs,
round=r0,
success=0.99,
timeout=0,
connect=0,
function=0,
time_overhead=0.10,
memory_overhead=0.20)
csf.poll_feedback = p
csf.save()
assert_almost_equal(p.availability, 0.6830134553650711)
assert_almost_equal(p.cb_score, 0.6830134553650711)
# Check full overhead
p = PollFeedback.create(cs=cs,
round=r0,
success=0.99,
timeout=0,
connect=0,
function=0,
time_overhead=1.00,
memory_overhead=0.20)
csf.poll_feedback = p
csf.save()
assert_equal(p.availability, 0)
assert_equal(p.cb_score, 0)