def test_create_or_update_submission(self):
r1 = Round.create(num=0)
team = Team.create(name=Team.OUR_NAME)
cs = ChallengeSet.create(name="foo")
cbn1 = ChallengeBinaryNode.create(name="foo", cs=cs, blob="BLOB")
cbn2 = ChallengeBinaryNode.create(name="foo", cs=cs, blob="BLOB2")
assert_equals(len(cs.fieldings), 0)
# Submit 2 patches at once
ChallengeSetFielding.create_or_update_submission(team=team,
cbns=[cbn1, cbn2],
round=r1)
assert_equals(len(cs.fieldings), 1)
assert_equals(len(cs.fieldings.get().cbns), 2)
assert_items_equal(cs.fieldings.get().cbns, [cbn1, cbn2])
assert_equals(cs.fieldings.get().submission_round,
Round.current_round())
assert_is_none(cs.fieldings.get().available_round)
# Submit again fails
cbn3 = ChallengeBinaryNode.create(name="foo3", cs=cs, blob="BLOB3")
ChallengeSetFielding.create_or_update_submission(team=team,
cbns=[cbn3],
round=r1)
assert_equals(len(cs.fieldings), 1)
assert_items_equal(cs.fieldings.get().cbns, [cbn3])
def test_create(self):
r0 = Round.create(num=0)
r1 = Round.create(num=1)
team = Team.create(name=Team.OUR_NAME)
cs = ChallengeSet.create(name="foo")
cbn1 = ChallengeBinaryNode.create(name="foo_1", cs=cs, blob="aaa1")
cbn2 = ChallengeBinaryNode.create(name="foo_2", cs=cs, blob="aaa2")
csf = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team,
available_round=r0)
assert_equals(
csf.sha256,
"04de190c8dbd04bdb5768118c2cd745c7918f6858eddd765354819fc59c6d46e")
csf2 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1, cbn2],
team=team,
available_round=r1)
assert_equals(
csf2.sha256,
"277b0b746f1937a8f54797e2698e54f8646f0413ad353da19d93522c05817e73")
# insert duplicate team+cs+round fails
assert_raises(IntegrityError,
ChallengeSetFielding.create,
cs=cs,
cbns=[cbn1],
team=team,
available_round=r0)
def test_latest(self):
team1 = Team.create(name=Team.OUR_NAME)
team2 = Team.create(name="other_team")
cs = ChallengeSet.create(name="foo")
cbn1 = ChallengeBinaryNode.create(name="foo_1",
cs=cs,
sha256="sum1",
blob="blob1")
cbn2 = ChallengeBinaryNode.create(name="foo_2",
cs=cs,
sha256="sum2",
blob="blob2")
cbn3 = ChallengeBinaryNode.create(name="foo_3",
cs=cs,
sha256="sum3",
blob="blob3")
r1 = Round.create(num=1)
csf11 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team1,
available_round=r1)
csf12 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team2,
available_round=r1)
assert_equals(csf11, ChallengeSetFielding.latest(cs, team1))
assert_equals(csf12, ChallengeSetFielding.latest(cs, team2))
r2 = Round.create(num=2)
csf21 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team1,
available_round=r2)
csf22 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn2],
team=team2,
available_round=r2)
assert_equals(csf21, ChallengeSetFielding.latest(cs, team1))
assert_equals(csf22, ChallengeSetFielding.latest(cs, team2))
r3 = Round.create(num=3)
csf31 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn3],
team=team1,
available_round=r3)
csf32 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team2,
available_round=r3)
assert_equals(csf31, ChallengeSetFielding.latest(cs, team1))
assert_equals(csf32, ChallengeSetFielding.latest(cs, team2))
def test_esimated_cb_score(self):
r6 = Round.create(num=6)
team = Team.get_our()
cs = ChallengeSet.create(name="foo3")
cbn1 = ChallengeBinaryNode.create(name="foo_3",
cs=cs,
sha256="sum3",
blob="asdf")
pt = PatchType.create(name="patch1",
functionality_risk=0.0,
exploitability=0.2)
# PatchType exists, but no patched binary
assert_is_none(cbn1.estimated_cb_score)
# Patched
cbn2 = ChallengeBinaryNode.create(name="foo_3",
cs=cs,
sha256="sum4",
blob="asdf",
patch_type=pt)
pf = PollFeedback.create(cs=cs,
round=r6,
success=1.0,
timeout=0,
connect=0,
function=0,
time_overhead=0.0,
memory_overhead=0.0)
perf_score = {
'score': {
'rep': {
'task_clock': 4,
'rss': 3,
'flt': 1,
'file_size': 5
},
'ref': {
'task_clock': 4,
'rss': 3,
'flt': 2,
'file_size': 6
}
}
}
ps = PatchScore.create(cs=cs,
round=r6,
num_polls=1,
perf_score=perf_score,
patch_type=pt)
csf_orig = ChallengeSetFielding.create(cs=cs,
cbns=[cbn2],
team=team,
available_round=r6,
submission_round=r6,
poll_feedback=pf)
assert_almost_equals(cbn2.estimated_cb_score, 1.8)
def test_add_cbns_if_missing(self):
r0 = Round.create(num=0)
team = Team.create(name=Team.OUR_NAME)
cs = ChallengeSet.create(name="foo")
cbn1 = ChallengeBinaryNode.create(name="foo_1", cs=cs, blob="aaa1")
cbn2 = ChallengeBinaryNode.create(name="foo_2", cs=cs, blob="aaa2")
cbn3 = ChallengeBinaryNode.create(name="foo_3", cs=cs, blob="aaa3")
csf = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team,
available_round=r0)
assert_equals(len(csf.cbns), 1)
assert_equals(
csf.sha256,
"04de190c8dbd04bdb5768118c2cd745c7918f6858eddd765354819fc59c6d46e")
csf.add_cbns_if_missing(cbn2, cbn3)
assert_equals(len(csf.cbns), 3)
assert_equals(
csf.sha256,
"85b75a70a55b3e7606d1a1cc19ee1a16f1ad510dfb4dc84648a9df3ec6f83fe0")
csf.add_cbns_if_missing(cbn2, cbn3)
assert_equals(len(csf.cbns), 3)
assert_equals(
csf.sha256,
"85b75a70a55b3e7606d1a1cc19ee1a16f1ad510dfb4dc84648a9df3ec6f83fe0")
def test_latest(self):
team1 = Team.create(name=Team.OUR_NAME)
team2 = Team.create(name="other_team")
cs = ChallengeSet.create(name="foo")
ids1 = IDSRule.create(cs=cs, rules="ids1", sha256="sum1")
ids2 = IDSRule.create(cs=cs, rules="ids2", sha256="sum2")
ids3 = IDSRule.create(cs=cs, rules="ids3", sha256="sum3")
r1 = Round.create(num=1)
idsrf11 = IDSRuleFielding.create(cs=cs,
ids_rule=ids1,
team=team1,
available_round=r1)
idsrf12 = IDSRuleFielding.create(cs=cs,
ids_rule=ids1,
team=team2,
available_round=r1)
assert_equals(idsrf11, IDSRuleFielding.latest(cs, team1))
assert_equals(idsrf12, IDSRuleFielding.latest(cs, team2))
r2 = Round.create(num=2)
idsrf21 = IDSRuleFielding.create(cs=cs,
ids_rule=ids1,
team=team1,
available_round=r2)
idsrf22 = IDSRuleFielding.create(cs=cs,
ids_rule=ids2,
team=team2,
available_round=r2)
assert_equals(idsrf21, IDSRuleFielding.latest(cs, team1))
assert_equals(idsrf22, IDSRuleFielding.latest(cs, team2))
r3 = Round.create(num=3)
idsrf31 = IDSRuleFielding.create(cs=cs,
ids_rule=ids3,
team=team1,
available_round=r3)
idsrf32 = IDSRuleFielding.create(cs=cs,
ids_rule=ids2,
team=team2,
available_round=r3)
assert_equals(idsrf31, IDSRuleFielding.latest(cs, team1))
assert_equals(idsrf32, IDSRuleFielding.latest(cs, team2))
def test_size_overhead(self):
ra = Round.create(num=6)
rb = Round.create(num=7)
cs = ChallengeSet.create(name="foo3")
team = Team.get_our()
cbn1 = ChallengeBinaryNode.create(name="foo_3",
cs=cs,
sha256="sum3",
blob="asdf")
cbn2 = ChallengeBinaryNode.create(name="foo_3a",
cs=cs,
sha256="sum4",
blob="asdf11")
pf = PollFeedback.create(cs=cs,
round=ra,
success=1.0,
timeout=0,
connect=0,
function=0,
time_overhead=0.0,
memory_overhead=0.0)
csf_orig = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team,
available_round=ra,
submission_round=ra,
poll_feedback=pf)
pf = PollFeedback.create(cs=cs,
round=ra,
success=1.0,
timeout=0,
connect=0,
function=0,
time_overhead=0.0,
memory_overhead=0.0)
csf_patched = ChallengeSetFielding.create(cs=cs,
cbns=[cbn2],
team=team,
available_round=rb,
poll_feedback=pf)
assert_less(cbn2.min_cb_score, 1.0)
def test_create_or_update_available(self):
r0 = Round.create(num=0)
team = Team.create(name=Team.OUR_NAME)
cs = ChallengeSet.create(name="foo")
cbn1 = ChallengeBinaryNode.create(name="foo_1", cs=cs, blob="aaa1")
cbn2 = ChallengeBinaryNode.create(name="foo_2", cs=cs, blob="aaa2")
csf1 = ChallengeSetFielding.create_or_update_available(team=team,
round=r0,
cbn=cbn1)
assert_in(cbn1, csf1.cbns)
assert_equals(len(csf1.cbns), 1)
csf2 = ChallengeSetFielding.create_or_update_available(team=team,
round=r0,
cbn=cbn2)
assert_equals(csf1, csf2)
assert_items_equal(csf1.cbns, [cbn1, cbn2])
def test_best(self):
team = Team.create(name=Team.OUR_NAME)
cs = ChallengeSet.create(name="foo")
r1 = Round.create(num=1)
ids1 = IDSRule.create(cs=cs, rules="ids1", sha256="sum1")
idsrf1 = IDSRuleFielding.create(cs=cs,
ids_rule=ids1,
team=team,
available_round=r1)
cbn1 = ChallengeBinaryNode.create(name="foo_3",
cs=cs,
sha256="sum3",
blob="blob3")
csf1 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team,
available_round=r1)
r2 = Round.create(num=2)
cbn2 = ChallengeBinaryNode.create(name="foo_3",
cs=cs,
sha256="sum4",
blob="blob4")
csf2 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn2],
team=team,
available_round=r2)
job = AFLJob.create(cbn=cbn1, cs=cs)
exploit = Exploit.create(job=job,
cs=cs,
blob="exploit",
pov_type='type1',
c_code="c_code")
assert_is_none(PovTestResult.best(csf1, idsrf1))
pov1 = PovTestResult.create(exploit=exploit,
cs_fielding=csf1,
num_success=2)
assert_equals(pov1, PovTestResult.best(csf1, None))
assert_is_none(PovTestResult.best(csf1, idsrf1))
assert_is_none(PovTestResult.best(csf2, idsrf1))
pov2 = PovTestResult.create(exploit=exploit,
cs_fielding=csf1,
ids_fielding=idsrf1,
num_success=2)
assert_equals(pov2, PovTestResult.best(csf1, idsrf1))
assert_is_none(PovTestResult.best(csf2, idsrf1))
pov3 = PovTestResult.create(exploit=exploit,
cs_fielding=csf1,
ids_fielding=idsrf1,
num_success=1)
assert_equals(pov2, PovTestResult.best(csf1, idsrf1))
assert_is_none(PovTestResult.best(csf2, idsrf1))
pov4 = PovTestResult.create(exploit=exploit,
cs_fielding=csf1,
ids_fielding=idsrf1,
num_success=4)
pov5 = PovTestResult.create(exploit=exploit,
cs_fielding=csf2,
ids_fielding=idsrf1,
num_success=8)
assert_equals(pov4, PovTestResult.best(csf1, idsrf1))
assert_equals(pov5, PovTestResult.best(csf2, idsrf1))
def run(self, current_round=None, random_submit=False):
for team in Team.opponents():
throws = 10
for cs in ChallengeSet.fielded_in_round():
target_cs_fielding = ChallengeSetFielding.latest(cs, team)
target_ids_fielding = IDSRuleFielding.latest(cs, team)
to_submit_pov = None
results = None
"""
if target_cs_fielding is not None:
results = PovTestResult.best(target_cs_fielding, target_ids_fielding)
if results is None:
results = PovTestResult.best_against_cs_fielding(target_cs_fielding)
if results is None:
results = PovTestResult.best_against_cs(target_cs_fielding.cs)
if results is not None:
# Good, we have a PovTestResult to submit.
# FIXME: Should we take the most reliable against this CS and IDS?
to_submit_pov = results.exploit
LOG.info("Submitting a tested PoV %s against team=%s cs=%s",
to_submit_pov.id, team.name, cs.name)
else:
# No, latest CS fielding, something wrong!!
LOG.warn("No CS fielding available for team=%s cs=%s", team.name, cs.name)
# if the best PoV we have has absolutely no successes, it may as well be nothing
if to_submit_pov is not None and results.num_success == 0:
to_submit_pov = None
"""
# We do not have a specific PoV, hence submit the most reliable PoV we have
if to_submit_pov is None and cs.exploits:
most_reliable = cs.most_reliable_exploit
if most_reliable is not None:
to_submit_pov = most_reliable
LOG.info(
"Submitting most reliable POV %s against team=%s cs=%s",
to_submit_pov.id, team.name, cs.name)
# Submit our PoV
if to_submit_pov is not None:
LOG.info("Submitting PoV %s against team=%s cs=%s",
to_submit_pov.id, team.name, cs.name)
round_ = Round.current_round()
if ExploitSubmissionCable.cable_exists(team,
cs,
round_=round_):
existing_cable = ExploitSubmissionCable.get(
team=team, cs=cs, round=round_)
existing_cable.exploit = to_submit_pov
existing_cable.save()
else:
ExploitSubmissionCable.create(
team=team,
cs=cs,
exploit=to_submit_pov,
throws=throws,
round=Round.current_round())
LOG.debug("POV %s marked for submission", to_submit_pov.id)
else:
LOG.warn("No POV to submit for team=%s cs=%s", team.name,
cs.name)
def test_cbn_to_polls(self):
r3 = Round.create(num=3)
r4 = Round.create(num=4)
r5 = Round.create(num=5)
cs = ChallengeSet.create(name="foo2")
team = Team.get_our()
cbn1 = ChallengeBinaryNode.create(name="foo_2",
cs=cs,
sha256="sum2",
blob="asdf")
assert_is_none(cbn1.avg_cb_score)
assert_is_none(cbn1.min_cb_score)
pf = PollFeedback.create(cs=cs,
round=r3,
success=0.0,
timeout=0,
connect=0,
function=0,
time_overhead=0.0,
memory_overhead=0.0)
csf3 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team,
available_round=r3,
submission_round=r3,
poll_feedback=pf)
pf = PollFeedback.create(cs=cs,
round=r4,
success=0.99,
timeout=0,
connect=0,
function=0,
time_overhead=0.0,
memory_overhead=0.0)
csf4 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team,
available_round=r4,
poll_feedback=pf)
pf = PollFeedback.create(cs=cs,
round=r5,
success=0.99,
timeout=0,
connect=0,
function=0,
time_overhead=0.0,
memory_overhead=0.2)
csf5 = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team,
available_round=r5,
poll_feedback=pf)
assert_equal(len(cbn1.poll_feedbacks), 2)
assert_almost_equal(cbn1.avg_cb_score,
((0.9609803444828162 + 0.6830134553650711) / 2))
assert_almost_equal(cbn1.min_cb_score, 0.6830134553650711)
def test_poll_feedback(self):
r0 = Round.create(num=0)
cs = ChallengeSet.create(name="foo")
team = Team.get_our()
cbn1 = ChallengeBinaryNode.create(name="foo_1",
cs=cs,
sha256="sum1",
blob="asdf")
csf = ChallengeSetFielding.create(cs=cs,
cbns=[cbn1],
team=team,
available_round=r0)
# Check full score
p = PollFeedback.create(cs=cs,
round=r0,
success=1.0,
timeout=0,
connect=0,
function=0,
time_overhead=0.0,
memory_overhead=0.0)
csf.poll_feedback = p
csf.save()
assert_equal(p.cqe_performance_score, 1)
assert_equal(p.cqe_functionality_score, 1)
assert_equal(p.availability, 1)
assert_equal(p.cb_score, 1)
# Check minimum overhead
p = PollFeedback.create(cs=cs,
round=r0,
success=1.0,
timeout=0,
connect=0,
function=0,
time_overhead=0.0,
memory_overhead=0.09)
csf.poll_feedback = p
csf.save()
assert_equal(p.memory_overhead, 0.09)
assert_equal(p.availability, 1)
assert_equal(p.cb_score, 1)
p = PollFeedback.create(cs=cs,
round=r0,
success=1.0,
timeout=0,
connect=0,
function=0,
time_overhead=0.03,
memory_overhead=0.03)
csf.poll_feedback = p
csf.save()
assert_equal(p.availability, 1)
assert_equal(p.cb_score, 1)
# Check failed polls
p = PollFeedback.create(
cs=cs,
round=r0,
success=0.99,
timeout=0,
connect=0,
function=0,
time_overhead=0.03,
memory_overhead=0.03,
)
csf.poll_feedback = p
csf.save()
assert_almost_equal(p.availability, 0.9609803444828162)
assert_almost_equal(p.cb_score, 0.9609803444828162)
# Check more failures
p = PollFeedback.create(cs=cs,
round=r0,
success=0.97,
timeout=0,
connect=0,
function=0,
time_overhead=0.0,
memory_overhead=0.0)
csf.poll_feedback = p
csf.save()
assert_almost_equal(p.availability, 0.8884870479156888)
assert_almost_equal(p.cb_score, 0.8884870479156888)
# Check high failures
p = PollFeedback.create(cs=cs,
round=r0,
success=0.01,
timeout=0,
connect=0,
function=0,
time_overhead=0.0,
memory_overhead=0.0)
csf.poll_feedback = p
csf.save()
assert_equal(p.availability, 0.00381)
assert_equal(p.cb_score, 0.00381)
# Check high overhead
p = PollFeedback.create(cs=cs,
round=r0,
success=0.99,
timeout=0,
connect=0,
function=0,
time_overhead=0.10,
memory_overhead=0.20)
csf.poll_feedback = p
csf.save()
assert_almost_equal(p.availability, 0.6830134553650711)
assert_almost_equal(p.cb_score, 0.6830134553650711)
# Check full overhead
p = PollFeedback.create(cs=cs,
round=r0,
success=0.99,
timeout=0,
connect=0,
function=0,
time_overhead=1.00,
memory_overhead=0.20)
csf.poll_feedback = p
csf.save()
assert_equal(p.availability, 0)
assert_equal(p.cb_score, 0)