def test_is_multi_cbn(self):
r0 = Round.create(num=0)
our_team = Team.create(name=Team.OUR_NAME)
# CS single binary
cs = ChallengeSet.create(name="single")
cs.rounds = [r0]
cbn = ChallengeBinaryNode.create(name="foo", cs=cs, blob="aaa1")
# CS multi binary
cs_multi = ChallengeSet.create(name="multi")
cs_multi.rounds = [r0]
cbn1 = ChallengeBinaryNode.create(name="foo1",
cs=cs_multi,
blob="aaa2")
cbn2 = ChallengeBinaryNode.create(name="foo2",
cs=cs_multi,
blob="aaa3")
# create fielding entries
ChallengeSetFielding.create(cs=cs,
cbns=[cbn],
team=our_team,
available_round=r0)
ChallengeSetFielding.create(cs=cs_multi,
cbns=[cbn1, cbn2],
team=our_team,
available_round=r0)
assert_false(cs.is_multi_cbn)
assert_true(cs_multi.is_multi_cbn)
def create_round(args):
if not len(args.challenge_sets):
fielded = [cs.name for cs in CS.fielded_in_round()]
else:
fielded = args.challenge_sets
new_round = Round.create(
num=args.number if args.number is not None else Round.current_round() +
1)
for f in fielded:
cs = CS.select().where(CS.name == f).get()
CSF.create(cs=cs, team=Team.get_our(), available_round=new_round)
def test_fielded_in_round(self):
now = datetime.now()
r1 = Round.create(num=0)
r2 = Round.create(num=1)
cs1 = ChallengeSet.create(name="foo")
cs1.rounds = [r1, r2]
cs2 = ChallengeSet.create(name="bar")
cs2.rounds = [r1]
assert_equals(len(ChallengeSet.fielded_in_round(r1)), 2)
assert_in(cs1, ChallengeSet.fielded_in_round(r1))
assert_in(cs2, ChallengeSet.fielded_in_round(r1))
def add_test_or_crash(args):
p = subprocess.Popen([shellphish_qemu.qemu_path('cgc-base'), args.cb],
stdin=subprocess.PIPE)
p.poll()
test = ""
if args.batch:
test = sys.stdin.read()
p.communicate(test)
else:
try:
while p.returncode is None:
r, _, _ = select([sys.stdin], [], [], 0.05)
if r is not None:
b = sys.stdin.read(1)
test += b
p.stdin.write(b)
p.poll()
except KeyboardInterrupt:
p.returncode = 0
except IOError:
p.returncode = 1
if p.returncode == 0:
print "Finished test, inserting now..."
cs = CS.select().where(CS.name == args.cs)
# first we have to make a fake job
job = Job.create(cs=cs,
completed_at=datetime.datetime.now(),
worker="garbage")
Test.create(cs=cs, job=job, blob=test)
print "Test inserted!"
else:
print "Found a crash, inserting now..."
qc = QuickCrash(args.cb, test)
print "appears to be of type " + qc.kind
print "cs = " + args.cs
cs = CS.select().where(CS.name == args.cs)
# first we have to make a fake job
job = Job.create(cs=cs,
completed_at=datetime.datetime.now(),
worker="garbage")
Crash.create(cs=cs, job=job, blob=test, kind=qc.kind)
print "Crash inserted!"
def test_get_or_create(self):
cs = ChallengeSet.create(name="foo")
cbn = ChallengeBinaryNode.create(name="foo", cs=cs, blob="aaa")
job1, job1_created = RexJob.get_or_create(cbn=cbn, payload={'something': 'xxx'})
job2, job2_created = AFLJob.get_or_create(cbn=cbn, payload={'something': 'xxx'})
assert_not_equal(job1.id, job2.id)
assert_true(job1_created)
assert_true(job2_created)
Job.delete().execute()
ChallengeBinaryNode.delete().execute()
ChallengeSet.delete().execute()
Job._meta.database.commit()
def list_patches(args):
wanted_fields = (CBN.name, CBN.size, CBN.sha256, CBN.patch_type,
CBN.is_blacklisted)
q = CBN.select(*wanted_fields)
if args.cs is not None:
cs = CS.select().where(CS.name == args.cs)
for patch in q.where(CBN.cs == cs):
print format_patch(patch)
else:
for cs in CS.fielded_in_round():
print "CS {}:".format(cs.name)
for patch in q.where(CBN.cs == cs):
print format_patch(patch)
print ""
def test_submitted_and_unsubmitted_patches(self):
r0 = Round.create(num=0)
team = Team.create(name=Team.OUR_NAME)
cs = ChallengeSet.create(name="foo")
cs.rounds = [r0]
cbn = ChallengeBinaryNode.create(name="cbn", cs=cs, blob="aaa1")
patchtype1 = PatchType.create(name="PatchType1",
functionality_risk=0,
exploitability=0)
patchtype2 = PatchType.create(name="PatchType2",
functionality_risk=0,
exploitability=0)
patch1 = ChallengeBinaryNode.create(name="patch1",
patch_type=patchtype1,
cs=cs,
root=cbn,
blob="aaa2")
patch2 = ChallengeBinaryNode.create(name="patch2",
patch_type=patchtype2,
cs=cs,
root=cbn,
blob="aaa3")
assert_equals(len(cbn.unsubmitted_patches), 2)
assert_in(patch1, cbn.unsubmitted_patches)
assert_in(patch2, cbn.unsubmitted_patches)
assert_equals(len(cbn.submitted_patches), 0)
ChallengeSetFielding.create_or_update_submission(team=team,
cbns=[patch1, patch2],
round=r0)
assert_equals(len(cbn.submitted_patches), 2)
assert_equals(len(cbn.unsubmitted_patches), 0)
def test_get_or_create(self):
r = Round.create(num=0)
cs = ChallengeSet.create(name="foo")
cbn1 = ChallengeBinaryNode.create(name="foo1", cs=cs, blob="aaa1")
cbn2 = ChallengeBinaryNode.create(name="foo1", cs=cs, blob="aaa2")
ids = IDSRule.create(cs=cs, rules="aaa", sha256="sum")
cbl1, crtd1 = CSSubmissionCable.get_or_create(cs=cs,
ids=ids,
cbns=[cbn1],
round=r)
assert_true(crtd1)
cbl2, crtd2 = CSSubmissionCable.get_or_create(cs=cs,
ids=ids,
cbns=[cbn1],
round=r)
assert_false(crtd2)
assert_equals(cbl1.id, cbl2.id)
cbl3, crtd3 = CSSubmissionCable.get_or_create(cs=cs,
ids=ids,
cbns=[cbn1, cbn2],
round=r)
assert_true(crtd3)
cbl4, crtd4 = CSSubmissionCable.get_or_create(cs=cs,
ids=ids,
cbns=[cbn1, cbn2],
round=r)
assert_false(crtd4)
assert_equals(cbl3.id, cbl4.id)
def test_unsubmitted_exploits(self):
r1 = Round.create(num=0)
team = Team.create(name=Team.OUR_NAME)
cs = ChallengeSet.create(name="foo")
cs.rounds = [r1]
job = RexJob.create(cs=cs)
pov1 = Exploit.create(cs=cs,
job=job,
pov_type='type1',
exploitation_method='rop',
blob="exploit",
c_code="exploit it")
pov2 = Exploit.create(cs=cs,
job=job,
pov_type='type2',
exploitation_method='rop',
blob="exploit",
c_code="exploit it")
assert_equals(len(cs.unsubmitted_exploits), 2)
assert_in(pov1, cs.unsubmitted_exploits)
assert_in(pov2, cs.unsubmitted_exploits)
pov1.submit_to(team, 10)
assert_equals(len(cs.unsubmitted_exploits), 1)
assert_not_in(pov1, cs.unsubmitted_exploits)
assert_in(pov2, cs.unsubmitted_exploits)
pov2.submit_to(team, 10)
assert_equals(len(cs.unsubmitted_exploits), 0)
assert_not_in(pov1, cs.unsubmitted_exploits)
assert_not_in(pov2, cs.unsubmitted_exploits)
def test_cbns_by_patch_type(self):
cs = ChallengeSet.create(name="foo")
cbn = ChallengeBinaryNode.create(name="foo", cs=cs, blob="aaa")
patch0 = PatchType.create(
name="patch0",
functionality_risk=0,
exploitability=1,
)
patch1 = PatchType.create(
name="patch1",
functionality_risk=0,
exploitability=1,
)
cbn1 = ChallengeBinaryNode.create(name="foo1",
cs=cs,
patch_type=patch0,
blob="aaa1")
cbn2 = ChallengeBinaryNode.create(name="foo2",
cs=cs,
patch_type=patch0,
blob="aaa2")
cbn3 = ChallengeBinaryNode.create(name="foo3",
cs=cs,
patch_type=patch1,
blob="aaa3")
assert_in(patch0, cs.cbns_by_patch_type().keys())
assert_in(patch1, cs.cbns_by_patch_type().keys())
assert_in(cbn1, cs.cbns_by_patch_type()[patch0])
assert_in(cbn2, cs.cbns_by_patch_type()[patch0])
assert_in(cbn3, cs.cbns_by_patch_type()[patch1])
def run(self):
"""A run() method. Happy now pylint?"""
cgc_polls = self._get_feedback('poll')
Feedback.update_or_create(self._round,
polls=cgc_polls,
povs=self._get_feedback('pov'),
cbs=self._get_feedback('cb'))
fielded = {
c.name: c
for c in ChallengeSet.fielded_in_round(self._round)
}
for c in cgc_polls:
cs = fielded[c['csid']]
fieldings = cs.fieldings.where(
(ChallengeSetFielding.available_round == self._round)
& (ChallengeSetFielding.team == Team.get_our()))
if fieldings:
fielding = fieldings.get()
if fielding.poll_feedback is None:
fnct, perf = c['functionality'], c['performance']
pf = PollFeedback.create(
cs=cs,
round=self._round,
success=float(fnct['success']) / 100.0,
timeout=float(fnct['timeout']) / 100.0,
connect=float(fnct['connect']) / 100.0,
function=float(fnct['function']) / 100.0,
time_overhead=float(perf['time']) / 100.0 - 1,
memory_overhead=float(perf['memory']) / 100.0 - 1)
fielding.poll_feedback = pf
fielding.save()
def challenge_sets(self, round_=None):
"""Return the list of challenge sets that are active in a round.
:keyword round_: The round number for which the binaries should be
returned (default: current round).
"""
return ChallengeSet.fielded_in_round(round_)
def get_cs_from_id(target_cs_id):
"""
Get ChallengeSet for given id
:param target_cs_id: id for which ChallengeSet need to be fetched.
:return: ChallengeSet
"""
return ChallengeSet.get(ChallengeSet.id == target_cs_id)
def test_has_submissions_in_round(self):
r0 = Round.create(num=0)
r1 = Round.create(num=1)
cs = ChallengeSet.create(name="foo")
cbn = ChallengeBinaryNode.create(name="foo1", cs=cs, blob="aaa")
our_team = Team.create(name=Team.OUR_NAME)
other_team = Team.create(name="enemy")
ChallengeSetFielding.create(cs=cs,
cbns=[cbn],
team=our_team,
submission_round=r1)
assert_false(cs.has_submissions_in_round(r0))
assert_true(cs.has_submissions_in_round(r1))
ChallengeSetFielding.create(cs=cs,
cbns=[cbn],
team=other_team,
submission_round=r0)
assert_false(cs.has_submissions_in_round(r0))
ChallengeSetFielding.create(cs=cs,
cbns=[cbn],
team=our_team,
submission_round=r0)
assert_true(cs.has_submissions_in_round(r0))
def test_all_tests_for_this_cs(self):
cs = ChallengeSet.create(name="foo")
job = AFLJob.create(cs=cs)
test1 = farnsworth.models.Test.create(cs=cs, job=job, blob="test1")
test2 = farnsworth.models.Test.create(cs=cs, job=job, blob="test2")
assert_equals(len(cs.tests), 2)
def upload_cbns(args):
cs = CS.select().where(CS.name == args.cs)
patch_type, _ = PT.get_or_create(name="manual",
functionality_risk=1.0,
exploitability=0.0)
cbns = []
for patched_file in args.patched_files:
with open(patched_file) as f:
content = f.read()
try:
cbn = CBN.create(cs=cs,
blob=content,
name=patched_file,
patch_type=patch_type)
except peewee.IntegrityError:
print "CBN already exists. Fetching."
cbn = CBN.select().where(CBN.name == args.patched_file,
CBN.cs == cs, CBN.blob == content)
cbns.append(cbn)
if args.field:
ids = IDSRule.get_or_create(cs=target_cs, rules='')
CSSubmissionCable.get_or_create(cs=target_cs,
cbns=cbns,
ids=ids,
round=Round.current_round())
if args.eF is not None and args.eT is not None and args.eM is not None and cbn.patch_type is not None:
perf_score = {
'score': {
'ref': {
'task_clock': 1.0,
'rss': 1.0,
'flt': 1.0,
'file_size': 1.0
},
'rep': {
'task_clock': args.eT,
'file_size': args.size_overhead,
'rss': args.eM,
'flt': args.eM,
}
}
}
PS.create(cs=cs,
perf_score=perf_score,
patch_type=cbn.patch_type,
has_failed_polls=args.eF != 0,
failed_polls=args.eF)
if args.pT is not None and args.pM is not None and args.pS is not None:
csf.poll_feedback = PF.create(cs=cs,
round_id=Round.current_round().id,
success=args.pS,
time_overhead=args.pT,
memory_overhead=args.pM,
size_overhead=args.size_overhead)
csf.save()
def test_cs_name_and_sha256_uniqueness(self):
cs1 = ChallengeSet.create(name="foo")
cs2 = ChallengeSet.create(name="bar")
# first binary is ok
ChallengeBinaryNode.create(name="test1", cs=cs1, blob=BLOB)
# same binary with different name is ok
ChallengeBinaryNode.create(name="test2", cs=cs1, blob=BLOB)
# same binary with different cs is ok
ChallengeBinaryNode.create(name="test1", cs=cs2, blob=BLOB)
# same cs and name but different binary is ok
ChallengeBinaryNode.create(name="test1", cs=cs2, blob=BLOB2)
# same cs, name and binary raises error
assert_raises(IntegrityError,
ChallengeBinaryNode.create,
name="test1",
cs=cs1,
blob=BLOB)
def test_get_by_sha256_or_create(self):
cs = ChallengeSet.create(name="foo")
assert_equals(len(IDSRule.all()), 0)
ids1 = IDSRule.get_by_sha256_or_create(rules="aaa", cs=cs)
ids2 = IDSRule.get_by_sha256_or_create(rules="aaa", cs=cs)
assert_equals(ids1, ids2)
assert_equals(len(IDSRule.all()), 1)
def test_most_recent(self):
cs = ChallengeSet.create(name="foo")
cs2 = ChallengeSet.create(name="bar")
team = Team.create(name="opponent")
exploit = Exploit.create(cs=cs,
job=RexJob.create(),
pov_type="type1",
blob="abc",
c_code="exploit it")
exploit2 = Exploit.create(cs=cs2,
job=RexJob.create(),
pov_type="type1",
blob="def",
c_code="delfino")
Round.create(num=0)
cable = ExploitSubmissionCable.create(team=team,
cs=cs,
exploit=exploit,
throws=10,
round=Round.current_round())
cable2 = ExploitSubmissionCable.create(team=team,
cs=cs2,
exploit=exploit2,
throws=10,
round=Round.current_round())
assert_equals(len(ExploitSubmissionCable.most_recent()), 2)
assert_items_equal(ExploitSubmissionCable.most_recent(),
[cable, cable2])
# assert we get back only the most recent exploit
r1 = Round.create(num=1)
new_exploit = Exploit.create(cs=cs,
job=RexJob.create(),
pov_type="type2",
blob="def",
c_code="don't exploit it")
new_cable = ExploitSubmissionCable.create(team=team,
cs=cs,
exploit=new_exploit,
throws=10,
round=r1)
assert_equals(len(ExploitSubmissionCable.most_recent()), 2)
assert_items_equal(ExploitSubmissionCable.most_recent(),
[new_cable, cable2])
def test_undrilled_tests_for_cs(self):
cs = ChallengeSet.create(name="foo")
job = AFLJob.create(cs=cs)
new_test = farnsworth.models.Test.create(cs=cs,
job=job,
blob="crash",
drilled=False)
assert_true(len(cs.undrilled_tests), 1)
def test_found_crash_for_cs(self):
cs = ChallengeSet.create(name="foo")
job = AFLJob.create(cs=cs)
crash = farnsworth.models.Crash.create(cs=cs,
job=job,
blob="crash",
crash_pc=0x41414141)
assert_true(cs.found_crash)
def test_has_type1(self):
from farnsworth.models.pov_test_result import PovTestResult
pov_type = 'type1'
cs = ChallengeSet.create(name="foo")
job = AFLJob.create(cs=cs)
Exploit.create(cs=cs,
job=job,
pov_type=pov_type,
blob=BLOB,
c_code="code",
reliability=0)
assert_false(cs.has_type1)
Exploit.create(cs=cs,
job=job,
pov_type=pov_type,
blob=BLOB,
c_code="code",
reliability=1)
assert_true(cs.has_type1)
# if not first condition is not met
r2 = Round.create(num=2)
cs2 = ChallengeSet.create(name="bar")
job2 = AFLJob.create(cs=cs2)
cbn2 = ChallengeBinaryNode.create(name="bar", cs=cs2, blob="aaa")
team2 = Team.create(name=Team.OUR_NAME)
exploit2 = Exploit.create(cs=cs2,
job=job2,
pov_type=pov_type,
blob=BLOB,
c_code="code",
reliability=0)
csf2 = ChallengeSetFielding.create_or_update_available(team=team2,
cbn=cbn2,
round=r2)
pov_result2 = PovTestResult.create(exploit=exploit2,
cs_fielding=csf2,
num_success=0)
assert_false(cs2.has_type1)
pov_result2.num_success = 10
pov_result2.save()
assert_true(cs2.has_type1)
def cbns(self, round_=None):
"""Return the list of binaries that are active in a round.
:keyword round_: The round instance for which the binaries should be
returned (default: current round).
"""
for cs in ChallengeSet.fielded_in_round(round_):
for cbn in cs.cbns_original:
LOG.debug("Found cbid: %s", cbn.name)
yield cbn
def test_all(self):
r1 = Round.create(num=0)
cs = ChallengeSet.create(name="foo")
cs.rounds = [r1]
cbn1 = ChallengeBinaryNode.create(name="foo", cs=cs, blob="blob1")
cbn2 = ChallengeBinaryNode.create(name="bar", cs=cs, blob="blob2")
assert_equals(len(ChallengeBinaryNode.all()), 2)
assert_equals(ChallengeBinaryNode.all()[0], cbn1)
assert_equals(ChallengeBinaryNode.all()[1], cbn2)
def test_get_or_create(self):
assert_equals(len(ChallengeSet.select()), 0)
r1 = Round.create(num=0)
cs1, _ = ChallengeSet.get_or_create(name="foo")
cs2, _ = ChallengeSet.get_or_create(name="foo")
cs3, _ = ChallengeSet.get_or_create(name="bar")
for cs in [cs1, cs2, cs3]:
cs.rounds = [r1.id]
assert_equals(len(ChallengeSet.select()), 2)
assert_equals(cs2.id, cs1.id)
assert_not_equals(cs3.id, cs1.id)
cs3.delete_instance(recursive=True)
cs2.delete_instance(recursive=True)
cs1.delete_instance(recursive=True)
r1.delete_instance(recursive=True)
ChallengeSet._meta.database.commit()
def insert_test(args):
cs = CS.select().where(CS.name == args.cs)
test = args.test.read()
# first we have to make a fake job
job = Job.create(cs=cs,
completed_at=datetime.datetime.now(),
worker="garbage")
Test.create(cs=cs, job=job, blob=test)
def run(self, current_round=None, random_submit=False): # pylint:disable=no-self-use,unused-argument
return
if current_round == 0:
return
# As ambassador will take care of actually submitting the binary.
for cs in ChallengeSet.fielded_in_round():
#if not self.should_submit(cs):
# continue
CBSubmitter.process_patch_submission(cs)
def test_pov_type(self):
cs = ChallengeSet.create(name="foo")
job = AFLJob.create(cs=cs)
exploit = Exploit.create(cs=cs, job=job, pov_type="type1", blob="exploit",
c_code="exploit it")
assert_equals(exploit.pov_type, "type1")
def invalid_pov_type():
Exploit.create(cs=cs, job=job, pov_type="bar")
assert_raises(Exception, invalid_pov_type)
def field_cbns(args):
cs = CS.select().where(CS.name == args.cs)
# i know i could use one query for this, but then we might get duplicate CBNs and more than we want
cbns = [
CBN.get(CBN.cs == cs, CBN.sha256 == sha) for sha in args.patched_shas
]
ids, _ = IDSRule.get_or_create(cs=cs, rules='')
CSSubmissionCable.get_or_create(cs=cs,
cbns=cbns,
ids=ids,
round=Round.current_round())
def insert_crash(args):
cs = CS.select().where(CS.name == args.cs)
test = args.crash.read()
kind = args.crash_kind
# first we have to make a fake job
job = Job.create(cs=cs,
completed_at=datetime.datetime.now(),
worker="garbage")
Crash.create(cs=cs, kind=kind, job=job, blob=test)