def LoadPositiveNegativeProtos(path: pathlib.Path) -> PositiveNegativeDataset:
"""Load positive and negative training protos from a directory."""
positive_protos = [
pbutil.FromFile(p, fish_pb2.CompilerCrashDiscriminatorTrainingExample())
for p in path.iterdir() if p.name.startswith('positive-')
]
logging.info(
'Loaded %s positive protos', humanize.intcomma(len(positive_protos)))
negative_protos = [
pbutil.FromFile(p, fish_pb2.CompilerCrashDiscriminatorTrainingExample())
for p in path.iterdir() if p.name.startswith('negative-')
]
logging.info(
'Loaded %s negative protos', humanize.intcomma(len(negative_protos)))
return PositiveNegativeDataset(positive_protos, negative_protos)
def ExportOpenCLResults(cursor, start_id, proto_dir):
batch_size = 1000
result_id = start_id
while True:
logging.info('Exporting batch of %s results',
humanize.intcomma(batch_size))
cursor.execute(
"""
SELECT
results.id,
assertions.assertion,
results.outcome,
programs.src
FROM results
LEFT JOIN testbeds ON results.testbed_id = testbeds.id
LEFT JOIN platforms ON testbeds.platform_id = platforms.id
LEFT JOIN testcases ON results.testcase_id = testcases.id
LEFT JOIN programs ON testcases.program_id = programs.id
LEFT JOIN stderrs ON results.stderr_id = stderrs.id
LEFT JOIN assertions ON stderrs.assertion_id = assertions.id
WHERE results.id >= %s
AND programs.generator = 1
AND testbeds.id = (
SELECT testbeds.id
FROM testbeds
LEFT JOIN platforms ON testbeds.platform_id=platforms.id
WHERE platform = 'clang'
AND driver = '3.6.2'
)
ORDER BY results.id
LIMIT %s
""", (result_id, batch_size))
i = 0
for row in cursor:
i += 1
(
result_id,
assertion_text,
outcome_num,
program_src,
) = row
outcome = fish_pb2.CompilerCrashDiscriminatorTrainingExample.Outcome.Name(
outcome_num).lower()
proto = fish_pb2.CompilerCrashDiscriminatorTrainingExample(
src=program_src,
outcome=outcome_num,
raised_assertion=True if assertion_text else False,
assertion_name=(GetClangAssertionStub(assertion_text)
if assertion_text else ''))
pbutil.ToFile(proto,
proto_dir / outcome / (str(result_id) + '.pbtxt'))
# If we received fewer results than the requested batch size, then we have
# ran out of data.
if i < batch_size:
return
def CreateTempFileFromProto(
tempdir: pathlib.Path, proto_path: pathlib.Path
) -> pathlib.Path:
"""Write testcase to a file in directory."""
proto = pbutil.FromFile(
proto_path, fish_pb2.CompilerCrashDiscriminatorTrainingExample()
)
path = tempdir / proto_path.name
with open(path, "w") as f:
f.write(proto.src)
return path