def generate_fake_bases(output_dir):
analyses = []
for tool, tool_version in analysis_tools:
for sut, sut_version in software:
issues = []
metadata = Metadata(Generator(tool, tool_version),
DebianSource(sut, sut_version, None),
None, None)
for i in range(random.randint(0, 4)):
file_, function = functions[random.randint(0, len(functions)-1)]
file_ = File(file_, None, None)
function = Function(function)
location = Location(file_, function)
try:
fortune = subprocess.check_output(["fortune"])
except OSError:
print("'fortune' is not installed and required to"
"generate fake messages")
import sys; sys.exit()
message = Message(fortune)
issue = Issue(None, None, location, message, None, None)
issues.append(issue)
analysis = Analysis(metadata, issues)
analyses.append(analysis)
i = 0
for analysis in analyses:
f = open(os.path.join(output_dir, "analysis" + str(i) + ".xml"), "w")
f.write(analysis.to_xml_bytes())
f.write("\n")
f.close()
i += 1
def parse_file(data_file_name,
findbugs_version=None,
sut=None,
file_=None,
stats=None):
"""
:param data_file: str object containing findbugs scan result
:type data_file: str
:param findbugs_version: version of findbugs
:type findbugs_version: str
:return: Analysis instance
"""
data_file = open(data_file_name)
generator = Generator(name="findbugs", version=findbugs_version)
metadata = Metadata(generator, sut, file_, stats)
analysis = Analysis(metadata, [])
for line in data_file.readlines():
issue = parse_line(line)
if issue:
analysis.results.append(issue)
else:
sys.stderr.write("fail to pass line=[%s]" % line)
data_file.close()
return analysis
def test_multiple_source_files(self):
"""
Verify invocations that cover multiple source files, linking
to an executable.
"""
driver = self.make_driver()
args = ['gcc',
'test-sources/multiple-1.c',
'test-sources/multiple-2.c',
'-o', 'multiple.exe']
r = driver.invoke(args)
self.assertEqual(driver.ctxt.stderr.getvalue(), '')
self.assertEqual(driver.returncode, 0)
# Verify that it wrote out "firehose" XML files to 'outputdir':
analyses = []
for xmlpath in glob.glob(os.path.join(driver.outputdir, '*.xml')):
with open(xmlpath) as f:
analysis = Analysis.from_xml(f)
analyses.append(analysis)
# Verify that none of the side-effects failed (e.g. with
# linker errors):
self.assertEqual(analysis.customfields['returncode'], 0)
os.unlink(xmlpath)
self.assertEqual(len(analyses), 8)
# We should have a ./multiple.exe that we can run
self.assertTrue(os.path.exists('./multiple.exe'))
os.unlink('./multiple.exe')
os.rmdir(driver.outputdir)
def invoke(self, sourcefile, extraargs=None):
driver = self.make_driver()
args = ['gcc', '-c', sourcefile]
if extraargs:
args += extraargs
r = driver.invoke(args)
if sourcefile and os.path.exists(sourcefile):
with open(sourcefile) as f:
content = f.read()
expected_hexdigest = hashlib.sha1(content).hexdigest()
else:
expected_hexdigest = None
# Verify that it wrote out "firehose" XML files to 'outputdir':
analyses = []
for xmlpath in glob.glob(os.path.join(driver.outputdir, '*.xml')):
with open(xmlpath) as f:
analysis = Analysis.from_xml(f)
if 0:
print(analysis)
if expected_hexdigest:
self.assertEqual(analysis.metadata.file_.hash_.alg, 'sha1')
self.assertEqual(analysis.metadata.file_.hash_.hexdigest,
expected_hexdigest)
analyses.append(analysis)
os.unlink(xmlpath)
self.assertEqual(len(analyses), 4)
os.rmdir(driver.outputdir)
return driver
def parse_plist(pathOrFile,
analyzerversion=None,
sut=None,
file_=None,
stats=None):
"""
Given a .plist file emitted by clang-static-analyzer (e.g. via
scan-build), parse it and return an Analysis instance
"""
plist = plistlib.readPlist(pathOrFile)
# We now have the .plist file as a hierarchy of dicts, lists, etc
# Handy debug dump:
if 0:
pprint(plist)
# A list of filenames, apparently referenced by index within
# diagnostics:
files = plist['files']
generator = Generator(name='clang-analyzer', version=analyzerversion)
metadata = Metadata(generator, sut, file_, stats)
analysis = Analysis(metadata, [])
for diagnostic in plist['diagnostics']:
if 0:
pprint(diagnostic)
cwe = None
# TODO: we're not yet handling the following:
# diagnostic['category']
# diagnostic['type']
message = Message(text=diagnostic['description'])
loc = diagnostic['location']
location = Location(
file=File(givenpath=files[loc.file], abspath=None),
# FIXME: doesn't tell us function name
# TODO: can we patch this upstream?
function=None,
point=Point(int(loc.line), int(loc.col)))
notes = None
trace = make_trace(files, diagnostic['path'])
issue = Issue(
cwe,
None, # FIXME: can we get at the test id?
location,
message,
notes,
trace)
analysis.results.append(issue)
return analysis
def convert_reports_to_firehose():
reports = []
# scan-build (clang analyzer)
scanbuild_report_results = []
metadata = None
for resultdir in glob.glob(os.path.join('reports/scan-build', '*')):
scanbuild_tmp_firehose_report = clanganalyzer.parse_scandir(resultdir)
for report in scanbuild_tmp_firehose_report:
metadata = report.metadata
scanbuild_report_results += report.results
scanbuild_firehose_report = Analysis(metadata, scanbuild_report_results)
reports.append(scanbuild_firehose_report)
# framac
with open('reports/frama-c/frama-c.log') as framac_raw_report:
framac_firehose_report = frama_c.parse_file(framac_raw_report)
reports.append(framac_firehose_report)
# flawfinder
with open('reports/flawfinder/flawfinder.log') as flawfinder_raw_report:
flawfinder_firehose_report = flawfinder.parse_file(flawfinder_raw_report)
reports.append(flawfinder_firehose_report)
# cppcheck
with open('reports/cppcheck/cppcheck.log') as cppcheck_raw_report:
cppcheck_firehose_report = cppcheck.parse_file(cppcheck_raw_report)
reports.append(cppcheck_firehose_report)
os.makedirs('reports/firehose')
for report in reports:
tool_name = report.metadata.generator.name
fh_report_path = 'reports/firehose/' + tool_name + '.xml'
with open(fh_report_path, 'wb') as fh_report:
report.to_xml().write(fh_report, encoding='utf-8')
def parse_coccinelle(lines):
# we have to emulate a file for from_xml()
firehose = StringIO.StringIO(lines)
analysis = Analysis.from_xml(firehose)
for result in analysis.results:
yield result
def invoke_side_effects(argv):
log("invoke_side_effects: %s"
% ' '.join(sys.argv))
gccinv = GccInvocation(argv)
# Try to run each side effect in a subprocess, passing in a path
# for the XML results to be written to.
# Cover a multitude of possible failures by detecting if no output
# was written, and capturing *that* as a failure
for sourcefile in gccinv.sources:
if sourcefile.endswith('.c'): # FIXME: other extensions?
for script, genname in [('invoke-cppcheck', 'cppcheck'),
('invoke-clang-analyzer', 'clang-analyzer'),
('invoke-cpychecker', 'cpychecker'),
# Uncomment the following to test a
# checker that fails to write any XML:
# ('echo', 'failing-checker'),
]:
with tempfile.NamedTemporaryFile() as f:
dstxmlpath = f.name
assert not os.path.exists(dstxmlpath)
# Restrict the invocation to just one source file at a
# time:
singleinv = gccinv.restrict_to_one_source(sourcefile)
singleargv = singleinv.argv
TIMEOUT=60
t = Timer()
args = [script, dstxmlpath] + singleargv
log('invoking args: %r' % args)
p = Popen(args,
stdout=PIPE, stderr=PIPE)
try:
out, err = p.communicate(timeout=TIMEOUT)
write_streams(script, out, err)
if os.path.exists(dstxmlpath):
with open(dstxmlpath) as f:
analysis = Analysis.from_xml(f)
else:
analysis = make_failed_analysis(genname, sourcefile, t,
msgtext=('Unable to locate XML output from %s'
% script),
failureid='no-output-found')
analysis.set_custom_field('stdout', out)
analysis.set_custom_field('stderr', err)
analysis.set_custom_field('returncode', p.returncode)
except TimeoutExpired:
analysis = make_failed_analysis(genname, sourcefile, t,
msgtext='Timeout running %s' % genname,
failureid='timeout')
analysis.set_custom_field('timeout', TIMEOUT)
analysis.set_custom_field('gcc-invocation', ' '.join(argv))
write_analysis_as_xml(analysis)
def parse_file(fileobj, sut=None, file_=None, stats=None):
tree = ET.parse(fileobj)
root = tree.getroot()
node_cppcheck = root.find('cppcheck')
version = node_cppcheck.get('version')
node_errors = root.find('errors')
generator = Generator(name='cppcheck',
version=node_cppcheck.get('version'))
metadata = Metadata(generator, sut, file_, stats)
analysis = Analysis(metadata, [])
for node_error in node_errors.findall('error'):
# e.g.:
# <error id="nullPointer" severity="error" msg="Possible null pointer dereference: end - otherwise it is redundant to check it against null." verbose="Possible null pointer dereference: end - otherwise it is redundant to check it against null.">
# <location file="python-ethtool/ethtool.c" line="139"/>
# <location file="python-ethtool/ethtool.c" line="141"/>
# </error>
testid = node_error.get('id')
str_msg = node_error.get('msg')
str_verbose = node_error.get('verbose')
message = Message(text=str_msg)
if str_verbose != str_msg:
notes = Notes(str_verbose)
else:
notes = None
location_nodes = list(node_error.findall('location'))
for node_location in location_nodes:
location = Location(
file=File(node_location.get('file'), None),
# FIXME: doesn't tell us function name
# TODO: can we patch this upstream?
function=None,
# doesn't emit column
point=Point(int(node_location.get('line')),
0)) # FIXME: bogus column
issue = Issue(None,
testid,
location,
message,
notes,
None,
severity=node_error.get('severity'))
analysis.results.append(issue)
if not location_nodes:
customfields = CustomFields()
if str_verbose != str_msg:
customfields['verbose'] = str_verbose
failure = Failure(failureid=testid,
location=None,
message=message,
customfields=customfields)
analysis.results.append(failure)
return analysis
def test_from_xml(self):
num_analyses = 0
for filename in sorted(glob.glob('examples/example-*.xml')):
with open(filename) as f:
r = Analysis.from_xml(f)
num_analyses += 1
# Ensure that all of the reports were indeed parsed:
self.assertEqual(num_analyses, 9)
def test_from_xml(self):
num_analyses = 0
for filename in sorted(glob.glob('examples/example-*.xml')):
with open(filename) as f:
r = Analysis.from_xml(f)
num_analyses += 1
# Ensure that all of the reports were indeed parsed:
self.assertEqual(num_analyses, 10)
class Pep8TestCase(unittest.TestCase):
filepath = "tests/resources/python-firehose_0.3-1.dsc"
firehose_results = Analysis(
metadata=Metadata(
generator=Generator(
name='pep8'
),
sut=None,
file_=None,
stats=None),
results=[]
)
@mock.patch('debile.slave.runners.pep8.run_command',
return_value=('1.5.7', '', 0))
def test_pep8_version(self, mock):
name, ver = version()
self.assertEquals(name, 'pep8')
self.assertEquals(ver, '1.5.7')
@mock.patch('debile.slave.runners.pep8.run_command',
return_value=('1.5.7', '', 1))
def test_pep8_version(self, mock):
self.assertRaises(Exception, version)
def test_pep8(self):
pep8_analysis = pep8(self.filepath, self.firehose_results)
content = pep8_analysis[1]
self.assertTrue("missing whitespace around operator" in content)
# It think it is safe to say that the string is not 4 chars long
self.assertTrue(len(content) > 4)
def test_pep8_wrappers(self):
pep8_analysis = pep8(self.filepath, self.firehose_results)
issues = parse_pep8(pep8_analysis[1].splitlines())
i = 0
for issue in issues:
if issue.location.file.givenpath == "./firehose/model.py" and \
issue.location.point.line==96 and issue.location.point.column==1:
found = issue
i += 1
print found
self.assertEquals(found.testid, "E302")
self.assertEquals(found.location.file.givenpath, "./firehose/model.py")
self.assertEquals(found.location.point.line, 96)
self.assertEquals(found.location.point.column, 1)
self.assertEquals(found.severity, "error")
self.assertIsNone(found.notes)
self.assertIsNone(found.customfields)
self.assertTrue("E302 expected 2 blank lines, found 1" in found.message.text)
self.assertTrue(i > 100)
def roundtrip_example(self, filename):
fullpath = os.path.join('../firehose/examples', filename) # FIXME
a = Analysis.from_xml(fullpath)
self.session.add(a)
self.session.commit()
session2 = self.Session()
b = session2.query(Analysis).first()
#print(b)
self.assertEqual(a, b)
def test_cppcheck_common():
firehorse_results = Analysis(
metadata=Metadata(generator=Generator(name='cppcheck'),
sut=None,
file_=None,
stats=None),
results=[])
return cppcheck("tests/resources/libjsoncpp_0.6.0~rc2-3.1.dsc",
firehorse_results)
def get_labeled_reports(*exclude):
results = []
firehose_reports = glob.glob(os.path.join('reports', 'firehose', 'labeled_reports', '*.xml'))
for analyzer in exclude:
if analyzer == 'scan-build':
analyzer = 'clang-analyzer'
firehose_reports.remove('reports/firehose/labeled_reports/%s.xml' % analyzer)
for fh_xml_file in firehose_reports:
results.append(Analysis.from_xml(fh_xml_file))
return results
def roundtrip_example(self, filename):
fullpath = os.path.join('../firehose/examples', filename) # FIXME
a = Analysis.from_xml(fullpath)
self.session.add(a)
self.session.commit()
session2 = self.Session()
b = session2.query(Analysis).first()
#print(b)
self.assertEqual(a, b)
def test_debian_source(self):
""" Test to ensure that Debian source package Sut loading works. """
with open('examples/example-debian-source.xml') as f:
a = Analysis.from_xml(f)
self.assertEqual(a.metadata.generator.name, 'handmade')
self.assertEqual(a.metadata.generator.version, '0.1')
self.assertIsInstance(a.metadata.sut, DebianSource)
self.assertEqual(a.metadata.sut.name, 'python-ethtool')
self.assertEqual(a.metadata.sut.version, '0.7')
self.assertEqual(a.metadata.sut.release, '4.1+b1')
self.assertFalse(hasattr(a.metadata.sut, 'buildarch'))
def test_debian_binary(self):
""" Test to ensure that Debian binary package Sut loading works. """
with open('examples/example-debian-binary.xml') as f:
a = Analysis.from_xml(f)
self.assertEqual(a.metadata.generator.name, 'handmade')
self.assertEqual(a.metadata.generator.version, '0.1')
self.assertIsInstance(a.metadata.sut, DebianBinary)
self.assertEqual(a.metadata.sut.name, 'python-ethtool')
self.assertEqual(a.metadata.sut.version, '0.7')
self.assertEqual(a.metadata.sut.buildarch, 'amd64')
self.assertEqual(a.metadata.sut.release, '1.1')
def test_debian_source(self):
""" Test to ensure that Debian source package Sut loading works. """
with open('examples/example-debian-source.xml') as f:
a = Analysis.from_xml(f)
self.assertEqual(a.metadata.generator.name, 'handmade')
self.assertEqual(a.metadata.generator.version, '0.1')
self.assertIsInstance(a.metadata.sut, DebianSource)
self.assertEqual(a.metadata.sut.name, 'python-ethtool')
self.assertEqual(a.metadata.sut.version, '0.7')
self.assertEqual(a.metadata.sut.release, '4.1+b1')
self.assertFalse(hasattr(a.metadata.sut, 'buildarch'))
def test_debian_binary(self):
""" Test to ensure that Debian binary package Sut loading works. """
with open('examples/example-debian-binary.xml') as f:
a = Analysis.from_xml(f)
self.assertEqual(a.metadata.generator.name, 'handmade')
self.assertEqual(a.metadata.generator.version, '0.1')
self.assertIsInstance(a.metadata.sut, DebianBinary)
self.assertEqual(a.metadata.sut.name, 'python-ethtool')
self.assertEqual(a.metadata.sut.version, '0.7')
self.assertEqual(a.metadata.sut.buildarch, 'amd64')
self.assertEqual(a.metadata.sut.release, '1.1')
def parse_json_v2(path):
"""
Given a JSON file emitted by:
cov-format-errors --json-output-v2=<filename>
parse it and return an Analysis instance
"""
with open(path) as f:
js = json.load(f)
if 0:
pprint(js)
generator = Generator(name='coverity')
metadata = Metadata(generator, sut=None, file_=None, stats=None)
analysis = Analysis(metadata, [])
for issue in js['issues']:
if 0:
pprint(issue)
cwe = None
# Use checkerName (e.g. "RESOURCE_LEAK") for
# the testid:
testid = issue['checkerName']
# Use the eventDescription of the final event for the message:
message = Message(text=issue['events'][-1]['eventDescription'])
location = Location(
file=File(givenpath=issue['mainEventFilePathname'], abspath=None),
function=Function(name=issue['functionDisplayName']),
point=Point(int(issue['mainEventLineNumber']), int(0)))
notes = None
trace = make_trace(issue)
customfields = CustomFields()
for key in ['mergeKey', 'subcategory', 'domain']:
if key in issue:
customfields[key] = issue[key]
issue = Issue(cwe,
testid,
location,
message,
notes,
trace,
customfields=customfields)
analysis.results.append(issue)
return analysis
class PylintTestCase(unittest.TestCase):
filepath = "tests/resources/python-firehose_0.3-1.dsc"
firehose_results = Analysis(metadata=Metadata(
generator=Generator(name='pylint'), sut=None, file_=None, stats=None),
results=[])
@mock.patch(
'debile.slave.runners.pylint.run_command',
return_value=(
'pylint 1.4.3,\nastroid 1.3.6, common 0.62.0\nPython 2.7.10', '',
0))
def test_pylint_version(self, mock):
name, ver = version()
self.assertEquals(name, 'pylint')
self.assertEquals(ver, '1.4.3')
@mock.patch('debile.slave.runners.pylint.run_command',
return_value=('', '', 1))
def test_pylint_version_with_error(self, mock):
self.assertRaises(Exception, version)
def test_pylint(self):
pylint_analysis = pylint(self.filepath, self.firehose_results)
content = pylint_analysis[1]
self.assertTrue("Missing method docstring" in content)
# It think it is safe to say that the string is not 4 chars long
self.assertTrue(len(content) > 4)
def test_pylint_wrappers(self):
pylint_analysis = pylint(self.filepath, self.firehose_results)
issues = parse_pylint(pylint_analysis[1].splitlines())
i = 0
for issue in issues:
if issue.location.file.givenpath == \
"tests/parsers/test_clanganalyzer_parser.py" and \
issue.location.point.line==22 and issue.location.point.column==0:
found = issue
i += 1
print found
self.assertEquals(found.testid, "W0611")
self.assertEquals(found.location.file.givenpath,
"tests/parsers/test_clanganalyzer_parser.py")
self.assertEquals(found.location.point.line, 22)
self.assertEquals(found.location.point.column, 0)
self.assertEquals(found.severity, "warning")
self.assertIsNone(found.notes)
self.assertIsNone(found.customfields)
self.assertTrue(
"[unused-import]Unused Analysis imported from firehose.model" in
found.message.text)
self.assertTrue(i > 500)
class Flake8TestCase(unittest.TestCase):
filepath = "tests/resources/python-firehose_0.3-1.dsc"
firehose_results = Analysis(metadata=Metadata(
generator=Generator(name='flake8'), sut=None, file_=None, stats=None),
results=[])
@mock.patch(
'debile.slave.runners.flake8.run_command',
return_value=
('2.2.2 (pep8: 1.5.7, mccabe: 0.2.1, pyflakes: 0.8.1) CPython 2.7.10 on Linux',
'', 0))
def test_flake8_version(self, mock):
name, ver = version()
self.assertEquals(name, 'flake8')
self.assertEquals(ver, '2.2.2')
@mock.patch('debile.slave.runners.flake8.run_command',
return_value=('', '', 1))
def test_flake8_version_with_error(self, mock):
self.assertRaises(Exception, version)
def test_flake8(self):
flake8_analysis = flake8(self.filepath, self.firehose_results)
content = flake8_analysis[1]
self.assertTrue("missing whitespace around operator" in content)
# It think it is safe to say that the string is not 4 chars long
self.assertTrue(len(content) > 4)
def test_flake8_wrappers(self):
flake8_analysis = flake8(self.filepath, self.firehose_results)
issues = parse_flake8(flake8_analysis[1].splitlines())
i = 0
for issue in issues:
if issue.location.file.givenpath == "./firehose/parsers/cppcheck.py" and \
issue.location.point.line==37 and issue.location.point.column==5:
found = issue
i += 1
print found
self.assertEquals(found.testid, "F841")
self.assertEquals(found.location.file.givenpath,
"./firehose/parsers/cppcheck.py")
self.assertEquals(found.location.point.line, 37)
self.assertEquals(found.location.point.column, 5)
self.assertEquals(found.severity, "error")
self.assertIsNone(found.notes)
self.assertIsNone(found.customfields)
self.assertTrue(
"F841 local variable 'version' is assigned to but never used" in
found.message.text)
self.assertTrue(i > 100)
def create_firehose(package, version_getter):
logging.info("Initializing empty firehose report")
sut = {
"sources": generate_sut_from_source,
"binaries": generate_sut_from_binary
}[package['_type']](package)
gname_, gversion = version_getter()
gname = "ethel/%s" % gname_
return Analysis(metadata=Metadata(
generator=Generator(name=gname, version=gversion),
sut=sut, file_=None, stats=None), results=[])
def test_example_6(self):
with open('examples/example-6.xml') as f:
a = Analysis.from_xml(f)
self.assertEqual(a.metadata.generator.name, 'cpychecker')
self.assertEqual(len(a.results), 1)
w = a.results[0]
self.assertIsInstance(w, Failure)
self.assertEqual(w.failureid, 'too-complicated')
self.assertEqual(w.message.text,
'this function is too complicated for the'
' reference-count checker to fully analyze:'
' not all paths were analyzed')
self.assertEqual(w.customfields, None)
def test_example_6(self):
with open('examples/example-6.xml') as f:
a = Analysis.from_xml(f)
self.assertEqual(a.metadata.generator.name, 'cpychecker')
self.assertEqual(len(a.results), 1)
w = a.results[0]
self.assertIsInstance(w, Failure)
self.assertEqual(w.failureid, 'too-complicated')
self.assertEqual(
w.message.text, 'this function is too complicated for the'
' reference-count checker to fully analyze:'
' not all paths were analyzed')
self.assertEqual(w.customfields, None)
def simple_tests():
ex = Analysis.from_xml('../firehose/examples/example-1.xml')
print(ex)
session.add(ex)
print(ex.id)
print(session.query(Analysis).first())
print(session.query(Failure).first())
print(ex.id)
session.commit()
# Do it from a new session:
session2 = Session()
print(session2.query(Analysis).first())
print(session2.query(Failure).first())
def test_example_2(self):
# Verify that the parser works:
with open('examples/example-2.xml') as f:
a = Analysis.from_xml(f)
self.assertEqual(a.metadata.generator.name, 'cpychecker')
self.assertEqual(a.metadata.generator.version, '0.11')
self.assertIsInstance(a.metadata.sut, SourceRpm)
self.assertEqual(a.metadata.sut.name, 'python-ethtool')
self.assertEqual(a.metadata.sut.version, '0.7')
self.assertEqual(a.metadata.sut.release, '4.fc19')
self.assertEqual(a.metadata.sut.buildarch, 'x86_64')
self.assertEqual(len(a.results), 1)
w = a.results[0]
self.assertIsInstance(w, Issue)
self.assertEqual(w.cwe, 401)
self.assertEqual(w.testid, 'refcount-too-high')
self.assertEqual(w.location.file.givenpath,
'examples/python-src-example.c')
self.assertEqual(w.location.file.abspath, None)
self.assertEqual(w.location.file.hash_.alg, 'sha1')
self.assertEqual(w.location.file.hash_.hexdigest,
'6ba29daa94d64b48071e299a79f2a00dcd99eeb1')
self.assertEqual(w.location.function.name,
'make_a_list_of_random_ints_badly')
self.assertEqual(w.location.line, 40)
self.assertEqual(w.location.column, 4)
self.assertEqual(w.message.text,
"ob_refcnt of '*item' is 1 too high")
self.assertMultiLineEqual(w.notes.text, (
"was expecting final item->ob_refcnt to be N + 1 (for some unknown N)\n"
"due to object being referenced by: PyListObject.ob_item[0]\n"
"but final item->ob_refcnt is N + 2"))
self.assertIsInstance(w.trace, Trace)
self.assertEqual(len(w.trace.states), 3)
s0 = w.trace.states[0]
self.assertIsInstance(s0, State)
self.assertEqual(s0.location.file.givenpath,
'examples/python-src-example.c')
self.assertEqual(s0.location.function.name,
'make_a_list_of_random_ints_badly')
self.assertEqual(s0.location.line, 36)
self.assertEqual(s0.location.column, 14)
self.assertEqual(
s0.notes.text,
'PyLongObject allocated at: item = PyLong_FromLong(random());'
)
def make_info(self):
a = Analysis(
metadata=Metadata(generator=Generator(name='an-invented-checker'),
sut=None,
file_=None,
stats=None),
results=[
Info(infoid='gimple-stats',
location=Location(file=File('bar.c', None),
function=Function('sample_function'),
point=Point(10, 15)),
message=Message('sample message'),
customfields=CustomFields(num_stmts=57,
num_basic_blocks=10))
])
return a, a.results[0]
def make_complex_analysis(self):
"""
Construct a Analysis instance that uses all features
"""
a = Analysis(
metadata=Metadata(generator=Generator(name='cpychecker',
version='0.11'),
sut=SourceRpm(name='python-ethtool',
version='0.7',
release='4.fc19',
buildarch='x86_64'),
file_=File(givenpath='foo.c',
abspath='/home/david/coding/foo.c'),
stats=Stats(wallclocktime=0.4)),
results=[
Issue(cwe=681,
testid='refcount-too-high',
location=Location(file=File(
givenpath='foo.c',
abspath='/home/david/coding/foo.c'),
function=Function('bar'),
point=Point(10, 15)),
message=Message(text='something bad involving pointers'),
notes=Notes('here is some explanatory text'),
trace=Trace([
State(location=Location(file=File('foo.c', None),
function=Function('bar'),
point=Point(7, 12)),
notes=Notes('first we do this')),
State(location=Location(file=File('foo.c', None),
function=Function('bar'),
point=Point(8, 10)),
notes=Notes('then we do that')),
State(location=Location(file=File('foo.c', None),
function=Function('bar'),
range_=Range(
Point(10, 15),
Point(10, 25))),
notes=Notes('then it crashes here'))
]),
severity='really bad',
customfields=CustomFields(foo='bar')),
],
customfields=CustomFields(
gccinvocation='gcc -I/usr/include/python2.7 -c foo.c'),
)
return a, a.results[0]
def parse_file(data_file_obj, findbugs_version=None, sut=None, file_=None,
stats=None):
"""
:param data_file_obj: file object containing findbugs scan result
in xml format, it can be generated using
command:
fb analyze -xml:withMessages [jar_file]
:type data_file_obj: file object
:param findbugs_version: version of findbugs
:type findbugs_version: str
:return: Analysis instance
"""
generator = Generator(name = "findbugs",
version = findbugs_version)
metadata = Metadata(generator, sut, file_, stats)
analysis = Analysis(metadata, [])
def parse_BugInstance(bugInstance):
message = Message(bugInstance.find("LongMessage").text)
# findbugs has no column information
sourceLine = bugInstance.find("SourceLine")
point = Point(int(sourceLine.get("start")), 0)
path = sourceLine.get("sourcepath")
path = File(path, None)
method = bugInstance.find("Method")
if method:
function = method.find("Message").text
tmpIndex = function.rfind("In method ") + len("In method ") - 1
function = Function(function[tmpIndex+1:])
else:
function = None
location = Location(path, function, point)
if DEBUG:
print(str(location)+" "+str(message))
return Issue(None, None, location, message, None, None)
tree = ET.parse(data_file_obj)
root = tree.getroot()
for bugInstance in root.findall("BugInstance"):
issue=parse_BugInstance(bugInstance)
if issue:
analysis.results.append(issue)
else:
sys.stderr.write("fail to pass bugInstance=[%s]\n" %
str(bugInstance))
return analysis
class LintianTestCase(unittest.TestCase):
filepath = "tests/resources/libjsoncpp0_0.6.0~rc2-3.1_amd64.deb"
firehose_results = Analysis(metadata=Metadata(
generator=Generator(name='lintian'), sut=None, file_=None, stats=None),
results=[])
@mock.patch('debile.slave.runners.lintian.run_command',
return_value=('Lintian v2.5.31', '', 0))
def test_version(self, mock):
name, ver = version()
self.assertEquals(name, 'Lintian')
self.assertEquals(ver, 'v2.5.31')
@mock.patch('debile.slave.runners.lintian.run_command',
return_value=('Lintian v2.5.31', '', 1))
def test_version_without_lintian(self, mock):
self.assertRaises(Exception, version)
def test_lintian(self):
lintian_analysis = lintian(self.filepath, self.firehose_results)
content = lintian_analysis[1]
self.assertTrue("no-symbols-control-file" in content)
# It think it is safe to say that the string is not 4 chars long
self.assertTrue(len(content) > 4)
def test_lintian_wrappers(self):
lintian_analysis = lintian(self.filepath, self.firehose_results)
issues = parse_lintian(lintian_analysis[1].splitlines(), self.filepath)
i = 0
for issue in issues:
if issue.testid == "no-symbols-control-file":
found = issue
i += 1
self.assertEquals(found.testid, "no-symbols-control-file")
self.assertEquals(
found.location.file.givenpath,
"tests/resources/libjsoncpp0_0.6.0~rc2-3.1_amd64.deb")
self.assertIsNone(found.location.point)
self.assertEquals(found.severity, "info")
self.assertIsNone(found.notes)
self.assertIsNone(found.customfields)
self.assertTrue(
"libjsoncpp0: no-symbols-control-file usr/lib/libjsoncpp.so.0.6.0"
in found.message.text)
self.assertTrue(i > 1)
def test_example_3(self):
# Verify that the parser works:
with open('examples/example-3.xml') as f:
a = Analysis.from_xml(f)
self.assertEqual(a.metadata.generator.name, 'cpychecker')
self.assertEqual(a.metadata.generator.version, '0.11')
self.assertIsInstance(a.metadata.sut, SourceRpm)
self.assertEqual(a.metadata.sut.name, 'python-ethtool')
self.assertEqual(a.metadata.sut.version, '0.7')
self.assertEqual(a.metadata.sut.release, '4.fc19')
self.assertEqual(a.metadata.sut.buildarch, 'x86_64')
self.assertEqual(len(a.results), 1)
w = a.results[0]
self.assertIsInstance(w, Failure)
self.assertEqual(w.failureid, 'bad-exit-code')
self.assertEqual(w.customfields['returncode'], -11)
def test_example_3(self):
# Verify that the parser works:
with open('examples/example-3.xml') as f:
a = Analysis.from_xml(f)
self.assertEqual(a.metadata.generator.name, 'cpychecker')
self.assertEqual(a.metadata.generator.version, '0.11')
self.assertIsInstance(a.metadata.sut, SourceRpm)
self.assertEqual(a.metadata.sut.name, 'python-ethtool')
self.assertEqual(a.metadata.sut.version, '0.7')
self.assertEqual(a.metadata.sut.release, '4.fc19')
self.assertEqual(a.metadata.sut.buildarch, 'x86_64')
self.assertEqual(len(a.results), 1)
w = a.results[0]
self.assertIsInstance(w, Failure)
self.assertEqual(w.failureid, 'bad-exit-code')
self.assertEqual(w.customfields['returncode'], -11)
def handle_output(self, result):
if os.path.exists(self.outputxmlpath):
with open(self.outputxmlpath) as f:
analysis = Analysis.from_xml(f)
analysis.metadata.file_ = make_file(result.sourcefile)
analysis.metadata.stats = make_stats(result.timer)
else:
analysis = \
self._make_failed_analysis(
result.sourcefile, result.timer,
msgtext=('Unable to locate XML output from %s'
% self.name),
failureid='no-output-found')
analysis.set_custom_field('cpychecker-invocation',
' '.join(result.argv))
result.set_custom_fields(analysis)
return analysis
def make_failed_analysis(self):
a = Analysis(
metadata=Metadata(generator=Generator(name='yet-another-checker'),
sut=None,
file_=None,
stats=None),
results=[
Failure(failureid='out-of-memory',
location=Location(
file=File('foo.c', None),
function=Function('something_complicated'),
point=Point(10, 15)),
message=Message('out of memory'),
customfields=CustomFields(stdout='sample stdout',
stderr='sample stderr',
returncode=-9)) # (killed)
])
return a, a.results[0]
def test_example_5(self):
# Ensure that we can load range information from XML
with open('examples/example-5.xml') as f:
a = Analysis.from_xml(f)
self.assertEqual(len(a.results), 1)
w = a.results[0]
self.assertIsInstance(w, Issue)
self.assertEqual(w.location.range_.start.line, 10)
self.assertEqual(w.location.range_.start.column, 9)
self.assertEqual(w.location.range_.end.line, 10)
self.assertEqual(w.location.range_.end.column, 44)
self.assertEqual(w.location.point, None)
# The line/column getters use the start:
self.assertEqual(w.location.line, 10)
self.assertEqual(w.location.column, 9)
def test_example_5(self):
# Ensure that we can load range information from XML
with open('examples/example-5.xml') as f:
a = Analysis.from_xml(f)
self.assertEqual(len(a.results), 1)
w = a.results[0]
self.assertIsInstance(w, Issue)
self.assertEqual(w.location.range_.start.line, 10)
self.assertEqual(w.location.range_.start.column, 9)
self.assertEqual(w.location.range_.end.line, 10)
self.assertEqual(w.location.range_.end.column, 44)
self.assertEqual(w.location.point, None)
# The line/column getters use the start:
self.assertEqual(w.location.line, 10)
self.assertEqual(w.location.column, 9)
def parse_splint_csv(path):
"""
Parse a .csv file written by splint's "-csv FILENAME" option.
Generate a list of Result instances.
"""
generator = Generator(name='splint')
metadata = Metadata(generator, None, None, None)
analysis = Analysis(metadata, [])
with open(path, 'r') as f:
reader = csv.reader(f)
for raw_row in reader:
# Skip the initial title row
if raw_row[0] == 'Warning':
continue
rowobj = parse_row(raw_row)
analysis.results.append(rowobj.to_issue())
return analysis
class FindbugsTestCase(unittest.TestCase):
filepath = 'tests/resources/libjdom1-java_1.1.3-1_all.deb'
firehose_results = Analysis(
metadata=Metadata(
generator=Generator(
name='findbugs'
),
sut=None,
file_=None,
stats=None),
results=[]
)
@mock.patch('debile.slave.runners.findbugs.run_command',
return_value=('2.0.3', '', 0))
def test_version(self, mock):
name, ver = version()
self.assertEquals(name, 'findbugs')
self.assertEquals(ver, '2.0.3')
@mock.patch('debile.slave.runners.findbugs.run_command',
return_value=('2.0.3', '', 1))
def test_version_without_findbugs(self, mock):
self.assertRaises(Exception, version)
def test_findbugs(self):
findbugs_analysis = findbugs(self.filepath, self.firehose_results)
content = findbugs_analysis[1]
self.assertTrue("The following classes needed for analysis" in content)
# It think it is safe to say that the string is not 4 chars long
self.assertTrue(len(content) > 4)
@mock.patch('debile.slave.runners.findbugs.run_command',
return_value=(0, 'error', -1))
def test_findbugs_with_exception(self, mock):
self.assertRaises(Exception, findbugs, self.filepath,
self.firehose_results)
def make_simple_analysis(self):
"""
Construct a minimal Analysis instance
"""
a = Analysis(
metadata=Metadata(generator=Generator(name='cpychecker'),
sut=None,
file_=None,
stats=None),
results=[
Issue(cwe=None,
testid=None,
location=Location(file=File('foo.c', None),
function=None,
point=Point(10, 15)),
message=Message(text='something bad involving pointers'),
notes=None,
trace=None)
])
return a, a.results[0]
def test_example_2(self):
# Verify that the parser works:
with open('examples/example-2.xml') as f:
a = Analysis.from_xml(f)
self.assertEqual(a.metadata.generator.name, 'cpychecker')
self.assertEqual(a.metadata.generator.version, '0.11')
self.assertIsInstance(a.metadata.sut, SourceRpm)
self.assertEqual(a.metadata.sut.name, 'python-ethtool')
self.assertEqual(a.metadata.sut.version, '0.7')
self.assertEqual(a.metadata.sut.release, '4.fc19')
self.assertEqual(a.metadata.sut.buildarch, 'x86_64')
self.assertEqual(len(a.results), 1)
w = a.results[0]
self.assertIsInstance(w, Issue)
self.assertEqual(w.cwe, 401)
self.assertEqual(w.testid, 'refcount-too-high')
self.assertEqual(w.location.file.givenpath, 'examples/python-src-example.c')
self.assertEqual(w.location.file.abspath, None)
self.assertEqual(w.location.file.hash_.alg, 'sha1')
self.assertEqual(w.location.file.hash_.hexdigest,
'6ba29daa94d64b48071e299a79f2a00dcd99eeb1')
self.assertEqual(w.location.function.name, 'make_a_list_of_random_ints_badly')
self.assertEqual(w.location.line, 40)
self.assertEqual(w.location.column, 4)
self.assertEqual(w.message.text, "ob_refcnt of '*item' is 1 too high")
self.assertMultiLineEqual(w.notes.text,
("was expecting final item->ob_refcnt to be N + 1 (for some unknown N)\n"
"due to object being referenced by: PyListObject.ob_item[0]\n"
"but final item->ob_refcnt is N + 2"))
self.assertIsInstance(w.trace, Trace)
self.assertEqual(len(w.trace.states), 3)
s0 = w.trace.states[0]
self.assertIsInstance(s0, State)
self.assertEqual(s0.location.file.givenpath, 'examples/python-src-example.c')
self.assertEqual(s0.location.function.name, 'make_a_list_of_random_ints_badly')
self.assertEqual(s0.location.line, 36)
self.assertEqual(s0.location.column, 14)
self.assertEqual(s0.notes.text,
'PyLongObject allocated at: item = PyLong_FromLong(random());')
def test_json_roundtrip(self):
verbose = False
def roundtrip_through_json(a):
jsondict = a.to_json()
if verbose:
from pprint import pprint
pprint(jsondict)
return Analysis.from_json(jsondict)
a1, w = self.make_simple_analysis()
a2 = roundtrip_through_json(a1)
self.assertEqual(a1.metadata, a2.metadata)
self.assertEqual(a1.results, a2.results)
self.assertEqual(a1, a2)
a3, w = self.make_complex_analysis()
a4 = roundtrip_through_json(a3)
self.assertEqual(a3.metadata, a4.metadata)
self.assertEqual(a3.results, a4.results)
self.assertEqual(a3, a4)
a5, f = self.make_failed_analysis()
a6 = roundtrip_through_json(a5)
self.assertEqual(a5.metadata, a6.metadata)
self.assertEqual(a5.results, a6.results)
self.assertEqual(a5, a6)
a7, info = self.make_info()
a8 = roundtrip_through_json(a7)
self.assertEqual(a7.metadata, a8.metadata)
self.assertEqual(a7.results, a8.results)
self.assertEqual(a7, a8)
a9 = Analysis.from_xml('examples/example-non-ascii.xml')
a10 = roundtrip_through_json(a9)
self.assertEqual(a9, a10)
def test_example_4(self):
with open('examples/example-4.xml') as f:
a = Analysis.from_xml(f)
self.assertEqual(a.metadata.generator.name, 'cpychecker')
self.assertEqual(a.metadata.generator.version, '0.11')
self.assertIsInstance(a.metadata.sut, SourceRpm)
self.assertEqual(a.metadata.sut.name, 'python-ethtool')
self.assertEqual(a.metadata.sut.version, '0.7')
self.assertEqual(a.metadata.sut.release, '4.fc19')
self.assertEqual(a.metadata.sut.buildarch, 'x86_64')
self.assertEqual(len(a.results), 1)
w = a.results[0]
self.assertIsInstance(w, Failure)
self.assertEqual(w.failureid, 'python-exception')
self.assertEqual(w.location.file.givenpath, 'wspy_register.c')
self.assertEqual(w.location.function.name,
'register_all_py_protocols_func')
self.assertEqual(w.location.line, 159)
self.assertEqual(w.location.column, 42)
self.assert_(w.customfields['traceback'].startswith('wspy_register.c: In function \'register_all_py_protocols_func\':\n'))
def test_non_ascii_example(self):
with open('examples/example-non-ascii.xml') as f:
a = Analysis.from_xml(f)
self.assertEqual(a.metadata.generator.name, u('\u2620') * 8)
self.assertEqual(len(a.results), 1)
w = a.results[0]
self.assertIsInstance(w, Issue)
# Verify the Japanese version of
# "comparison between signed and unsigned integer expressions"
# within the message:
self.assertEqual(w.message.text,
(u('\u7b26\u53f7\u4ed8\u304d\u3068\u7b26\u53f7'
'\u7121\u3057\u306e\u6574\u6570\u5f0f\u306e'
'\u9593\u3067\u306e\u6bd4\u8f03\u3067\u3059')))
# Verify the "mojibake" Kanji/Hiragana within the notes:
self.assertIn(u('\u6587\u5b57\u5316\u3051'),
w.notes.text)
self.assertEqual(w.location.function.name, u('oo\u025f'))
def test_xml_roundtrip(self):
def roundtrip_through_xml(a):
xmlbytes = a.to_xml_bytes()
buf = BytesIO(xmlbytes)
return Analysis.from_xml(buf)
a1, w = self.make_simple_analysis()
a2 = roundtrip_through_xml(a1)
self.assertEqual(a1.metadata, a2.metadata)
self.assertEqual(a1.results, a2.results)
self.assertEqual(a1, a2)
a3, w = self.make_complex_analysis()
a4 = roundtrip_through_xml(a3)
self.assertEqual(a3.metadata, a4.metadata)
self.assertEqual(a3.results, a4.results)
self.assertEqual(a3, a4)
a5, f = self.make_failed_analysis()
a6 = roundtrip_through_xml(a5)
self.assertEqual(a5.metadata, a6.metadata)
self.assertEqual(a5.results, a6.results)
self.assertEqual(a5, a6)
a7, info = self.make_info()
a8 = roundtrip_through_xml(a7)
self.assertEqual(a7.metadata, a8.metadata)
self.assertEqual(a7.results, a8.results)
self.assertEqual(a7, a8)
a9 = Analysis.from_xml('examples/example-non-ascii.xml')
a10 = roundtrip_through_xml(a9)
self.assertEqual(a9, a10)
def test_json_roundtrip(self):
def roundtrip_through_json(a):
jsondict = a.to_json()
from pprint import pprint
pprint(jsondict)
return Analysis.from_json(jsondict)
a1, w = self.make_simple_analysis()
a2 = roundtrip_through_json(a1)
self.assertEqual(a1.metadata, a2.metadata)
self.assertEqual(a1.results, a2.results)
self.assertEqual(a1, a2)
a3, w = self.make_complex_analysis()
a4 = roundtrip_through_json(a3)
self.assertEqual(a3.metadata, a4.metadata)
self.assertEqual(a3.results, a4.results)
self.assertEqual(a3, a4)
a5, f = self.make_failed_analysis()
a6 = roundtrip_through_json(a5)
self.assertEqual(a5.metadata, a6.metadata)
self.assertEqual(a5.results, a6.results)
self.assertEqual(a5, a6)
a7, info = self.make_info()
a8 = roundtrip_through_json(a7)
self.assertEqual(a7.metadata, a8.metadata)
self.assertEqual(a7.results, a8.results)
self.assertEqual(a7, a8)
a9 = Analysis.from_xml('examples/example-non-ascii.xml')
a10 = roundtrip_through_json(a9)
self.assertEqual(a9, a10)
def roundtrip_through_xml(a):
xmlbytes = a.to_xml_bytes()
buf = BytesIO(xmlbytes)
return Analysis.from_xml(buf)
def roundtrip_through_json(a):
jsondict = a.to_json()
from pprint import pprint
pprint(jsondict)
return Analysis.from_json(jsondict)
def parse_xml_bytes(self, xmlbytes):
f = BytesIO(xmlbytes)
a = Analysis.from_xml(f)
f.close()
return a
def get_analyses(mockdir):
analyses = []
for filename in glob.glob(os.path.join(mockdir, 'reports', '*.xml')):
r = Analysis.from_xml(filename)
analyses.append( (filename, r) )
return analyses
def get_analyses(self):
analyses = []
for filename in glob.glob(os.path.join(self.get_reports_dir(), '*.xml')):
r = Analysis.from_xml(filename)
analyses.append( (filename, r) )
return analyses
