def report(self):
"""Parse GNATmetric XML report and save data to the database.
Returns according to the success of the analysis:
* ``GNAThub.EXEC_SUCCESS``: transactions committed to database
* ``GNAThub.EXEC_FAILURE``: error while parsing the xml report
"""
# Clear existing references only if not incremental run
if not GNAThub.incremental():
self.info('clear existing results if any')
GNAThub.Tool.clear_references(self.name)
self.info('analyse report')
self.tool = GNAThub.Tool(self.name)
self.log.debug('parse XML report: %s', self.output)
try:
tree = ElementTree.parse(self.output)
# Parse the config first to create the GNAThub rules
self.parse_config(tree)
# Fetch all files
files = tree.findall('./file')
total = len(files)
# List of resource messages suitable for tool level bulk insertion
resources_messages = []
for index, node in enumerate(files, start=1):
resource = GNAThub.Resource.get(node.attrib.get('name'))
# Save file level metrics
if not resource:
self.warn('skip "%s" message (file not found)' %
node.attrib.get('name'))
continue
self.firstunit = True
resources_messages.append([resource, self.parse_metrics(node)])
self.tool.add_messages([], self.parse_units(node, resource))
Console.progress(index, total, new_line=(index == total))
# Retrieve the project metrics
resource = GNAThub.Resource(GNAThub.Project.name(),
GNAThub.PROJECT_KIND)
resources_messages.append([resource, self.parse_metrics(tree)])
self.tool.add_messages(resources_messages, [])
except ParseError as why:
self.log.exception('failed to parse XML report')
self.error('%s (%s:%s)' % (why, why.filename, why.lineno))
return GNAThub.EXEC_FAILURE
else:
return GNAThub.EXEC_SUCCESS
def report(self):
"""Parse GNATmetric XML report and save data to the database.
Returns according to the success of the analysis:
* ``GNAThub.EXEC_SUCCESS``: transactions committed to database
* ``GNAThub.EXEC_FAILURE``: error while parsing the xml report
"""
# Clear existing references only if not incremental run
if not GNAThub.incremental():
self.info('clear existing results if any')
GNAThub.Tool.clear_references(self.name)
if not os.path.isfile(self.output):
self.error('no XML report found')
return GNAThub.EXEC_FAILURE
self.info('analyse report')
self.tool = GNAThub.Tool(self.name)
self.log.debug('parse XML report: %s', self.output)
try:
tree = ElementTree.parse(self.output)
# Parse the config first to create the GNAThub rules
self.parse_config(tree)
# Fetch all files
files = tree.findall('./file')
total = len(files)
# List of resource messages suitable for tool level bulk insertion
resources_messages = []
for index, node in enumerate(files, start=1):
resource = GNAThub.Resource.get(node.attrib.get('name'))
# Save file level metrics
if not resource:
self.warn('skip "%s" message (file not found)' %
node.attrib.get('name'))
continue
self.firstunit = True
resources_messages.append([resource, self.parse_metrics(node)])
self.tool.add_messages([], self.parse_units(node, resource))
Console.progress(index, total, new_line=(index == total))
# Retrieve the project metrics
resource = GNAThub.Resource(GNAThub.Project.name(),
GNAThub.PROJECT_KIND)
resources_messages.append([resource, self.parse_metrics(tree)])
self.tool.add_messages(resources_messages, [])
except ParseError as why:
self.log.exception('failed to parse XML report')
self.error('%s (%s:%s)' % (why, why.filename, why.lineno))
return GNAThub.EXEC_FAILURE
else:
return GNAThub.EXEC_SUCCESS
def report(self):
"""Parse GNATcheck output file report.
Returns according to the success of the analysis:
* ``GNAThub.EXEC_SUCCESS``: on successful execution and analysis
* ``GNAThub.EXEC_FAILURE``: on any error
Identify two type of messages with different format:
* basic message
* message for package instantiation
"""
# Clear existing references only if not incremental run
if not GNAThub.incremental():
self.info('clear existing results if any')
GNAThub.Tool.clear_references(self.name)
self.info('analyse report')
self.tool = GNAThub.Tool(self.name)
self.log.debug('parse report: %s', self.output)
if not os.path.exists(self.output):
self.error('no report found')
return GNAThub.EXEC_FAILURE
try:
with open(self.output, 'r') as output:
lines = output.readlines()
total = len(lines)
# Local variables used for exemptions handling
exempted_violation = False
hide_exempted = GNAThub.gnatcheck_hide_exempted()
# Add the tag "exempted" for GNATcheck exempted violations
exempt_tag = GNAThub.Property('gnatcheck:exempted', 'Exempted')
prev_line = ""
for index, line in enumerate(lines, start=1):
self.log.debug('parse line: %s', line)
# check if is a section title
matchTitle = self._TITLE.match(line)
if matchTitle:
stitle = matchTitle.group('stitle')
exempted_violation = stitle in ('Exempted', 'EXEMPTED')
# filter messages if occurs in exempted violation section
import_violation = not exempted_violation or (
exempted_violation and not hide_exempted)
handle_exempted = exempted_violation and not hide_exempted
if import_violation:
if handle_exempted:
match1 = self._MESSAGE.match(line)
if match1:
self.log.debug('matched: %s',
str(match1.groups()))
# Store this line in order to gather next line
# justification if any
if prev_line == "":
prev_line = line
else:
# Second line is a new violation report
match_prev = self._MESSAGE.match(prev_line)
if match_prev:
self.__parse_line_exempted(
match_prev, [exempt_tag])
prev_line = line
# self.__parse_line_exempted(match1,
# [exempt_tag])
else:
if prev_line != "":
if len(line.strip()) != 0:
# Handle justification for prev_line
pmatch = self._MESSAGE.match(prev_line)
if pmatch:
self.__parse_line_exempted(
pmatch, [exempt_tag],
line.strip())
# Reset previous line value
prev_line = ""
else:
match = self._MESSAGE.match(line)
if match:
self.log.debug('matched: %s',
str(match.groups()))
self.__parse_line(match)
else:
match2 = self._MESSAGE_INST.match(line)
if match2:
self.log.debug('matched 2: %s',
str(match2.groups()))
self.__parse_line_inst(match2)
Console.progress(index, total, new_line=(index == total))
except IOError as why:
self.log.exception('failed to parse report')
self.error('%s (%s:%d)' % (
why, os.path.basename(self.output), total))
return GNAThub.EXEC_FAILURE
else:
self.__do_bulk_insert()
return GNAThub.EXEC_SUCCESS
def report(self):
"""Analyse the report files generated by :program:`Gcov`.
Finds all .gcov files in the object directory and parses them.
Sets the exec_status property according to the success of the analysis:
* ``GNAThub.EXEC_SUCCESS``: on successful execution and analysis
* ``GNAThub.EXEC_FAILURE``: on any error
"""
# Clear existing references only if not incremental run
if not GNAThub.incremental():
self.log.info('clear existing results if any')
GNAThub.Tool.clear_references(self.name)
self.info('parse coverage reports (%s)' % self.GCOV_EXT)
# Handle multiple object directories
if GNAThub.Project.object_dirs():
# If there are object directories defined in the project tree, look
# for .gcov files there.
files = []
for obj_dir in GNAThub.Project.object_dirs():
# Fetch all files in project object directories and retrieve
# only .gcov files, absolute path
for filename in os.listdir(obj_dir):
if filename.endswith(self.GCOV_EXT):
files.append(os.path.join(obj_dir, filename))
else:
# If any object directory is defined in .gpr, fetch all files in
# default project object directory and retrieve only .gcov files,
# absolute path
files = [os.path.join(GNAThub.Project.object_dir(), filename)
for filename in os.listdir(GNAThub.Project.object_dir())
if filename.endswith(self.GCOV_EXT)]
# If no .gcov file found, plugin returns on failure
if not files:
self.error('no %s file in object directory' % self.GCOV_EXT)
return GNAThub.EXEC_FAILURE
self.tool = GNAThub.Tool(self.name)
self.rule = GNAThub.Rule('coverage', 'coverage',
GNAThub.METRIC_KIND, self.tool)
total = len(files)
# List of resource messages suitable for tool level bulk insertion
resources_messages = []
try:
for index, filename in enumerate(files, start=1):
# Retrieve source fullname (`filename` is the *.gcov report
# file).
base, _ = os.path.splitext(os.path.basename(filename))
src = GNAThub.Project.source_file(base)
resource = GNAThub.Resource.get(src)
if resource:
self.__process_file(resource, filename, resources_messages)
Console.progress(index, total, new_line=(index == total))
# Tool level insert for resources messages
self.tool.add_messages(resources_messages, [])
except IOError as why:
self.log.exception('failed to parse reports')
self.error(str(why))
return GNAThub.EXEC_FAILURE
else:
return GNAThub.EXEC_SUCCESS
def report(self):
"""Parse GNATstack output file report.
Returns according to the success of the analysis:
* ``GNAThub.EXEC_SUCCESS``: on successful execution and analysis
* ``GNAThub.EXEC_FAILURE``: on any error
"""
# Clear existing references only if not incremental run
if not GNAThub.incremental():
self.log.info('clear existing results if any')
GNAThub.Tool.clear_references(self.name)
self.info('analyse report')
self.tool = GNAThub.Tool(self.name)
if not os.path.exists(self.output):
self.error('no report found')
return GNAThub.EXEC_FAILURE
self.log.debug('parse XML report: %s', self.output)
# List of resource messages suitable for tool level bulk insertion
resources_messages = []
# Map of list of messages by entities
entities_messages_map = {}
# List of entity messages suitable for tool level bulk insertion
entities_messages = []
# List of indirect call location
indirect_loc_list = []
try:
tree = ElementTree.parse(self.output)
global_node = tree.find('./global')
# Retrieve the metrics and the map of subprogram by id
subprograms = tree.find('./subprogramset').findall('./subprogram')
if subprograms:
unknown_global = GNAThub.Rule("Unknown Global Stack Usage",
"Unknown Global Stack Usage",
GNAThub.METRIC_KIND, self.tool)
static_global = GNAThub.Rule("Static Global Stack Usage",
"Static Global Stack Usage",
GNAThub.METRIC_KIND, self.tool)
unknown_local = GNAThub.Rule("Unknown Local Stack Usage",
"Unknown Local Stack Usage",
GNAThub.METRIC_KIND, self.tool)
static_local = GNAThub.Rule("Static Local Stack Usage",
"Static Local Stack Usage",
GNAThub.METRIC_KIND, self.tool)
for node in subprograms:
subprogram_id = node.attrib.get('id')
locations = node.find('./locationset').findall('./location')
global_usage = node.find('./globalstackusage')
local_usage = node.find('./localstackusage')
name = node.attrib.get('prefixname')
if name == "indirect call":
# The columns are only defined here so save them for later
line = locations[0].attrib.get('line')
column = locations[0].attrib.get('column')
indirect_loc_list.append([line, column])
continue
else:
name = self.pp_name(name)
if not locations:
if subprogram_id not in self.subprograms_without_location:
self.subprograms_without_location[subprogram_id] = name
for loc in locations:
file = loc.attrib.get('file')
line = loc.attrib.get('line')
column = loc.attrib.get('column')
if file in self.resources:
resource = self.resources[file]
else:
resource = GNAThub.Resource(file, GNAThub.FILE_KIND)
self.resources[file] = resource
# entities default value for kind is set to "procedure"
entity = GNAThub.Entity(name, "action", int(line),
int(column), int(column), resource)
# Only link the id to the first location of the entity
if subprogram_id not in self.subprograms:
self.subprograms[subprogram_id] = entity
else:
continue
size = global_usage.attrib.get('size')
if global_usage.attrib.get('qualifier') == "UNKNOWN":
metric = unknown_global
else:
metric = static_global
global_metric = GNAThub.Message(metric,
size,
ranking=GNATstack.RANKING)
size = local_usage.attrib.get('size')
if local_usage.attrib.get('qualifier') == "UNKNOWN":
metric = unknown_local
else:
metric = static_local
local_metric = GNAThub.Message(metric,
size,
ranking=GNATstack.RANKING)
entities_messages_map[subprogram_id] = ([[
global_metric, 0, 1, 1
], [local_metric, 0, 1, 1]])
# Analyse the indirect calls
indirects = global_node.find('./indirectset').findall('./indirect')
if indirects:
indirect_rule = GNAThub.Rule("Indirect Call", "Indirect Call",
GNAThub.RULE_KIND, self.tool)
for node in indirects:
indirect_id = node.attrib.get('id')
if indirect_id not in self.subprograms:
continue
set = node.find('./indirectcallset').findall('./indirectcall')
for call in set:
line = call.find('./line').find('./value').text
# Go through the list of saved locations and use the
# line to retrieve a corresponding column
column = 1
pos = -1
for ix in range(len(indirect_loc_list)):
if indirect_loc_list[ix][0] == line:
pos = ix
column = indirect_loc_list[pos][1]
continue
if pos != -1:
indirect_loc_list.pop(pos)
message = GNAThub.Message(indirect_rule,
self.pp_msg(
indirect_id,
"indirect call"),
ranking=GNATstack.RANKING)
entities_messages_map[indirect_id].append(
[message, int(line),
int(column),
int(column)])
# Analyse the external calls
externals = global_node.find('./externalset').findall('./external')
if externals:
external_rule = GNAThub.Rule("External Call", "External Call",
GNAThub.RULE_KIND, self.tool)
for node in externals:
subprogram_id = node.attrib.get('id')
if subprogram_id not in self.subprograms:
continue
message = GNAThub.Message(external_rule,
self.pp_msg(subprogram_id,
"external call"),
ranking=GNATstack.RANKING)
entities_messages_map[subprogram_id].append([message, 0, 1, 1])
# Analyse the potential cycle
cycles = global_node.find('./cycleset').findall('./cycle')
if cycles:
cycle_rule = GNAThub.Rule("Potential Cycle", "Potential Cycle",
GNAThub.RULE_KIND, self.tool)
for node in cycles:
cycle_subprograms = node.findall('./subprogram')
cycle_list = []
for sub in cycle_subprograms:
subprogram_id = sub.attrib.get('id')
cycle_list.append(self.subprograms[subprogram_id].name)
subprogram_id = cycle_subprograms[0].attrib.get('id')
cycle_list.append(self.subprograms[subprogram_id].name)
message = GNAThub.Message(cycle_rule,
"potential cycle detected:\n\t\t" +
"\n\t\t".join(cycle_list),
ranking=GNATstack.RANKING)
entities_messages_map[subprogram_id].append([message, 0, 1, 1])
# Analyse the unbounded frames
unboundeds = (
global_node.find('./unboundedset').findall('./unbounded'))
if unboundeds:
unbounded_rule = GNAThub.Rule("Unbounded Frame",
"Unbounded Frame",
GNAThub.RULE_KIND, self.tool)
for node in unboundeds:
subprogram_id = node.attrib.get('id')
if subprogram_id in self.subprograms:
message = GNAThub.Message(unbounded_rule,
self.pp_msg(
subprogram_id,
"unbounded frame"),
ranking=GNATstack.RANKING)
entities_messages_map[subprogram_id].append(
[message, 0, 1, 1])
# Analyse the entry points
entries = tree.find('./entryset').findall('./entry')
# There is always an entry, so create the rule anyway
entry_rule = GNAThub.Rule("Entry point", "Entry point",
GNAThub.RULE_KIND, self.tool)
for node in entries:
subprogram_id = node.attrib.get('id')
if subprogram_id not in self.subprograms:
continue
entity = self.subprograms[subprogram_id]
local_stack = node.find('./localstackusage')
size = local_stack.attrib.get('size')
qualifier = local_stack.attrib.get('qualifier')
if qualifier == "UNKNOWN":
text = "The estimated"
else:
text = "The"
text += (' call stack size for the entry point "%s" is %s' %
(entity.name, str(size)))
callchain_list = []
for sub in node.find('./callchain').findall('./subprogram'):
chain_id = sub.attrib.get('id')
if chain_id in self.subprograms:
callchain_list.append(self.subprograms[chain_id].name)
elif chain_id in self.subprograms_without_location:
callchain_list.append(
self.subprograms_without_location[chain_id])
else:
continue
text += (" and the callchain is:\n\t\t%s" %
"\n\t\t".join(callchain_list))
message = GNAThub.Message(entry_rule,
text,
ranking=GNATstack.RANKING)
entities_messages_map[subprogram_id].append([message, 0, 1, 1])
# Project message explaining the accuracy of the metrics
accurate = global_node.find('./accurate')
if accurate.find('./value').text == "FALSE":
project = GNAThub.Resource(GNAThub.Project.name(),
GNAThub.PROJECT_KIND)
rule = GNAThub.Rule("Accuracy", "Accuracy", GNAThub.RULE_KIND,
self.tool)
text = ("worst case may not be accurate because of: " +
("indirect calls/" if indirects else "") +
("cycles/" if cycles else "") +
("unbounded frames/" if unboundeds else "") +
("external calls" if externals else ""))
text = text[:-1] if text[-1] == '/' else text
message = GNAThub.Message(rule,
text,
ranking=GNATstack.RANKING)
resources_messages.append([project, [[message, 0, 1, 1]]])
# Insert the messages and the metrics
for key, value in entities_messages_map.items():
entities_messages.append([self.subprograms[key], value])
self.tool.add_messages(resources_messages, entities_messages)
except ParseError as why:
self.log.exception('failed to parse XML report')
self.error('%s (%s:%s)' % (why, why.filename, why.lineno))
return GNAThub.EXEC_FAILURE
else:
return GNAThub.EXEC_SUCCESS
def report(self):
"""Execute CodePeer message reader and parses the output.
Sets the exec_status property according to the success of the analysis:
* ``GNAThub.EXEC_SUCCESS``: on successful execution and analysis
* ``GNAThub.EXEC_FAILURE``: on any error
"""
# Get codepeer version
dest = os.path.join(self.output_dir, 'version.txt')
cmd = 'codepeer -v'
output = subprocess.check_output(cmd, shell=True)
version = ""
try:
version = re.match(".* ([0-9]+\.[0-9]+[w]?) .*",
output).groups()[0]
except Exception as e:
print e
self.log.debug('Codepeer version put in %s', dest)
self.info('Codepeer version put in %s', dest)
with open(dest, 'w') as fd:
fd.write(version)
# Clear existing references only if not incremental run
if not GNAThub.incremental():
self.info('clear existing results if any')
GNAThub.Tool.clear_references(self.name)
self.info('extract results with msg_reader to %s' % self.csv_report)
proc = GNAThub.Run(self.output_dir,
self.__msg_reader_cmd_line(self.csv_report))
if proc.status != 0:
return GNAThub.EXEC_FAILURE
self.info('analyse CSV report form %s' % self.csv_report)
self.tool = GNAThub.Tool(self.name)
self.log.debug('parse report: %s', self.csv_report)
if not os.path.isfile(self.csv_report):
self.error('no report found')
return GNAThub.EXEC_FAILURE
with open(self.csv_report, 'rb') as report:
# Compute the total number of lines for progress report (-1 because
# the first line in irrelevant to the analysis).
index, total = 0, len(report.readlines()) - 1
# Reset the read cursor to the first byte
report.seek(0)
# Create the tag "New" for new CodePeer messages
added_tag = GNAThub.Property('codepeer:added', 'Added')
removed_tag = GNAThub.Property('codepeer:removed', 'Removed')
unchanged_tag = GNAThub.Property('codepeer:unchanged', 'Unchanged')
try:
# Parse the file and drop the first line (containing the
# columns name).
reader = csv.reader(report, quotechar='\"')
# Drop the first line (containing the columns name)
header = reader.next()
self.log.debug('drop header line: %s', header)
# Iterate over each relevant record
for index, record in enumerate(reader, start=1):
self.log.debug('parse record: %r', record)
# Each row is a list of strings:
#
# File, Line, Column, Category, History, Has_Review,
# Ranking, Kind, Message, Classification, CWE, Checks,
# Primary_Checks, Subp, Timestamp, Approved By, Comment,
# Message_Id
(source, line, column, rule, history, has_review, severity,
category, message, classification, cwe, checks, pchecks,
subp, timestamp, app_by, comment,
message_id) = record[:18]
if not severity or severity == 'suppressed':
# Some versions of codepeer report an empty severity
# for suppressed messages: map this to 'info'.
severity = 'info'
rule_id = rule.lower()
self.__add_message(source, line, column, rule_id, message,
severity, message_id, [
added_tag if history == 'added' else
(removed_tag if history == 'removed'
else unchanged_tag)
])
if index % 100 == 1 or index == total:
Console.progress(index,
total,
new_line=(index == total))
except csv.Error as why:
self.log.exception('failed to parse CSV report')
self.error('%s (%s:%d)' %
(why, os.path.basename(self.csv_report), index))
return GNAThub.EXEC_FAILURE
else:
self.__do_bulk_insert()
return GNAThub.EXEC_SUCCESS
def report(self):
"""Analyse the report files generated by :program:`GNATcoverage`.
Finds all .xcov files in the object directory and parses them.
Sets the exec_status property according to the success of the analysis:
* ``GNAThub.EXEC_SUCCESS``: on successful execution and analysis
* ``GNAThub.EXEC_FAILURE``: on any error
"""
# Clear existing references only if not incremental run
if not GNAThub.incremental():
self.log.info('clear existing results if any')
GNAThub.Tool.clear_references(self.name)
# Check if gnatcoverage output folder exist
if not os.path.exists(self.GNATCOVERAGE_OUTPUT):
self.log.info('No gnatcoverage folder in object directory')
return GNAThub.EXEC_FAILURE
self.info('parse coverage reports (%s)' % self.XML_EXT)
# Fetch all files in project object directory and retrieve only
# .xml files, absolute path
file_path = os.path.join(self.GNATCOVERAGE_OUTPUT, 'index.xml')
if not os.path.exists(file_path):
self.error('no index.xml file in object directory')
return GNAThub.EXEC_FAILURE
index_xml = minidom.parse(file_path)
if not index_xml:
self.error('no %s file in object directory' % self.XML_EXT)
return GNAThub.EXEC_FAILURE
files = index_xml.getElementsByTagName('file')
self.tool = GNAThub.Tool(self.name)
for cov_level in ('statement', 'decision', 'condition', 'coverage'):
self.issue_rules[cov_level] = GNAThub.Rule(cov_level, cov_level,
GNAThub.RULE_KIND,
self.tool)
total = files.length
# List of resource messages suitable for tool level bulk insertion
resources_messages = []
try:
for index, file in enumerate(files, start=1):
# Retrieve source fullname
filename = file.attributes['name'].value
src = GNAThub.Project.source_file(filename)
resource = GNAThub.Resource.get(src)
if resource:
self.__process_file(resource, filename, resources_messages)
Console.progress(index, total, new_line=(index == total))
# Tool level insert for resources messages
self.tool.add_messages(resources_messages, [])
except (IOError, ValueError) as why:
self.log.exception('failed to parse reports')
self.error(str(why))
return GNAThub.EXEC_FAILURE
else:
return GNAThub.EXEC_SUCCESS
def report(self):
"""Execute CodePeer message reader and parses the output.
Sets the exec_status property according to the success of the analysis:
* ``GNAThub.EXEC_SUCCESS``: on successful execution and analysis
* ``GNAThub.EXEC_FAILURE``: on any error
"""
# Clear existing references only if not incremental run
if not GNAThub.incremental():
self.info('clear existing results if any')
GNAThub.Tool.clear_references(self.name)
self.info('extract results with msg_reader to %s' % self.csv_report)
proc = GNAThub.Run(
self.output_dir, self.__msg_reader_cmd_line(self.csv_report))
if proc.status != 0:
return GNAThub.EXEC_FAILURE
self.info('analyse CSV report form %s' % self.csv_report)
self.tool = GNAThub.Tool(self.name)
self.log.debug('parse report: %s', self.csv_report)
if not os.path.isfile(self.csv_report):
self.error('no report found')
return GNAThub.EXEC_FAILURE
with open(self.csv_report, 'rb') as report:
# Compute the total number of lines for progress report (-1 because
# the first line in irrelevant to the analysis).
index, total = 0, len(report.readlines()) - 1
# Reset the read cursor to the first byte
report.seek(0)
# Create the tag "New" for new CodePeer messages
added_tag = GNAThub.Property('codepeer:added', 'Added')
removed_tag = GNAThub.Property('codepeer:removed', 'Removed')
unchanged_tag = GNAThub.Property('codepeer:unchanged', 'Unchanged')
try:
# Parse the file and drop the first line (containing the
# columns name).
reader = csv.reader(report, quotechar='\"')
# Drop the first line (containing the columns name)
header = reader.next()
self.log.debug('drop header line: %s', header)
# Iterate over each relevant record
for index, record in enumerate(reader, start=1):
self.log.debug('parse record: %r', record)
# Each row is a list of strings:
#
# File, Line, Column, Category, History, Has_Review,
# Ranking, Kind, Message, Classification, CWE, Checks,
# Primary_Checks, Subp, Timestamp, Approved By, Comment,
# Message_Id
(
source, line, column, rule, history, has_review,
severity, category, message, classification, cwe,
checks, pchecks, subp, timestamp, app_by, comment,
message_id
) = record[:18]
if not severity or severity == 'suppressed':
# Some versions of codepeer report an empty severity
# for suppressed messages: map this to 'info'.
severity = 'info'
rule_id = rule.lower()
self.__add_message(
source, line, column, rule_id, message, severity,
message_id,
[added_tag if history == 'added' else
(removed_tag if history == 'removed' else
unchanged_tag)]
)
if index % 100 == 1 or index == total:
Console.progress(
index, total, new_line=(index == total))
except csv.Error as why:
self.log.exception('failed to parse CSV report')
self.error('%s (%s:%d)' % (
why, os.path.basename(self.csv_report), index))
return GNAThub.EXEC_FAILURE
else:
self.__do_bulk_insert()
return GNAThub.EXEC_SUCCESS
def report(self):
"""Execute GNATprove message reader and parses the output.
Sets the exec_status property according to the success of the analysis:
* ``GNAThub.EXEC_SUCCESS``: on successful execution and analysis
* ``GNAThub.EXEC_FAILURE``: on any error
"""
# Clear existing references only if not incremental run
if not GNAThub.incremental():
self.info('clear existing results if any')
GNAThub.Tool.clear_references(self.name)
self.info('extract results with msg_reader')
proc = GNAThub.Run(self.output_dir,
self.__msg_reader_cmd_line(),
out=self.output)
if proc.status != 0:
return GNAThub.EXEC_FAILURE
self.info('analyse report')
self.tool = GNAThub.Tool(self.name)
self.log.debug('parse report: %s', self.output_dir)
if not os.path.isdir(self.output_dir):
self.error('no report found')
return GNAThub.EXEC_FAILURE
for entry in os.listdir(self.output_dir):
filename, ext = os.path.splitext(entry)
if not ext == '.spark':
continue
self.log.debug('parse file: %s', entry)
try:
with open(os.path.join(self.output_dir, entry), 'rb') as spark:
results = json.load(spark)
for record in chain(results['flow'], results['proof']):
if 'msg_id' not in record or 'file' not in record:
continue
self.log.debug('found record %s', json.dumps(record))
msg_id = record['msg_id']
filename = record['file']
self.msg_ids[(filename, msg_id)] = record
except IOError as why:
self.log.exception('failed to parse GNATprove .spark file')
self.error('%s (%s:%d)' % (why, os.path.basename(self.output)))
try:
with open(self.output, 'rb') as fdin:
# Compute the total number of lines for progress report
lines = fdin.readlines()
index, total = 0, len(lines)
for index, line in enumerate(lines, start=1):
self.log.debug('parse line: %r', line)
match = self._MESSAGE.match(line)
if match:
self.log.debug('matched: %s', str(match.groups()))
self.__parse_line(match)
Console.progress(index, total, new_line=(index == total))
except IOError as why:
self.log.exception('failed to parse GNATprove output')
self.error('%s (%s:%d)' %
(why, os.path.basename(self.output), total))
return GNAThub.EXEC_FAILURE
else:
self.__do_bulk_insert()
return GNAThub.EXEC_SUCCESS
def report(self):
"""Execute GNATprove message reader and parses the output.
Sets the exec_status property according to the success of the analysis:
* ``GNAThub.EXEC_SUCCESS``: on successful execution and analysis
* ``GNAThub.EXEC_FAILURE``: on any error
"""
# Clear existing references only if not incremental run
if not GNAThub.incremental():
self.info('clear existing results if any')
GNAThub.Tool.clear_references(self.name)
self.info('extract results with msg_reader')
proc = GNAThub.Run(
self.output_dir, self.__msg_reader_cmd_line(), out=self.output)
if proc.status != 0:
return GNAThub.EXEC_FAILURE
self.info('analyse report')
self.tool = GNAThub.Tool(self.name)
self.log.debug('parse report: %s', self.output_dir)
if not os.path.isdir(self.output_dir):
self.error('no report found')
return GNAThub.EXEC_FAILURE
for entry in os.listdir(self.output_dir):
filename, ext = os.path.splitext(entry)
if not ext == '.spark':
continue
self.log.debug('parse file: %s', entry)
try:
with open(os.path.join(self.output_dir, entry), 'rb') as spark:
results = json.load(spark)
for record in chain(results['flow'], results['proof']):
if 'msg_id' not in record or 'file' not in record:
continue
self.log.debug('found record %s', json.dumps(record))
msg_id = record['msg_id']
filename = record['file']
self.msg_ids[(filename, msg_id)] = record
except IOError as why:
self.log.exception('failed to parse GNATprove .spark file')
self.error('%s (%s:%d)' % (
why, os.path.basename(self.output)))
try:
with open(self.output, 'rb') as fdin:
# Compute the total number of lines for progress report
lines = fdin.readlines()
index, total = 0, len(lines)
for index, line in enumerate(lines, start=1):
self.log.debug('parse line: %r', line)
match = self._MESSAGE.match(line)
if match:
self.log.debug('matched: %s', str(match.groups()))
self.__parse_line(match)
Console.progress(index, total, new_line=(index == total))
except IOError as why:
self.log.exception('failed to parse GNATprove output')
self.error('%s (%s:%d)' % (
why, os.path.basename(self.output), total))
return GNAThub.EXEC_FAILURE
else:
self.__do_bulk_insert()
return GNAThub.EXEC_SUCCESS
def report(self):
"""Execute GNATprove message reader and parses the output.
Sets the exec_status property according to the success of the analysis:
* ``GNAThub.EXEC_SUCCESS``: on successful execution and analysis
* ``GNAThub.EXEC_FAILURE``: on any error
"""
# Clear existing references only if not incremental run
if not GNAThub.incremental():
self.info('clear existing results if any')
GNAThub.Tool.clear_references(self.name)
self.info('extract results with msg_reader')
proc = GNAThub.Run(self.output_dir,
self.__msg_reader_cmd_line(),
out=self.output)
if proc.status != 0:
return GNAThub.EXEC_FAILURE
self.info('analyse report')
self.tool = GNAThub.Tool(self.name)
# Handle multiple object directories for a given project
if GNAThub.Project.object_dirs():
# If there are multiple object directories defined in the project
# tree will pass here and will look for all .spark files that was
# generated under the object dirs gnatprove's folder
for obj_dir in GNAThub.Project.object_dirs():
# Fetch all files in project object directories and retrieve
# only .spark files from gnatprove folders
gnatprove_dir = os.path.join(obj_dir, 'gnatprove')
if os.path.isdir(gnatprove_dir):
self.log.debug('parse report: %s', gnatprove_dir)
self.__parse_spark_files(gnatprove_dir)
else:
self.log.debug('parse report: %s', self.output_dir)
if not os.path.isdir(self.output_dir):
self.error('no report found')
return GNAThub.EXEC_FAILURE
self.__parse_spark_files(self.output_dir)
try:
with open(self.output, 'r') as fdin:
# Compute the total number of lines for progress report
lines = fdin.readlines()
index, total = 0, len(lines)
for index, line in enumerate(lines, start=1):
self.log.debug('parse line: %r', line)
match = self._MESSAGE.match(line)
if match:
self.log.debug('matched: %s', str(match.groups()))
self.__parse_line(match)
Console.progress(index, total, new_line=(index == total))
except IOError as why:
self.log.exception('failed to parse GNATprove output')
self.error('%s (%s:%d)' %
(why, os.path.basename(self.output), total))
return GNAThub.EXEC_FAILURE
else:
self.__do_bulk_insert()
return GNAThub.EXEC_SUCCESS
def report(self):
"""Parse GNATstack output file report.
Returns according to the success of the analysis:
* ``GNAThub.EXEC_SUCCESS``: on successful execution and analysis
* ``GNAThub.EXEC_FAILURE``: on any error
"""
# Clear existing references only if not incremental run
if not GNAThub.incremental():
self.log.info('clear existing results if any')
GNAThub.Tool.clear_references(self.name)
self.info('analyse report')
self.tool = GNAThub.Tool(self.name)
if not os.path.exists(self.output):
self.error('no report found')
return GNAThub.EXEC_FAILURE
self.log.debug('parse XML report: %s', self.output)
# List of resource messages suitable for tool level bulk insertion
resources_messages = []
# Map of list of messages by entities
entities_messages_map = {}
# List of entity messages suitable for tool level bulk insertion
entities_messages = []
# List of indirect call location
indirect_loc_list = []
try:
tree = ElementTree.parse(self.output)
global_node = tree.find('./global')
# Retrieve the metrics and the map of subprogram by id
subprograms = tree.find('./subprogramset').findall('./subprogram')
if subprograms:
unknown_global = GNAThub.Rule("Unknown Global Stack Usage",
"Unknown Global Stack Usage",
GNAThub.METRIC_KIND,
self.tool)
static_global = GNAThub.Rule("Static Global Stack Usage",
"Static Global Stack Usage",
GNAThub.METRIC_KIND,
self.tool)
unknown_local = GNAThub.Rule("Unknown Local Stack Usage",
"Unknown Local Stack Usage",
GNAThub.METRIC_KIND,
self.tool)
static_local = GNAThub.Rule("Static Local Stack Usage",
"Static Local Stack Usage",
GNAThub.METRIC_KIND,
self.tool)
for node in subprograms:
subprogram_id = node.attrib.get('id')
locations = node.find('./locationset').findall('./location')
global_usage = node.find('./globalstackusage')
local_usage = node.find('./localstackusage')
name = node.attrib.get('prefixname')
if name == "indirect call":
# The columns are only defined here so save them for later
line = locations[0].attrib.get('line')
column = locations[0].attrib.get('column')
indirect_loc_list.append([line, column])
continue
else:
name = self.pretty_print_name(name)
for loc in locations:
file = loc.attrib.get('file')
line = loc.attrib.get('line')
column = loc.attrib.get('column')
if file in self.resources:
resource = self.resources[file]
else:
resource = GNAThub.Resource(file, GNAThub.FILE_KIND)
self.resources[file] = resource
# entities default value for kind is set to "porcedure"
entity = GNAThub.Entity(name, "action",
int(line), int(column),
int(column), resource)
# Only link the id to the first location of the entity
if subprogram_id not in self.subprograms:
self.subprograms[subprogram_id] = entity
else:
continue
size = global_usage.attrib.get('size')
if global_usage.attrib.get('qualifier') == "UNKNOWN":
metric = unknown_global
else:
metric = static_global
global_metric = GNAThub.Message(metric,
size,
ranking=GNATstack.RANKING)
size = local_usage.attrib.get('size')
if local_usage.attrib.get('qualifier') == "UNKNOWN":
metric = unknown_local
else:
metric = static_local
local_metric = GNAThub.Message(metric,
size,
ranking=GNATstack.RANKING)
entities_messages_map[subprogram_id] = (
[[global_metric, 0, 1, 1], [local_metric, 0, 1, 1]])
# Analyse the indirect calls
indirects = global_node.find('./indirectset').findall('./indirect')
if indirects:
indirect_rule = GNAThub.Rule("Indirect Call",
"Indirect Call",
GNAThub.RULE_KIND,
self.tool)
for node in indirects:
indirect_id = node.attrib.get('id')
if indirect_id not in self.subprograms:
continue
set = node.find('./indirectcallset').findall('./indirectcall')
for call in set:
line = call.find('./line').find('./value').text
# Go through the list of saved locations and use the
# line to retrieve a corresponding column
column = 1
pos = -1
for ix in range(len(indirect_loc_list)):
if indirect_loc_list[ix][0] == line:
pos = ix
column = indirect_loc_list[pos][1]
continue
if pos != -1:
indirect_loc_list.pop(pos)
message = GNAThub.Message(indirect_rule,
'indirect call',
ranking=GNATstack.RANKING)
entities_messages_map[indirect_id].append([message,
int(line),
int(column),
int(column)])
# Analyse the external calls
externals = global_node.find('./externalset').findall('./external')
if externals:
external_rule = GNAThub.Rule("External Call",
"External Call",
GNAThub.RULE_KIND,
self.tool)
for node in externals:
subprogram_id = node.attrib.get('id')
if subprogram_id not in self.subprograms:
continue
message = GNAThub.Message(external_rule,
"external call",
ranking=GNATstack.RANKING)
entities_messages_map[subprogram_id].append([message, 0, 1, 1])
# Analyse the potential cycle
cycles = global_node.find('./cycleset').findall('./cycle')
if cycles:
cycle_rule = GNAThub.Rule("Potential Cycle",
"Potential Cycle",
GNAThub.RULE_KIND,
self.tool)
for node in cycles:
cycle_subprograms = node.findall('./subprogram')
cycle_list = []
for sub in cycle_subprograms:
subprogram_id = sub.attrib.get('id')
cycle_list.append(self.subprograms[subprogram_id].name)
subprogram_id = cycle_subprograms[0].attrib.get('id')
cycle_list.append(self.subprograms[subprogram_id].name)
message = GNAThub.Message(cycle_rule,
"potential cycle detected:\n\t\t" +
"\n\t\t".join(cycle_list),
ranking=GNATstack.RANKING)
entities_messages_map[subprogram_id].append([message, 0, 1, 1])
# Analyse the unbounded frames
unboundeds = (
global_node.find('./unboundedset').findall('./unbounded'))
if unboundeds:
unbounded_rule = GNAThub.Rule("Unbounded Frame",
"Unbounded Frame",
GNAThub.RULE_KIND,
self.tool)
for node in unboundeds:
subprogram_id = node.attrib.get('id')
if subprogram_id in self.subprograms:
message = GNAThub.Message(unbounded_rule,
"This frame is unbounded",
ranking=GNATstack.RANKING)
entities_messages_map[subprogram_id].append([message,
0, 1, 1])
# Analyse the entry points
entries = tree.find('./entryset').findall('./entry')
# There is always an entry, so create the rule anyway
entry_rule = GNAThub.Rule("Entry point",
"Entry point",
GNAThub.RULE_KIND,
self.tool)
for node in entries:
subprogram_id = node.attrib.get('id')
if subprogram_id not in self.subprograms:
continue
entity = self.subprograms[subprogram_id]
local_stack = node.find('./localstackusage')
size = local_stack.attrib.get('size')
qualifier = local_stack.attrib.get('qualifier')
if qualifier == "UNKNOWN":
text = "The estimated"
else:
text = "The"
text += (' call stack size for the entry point "%s" is %s' %
(entity.name, str(size)))
callchain_list = []
for sub in node.find('./callchain').findall('./subprogram'):
chain_id = sub.attrib.get('id')
callchain_list.append(self.subprograms[chain_id].name)
text += (" and the callchain is:\n\t\t%s" %
"\n\t\t".join(callchain_list))
message = GNAThub.Message(entry_rule,
text,
ranking=GNATstack.RANKING)
entities_messages_map[subprogram_id].append([message, 0, 1, 1])
# Project message explaining the accuracy of the metrics
accurate = global_node.find('./accurate')
if accurate.find('./value').text == "FALSE":
project = GNAThub.Resource(GNAThub.Project.name(),
GNAThub.PROJECT_KIND)
rule = GNAThub.Rule("Accuracy", "Accuracy",
GNAThub.RULE_KIND, self.tool)
text = ("worst case may not be accurate because of: " +
("indirect calls/" if indirects else "") +
("cycles/" if cycles else "") +
("unbounded frames/" if unboundeds else "") +
("external calls" if externals else ""))
text = text[:-1] if text[-1] == '/' else text
message = GNAThub.Message(rule,
text,
ranking=GNATstack.RANKING)
resources_messages.append([project, [[message, 0, 1, 1]]])
# Insert the messages and the metrics
for key, value in entities_messages_map.iteritems():
entities_messages.append([self.subprograms[key], value])
self.tool.add_messages(resources_messages, entities_messages)
except ParseError as why:
self.log.exception('failed to parse XML report')
self.error('%s (%s:%s)' % (why, why.filename, why.lineno))
return GNAThub.EXEC_FAILURE
else:
return GNAThub.EXEC_SUCCESS
def report(self):
"""Analyse the report files generated by :program:`GNATcoverage`.
Finds all .xcov files in the object directory and parses them.
Sets the exec_status property according to the success of the analysis:
* ``GNAThub.EXEC_SUCCESS``: on successful execution and analysis
* ``GNAThub.EXEC_FAILURE``: on any error
"""
# Clear existing references only if not incremental run
if not GNAThub.incremental():
self.log.info('clear existing results if any')
GNAThub.Tool.clear_references(self.name)
self.info('parse coverage reports (%s)' % self.GNATCOV_EXT)
# Fetch all files in project object directory and retrieve only
# .xcov files, absolute path
files = [os.path.join(GNAThub.Project.object_dir(), filename)
for filename in os.listdir(GNAThub.Project.object_dir())
if filename.endswith(self.GNATCOV_EXT)]
# If no .xcov file found, plugin returns on failure
if not files:
self.error('no %s file in object directory' % self.GNATCOV_EXT)
return GNAThub.EXEC_FAILURE
self.tool = GNAThub.Tool(self.name)
for cov_level in ('stmt', 'decision', 'mcdc'):
self.issue_rules[cov_level] = GNAThub.Rule(
cov_level, cov_level, GNAThub.RULE_KIND, self.tool)
total = len(files)
# List of resource messages suitable for tool level bulk insertion
resources_messages = []
try:
for index, filename in enumerate(files, start=1):
# Retrieve source fullname (`filename` is the *.xcov report
# file).
base, _ = os.path.splitext(os.path.basename(filename))
src = GNAThub.Project.source_file(base)
resource = GNAThub.Resource.get(src)
if resource:
self.__process_file(resource, filename, resources_messages)
Console.progress(index, total, new_line=(index == total))
# Tool level insert for resources messages
self.tool.add_messages(resources_messages, [])
except (IOError, ValueError) as why:
self.log.exception('failed to parse reports')
self.error(str(why))
return GNAThub.EXEC_FAILURE
else:
return GNAThub.EXEC_SUCCESS
def report(self):
"""Parse GNATcheck output file report.
Returns according to the success of the analysis:
* ``GNAThub.EXEC_SUCCESS``: on successful execution and analysis
* ``GNAThub.EXEC_FAILURE``: on any error
Identify two type of messages with different format:
* basic message
* message for package instantiation
"""
# Clear existing references only if not incremental run
if not GNAThub.incremental():
self.info('clear existing results if any')
GNAThub.Tool.clear_references(self.name)
self.info('analyse report')
self.tool = GNAThub.Tool(self.name)
self.log.debug('parse report: %s', self.output)
if not os.path.exists(self.output):
self.error('no report found')
return GNAThub.EXEC_FAILURE
try:
with open(self.output, 'r') as output:
lines = output.readlines()
total = len(lines)
exempted_violation = False
for index, line in enumerate(lines, start=1):
self.log.debug('parse line: %s', line)
# check if is a section title
matchTitle = self._TITLE.match(line)
if matchTitle:
stitle = matchTitle.group('stitle')
exempted_violation = stitle in ('Exempted', 'EXEMPTED')
# filter messages if occurs in exempted violation section
if not exempted_violation:
match = self._MESSAGE.match(line)
if match:
self.log.debug('matched: %s', str(match.groups()))
self.__parse_line(match)
else:
match2 = self._MESSAGE_INST.match(line)
if match2:
self.log.debug('matched 2: %s',
str(match2.groups()))
self.__parse_line_inst(match2)
Console.progress(index, total, new_line=(index == total))
except IOError as why:
self.log.exception('failed to parse report')
self.error('%s (%s:%d)' % (
why, os.path.basename(self.output), total))
return GNAThub.EXEC_FAILURE
else:
self.__do_bulk_insert()
return GNAThub.EXEC_SUCCESS
