def _compare_file_regions(self, file_data, file_data_prev):
# compare every region with previous and return number of new lines
matcher = utils.FileRegionsMatcher(file_data, file_data_prev)
result = 0
for region in file_data.iterate_regions():
if matcher.is_matched(region.get_id()) == False:
# if added region, just add the lines
result += region.get_data('std.code.lines', 'total')
elif matcher.is_modified(region.get_id()):
# if modified, add the difference in lines
region_prev = file_data_prev.get_region(
matcher.get_prev_id(region.get_id()))
result += (region.get_data('std.code.lines', 'total') -
region_prev.get_data('std.code.lines', 'total'))
return result
def main(plugin, args):
exit_code = 0
loader_prev = plugin.get_plugin('metrixpp.mpp.dbf').get_loader_prev()
loader = plugin.get_plugin('metrixpp.mpp.dbf').get_loader()
paths = None
if len(args) == 0:
paths = [""]
else:
paths = args
# Try to optimise iterative change scans
modified_file_ids = None
if plugin.mode != plugin.MODE_ALL:
modified_file_ids = get_list_of_modified_files(loader, loader_prev)
for path in paths:
path = utils.preprocess_path(path)
for limit in plugin.iterate_limits():
warns_count = 0
logging.info("Applying limit: " + str(limit))
filters = [limit.filter]
if modified_file_ids != None:
filters.append(('file_id', 'IN', modified_file_ids))
sort_by = None
limit_by = None
limit_warnings = None
if plugin.hotspots != None:
sort_by = limit.field
if limit.type == "max":
sort_by = "-" + sort_by
if plugin.mode == plugin.MODE_ALL:
# if it is not ALL mode, the tool counts number of printed warnings below
limit_by = plugin.hotspots
limit_warnings = plugin.hotspots
selected_data = loader.load_selected_data(limit.namespace,
fields = [limit.field],
path=path,
filters = filters,
sort_by=sort_by,
limit_by=limit_by)
if selected_data == None:
utils.report_bad_path(path)
exit_code += 1
continue
for select_data in selected_data:
if limit_warnings != None and limit_warnings <= 0:
break
is_modified = None
diff = None
file_data = loader.load_file_data(select_data.get_path())
file_data_prev = loader_prev.load_file_data(select_data.get_path())
if file_data_prev != None:
if file_data.get_checksum() == file_data_prev.get_checksum():
diff = 0
is_modified = False
else:
matcher = utils.FileRegionsMatcher(file_data, file_data_prev)
prev_id = matcher.get_prev_id(select_data.get_region().get_id())
if matcher.is_matched(select_data.get_region().get_id()):
if matcher.is_modified(select_data.get_region().get_id()):
is_modified = True
else:
is_modified = False
diff = api.DiffData(select_data,
file_data_prev.get_region(prev_id)).get_data(limit.namespace, limit.field)
if (plugin.is_mode_matched(limit.limit,
select_data.get_data(limit.namespace, limit.field),
diff,
is_modified) == False):
continue
is_sup = is_metric_suppressed(limit.namespace, limit.field, loader, select_data)
if is_sup == True and plugin.no_suppress == False:
continue
region_cursor = 0
region_name = None
if select_data.get_region() != None:
if select_data.get_region().get_type() & limit.region_types == 0:
continue
region_cursor = select_data.get_region().cursor
region_name = select_data.get_region().name
warns_count += 1
exit_code += 1
report_limit_exceeded(select_data.get_path(),
region_cursor,
limit.namespace,
limit.field,
region_name,
select_data.get_data(limit.namespace, limit.field),
diff,
limit.limit,
is_modified,
is_sup)
if limit_warnings != None:
limit_warnings -= 1
cout.notify(path, None, cout.SEVERITY_INFO, "{0} regions exceeded the limit {1}".format(warns_count, str(limit)))
return exit_code
def append_regions(file_data_tree, file_data, file_data_prev, nest_regions):
regions_matcher = None
if file_data_prev != None:
file_data_tree = append_diff(file_data_tree,
file_data_prev.get_data_tree())
regions_matcher = utils.FileRegionsMatcher(file_data, file_data_prev)
if nest_regions == False:
regions = []
for region in file_data.iterate_regions():
region_data_tree = region.get_data_tree()
is_modified = None
if regions_matcher != None and regions_matcher.is_matched(
region.get_id()):
region_data_prev = file_data_prev.get_region(
regions_matcher.get_prev_id(region.get_id()))
region_data_tree = append_diff(
region_data_tree, region_data_prev.get_data_tree())
is_modified = regions_matcher.is_modified(region.get_id())
regions.append({
"info": {
"name": region.name,
'type': api.Region.T().to_str(region.get_type()),
'modified': is_modified,
'cursor': region.cursor,
'line_begin': region.line_begin,
'line_end': region.line_end,
'offset_begin': region.begin,
'offset_end': region.end
},
"data": region_data_tree
})
file_data_tree['regions'] = regions
else:
def append_rec(region_id, file_data_tree, file_data, file_data_prev):
region = file_data.get_region(region_id)
region_data_tree = region.get_data_tree()
is_modified = None
if regions_matcher != None and regions_matcher.is_matched(
region.get_id()):
region_data_prev = file_data_prev.get_region(
regions_matcher.get_prev_id(region.get_id()))
region_data_tree = append_diff(
region_data_tree, region_data_prev.get_data_tree())
is_modified = regions_matcher.is_modified(region.get_id())
result = {
"info": {
"name": region.name,
'type': api.Region.T().to_str(region.get_type()),
'modified': is_modified,
'cursor': region.cursor,
'line_begin': region.line_begin,
'line_end': region.line_end,
'offset_begin': region.begin,
'offset_end': region.end
},
"data": region_data_tree,
"subregions": []
}
for sub_id in file_data.get_region(
region_id).iterate_subregion_ids():
result['subregions'].append(
append_rec(sub_id, file_data_tree, file_data,
file_data_prev))
return result
file_data_tree['regions'] = []
file_data_tree['regions'].append(
append_rec(1, file_data_tree, file_data, file_data_prev))
def load_aggregated_data_with_mode(loader, loader_prev, path, mode):
if mode == Plugin.MODE_ALL:
aggregated_data = loader.load_aggregated_data(path)
aggregated_data_prev = loader_prev.load_aggregated_data(path)
else:
assert (mode == Plugin.MODE_NEW or mode == Plugin.MODE_TOUCHED)
class AggregatedFilteredData(api.AggregatedData):
def __init__(self, loader, path):
super(AggregatedFilteredData, self).__init__(loader, path)
self.in_processing_mode = True
for name in loader.iterate_namespace_names():
namespace = loader.get_namespace(name)
for field in namespace.iterate_field_names():
if namespace.get_field_python_type(field) == str:
# skip string type fields
continue
self.set_data(
name, field, {
'count': 0,
'nonzero': namespace.is_field_non_zero(field),
'min': None,
'max': None,
'total': 0.0,
'avg': None,
'distribution-bars': {},
'sup': 0
})
def get_data_tree(self, namespaces=None):
self.in_processing_mode = False
# need to convert distribution map to a list and calculate average
for name in loader.iterate_namespace_names():
namespace = loader.get_namespace(name)
for field in namespace.iterate_field_names():
if namespace.get_field_python_type(field) == str:
# skip string type fields
continue
data = self.get_data(name, field)
bars_list = []
for metric_value in sorted(
data['distribution-bars'].keys()):
bars_list.append({
'metric':
metric_value,
'count':
data['distribution-bars'][metric_value],
'ratio': ((float(
data['distribution-bars'][metric_value]) /
float(data['count'])))
})
data['distribution-bars'] = bars_list
if data['count'] != 0:
data['avg'] = float(data['total']) / float(
data['count'])
self.set_data(name, field, data)
return super(AggregatedFilteredData,
self).get_data_tree(namespaces=namespaces)
def _append_data(self, orig_data):
# flag to protect ourselves from getting incomplete data
# the workflow in this tool: append data first and after get it using get_data_tree()
assert (self.in_processing_mode == True)
sup_data = orig_data.get_data('std.suppress', 'list')
data = orig_data.get_data_tree()
for namespace in list(data.keys()):
for field in list(data[namespace].keys()):
aggr_data = self.get_data(namespace, field)
metric_value = data[namespace][field]
if isinstance(metric_value, str):
# skip string type fields
continue
if aggr_data['min'] == None or aggr_data[
'min'] > metric_value:
aggr_data['min'] = metric_value
if aggr_data['max'] == None or aggr_data[
'max'] < metric_value:
aggr_data['max'] = metric_value
aggr_data['count'] += 1
aggr_data['total'] += metric_value
# average is calculated later on get_data_tree
if metric_value not in list(
aggr_data['distribution-bars'].keys()):
aggr_data['distribution-bars'][metric_value] = 0
aggr_data['distribution-bars'][metric_value] += 1
if sup_data != None:
if sup_data.find('[{0}:{1}]'.format(
namespace, field)) != -1:
aggr_data['sup'] += 1
self.set_data(namespace, field, aggr_data)
def _append_file_data(self, file_data):
self._append_data(file_data)
for region in file_data.iterate_regions():
self._append_data(region)
result = AggregatedFilteredData(loader, path)
result_prev = AggregatedFilteredData(loader_prev, path)
prev_file_ids = set()
file_data_iterator = loader.iterate_file_data(path)
if file_data_iterator != None:
for file_data in file_data_iterator:
file_data_prev = loader_prev.load_file_data(
file_data.get_path())
if file_data_prev != None:
prev_file_ids.add(file_data_prev.get_id())
if (file_data_prev == None and
(mode == Plugin.MODE_NEW or mode == Plugin.MODE_TOUCHED)):
# new file and required mode matched
logging.info("Processing: " + file_data.get_path() +
" [new]")
result._append_file_data(file_data)
elif (file_data.get_checksum() !=
file_data_prev.get_checksum()):
# modified file and required mode matched
logging.info("Processing: " + file_data.get_path() +
" [modified]")
# append file data without appending regions...
if (mode == Plugin.MODE_TOUCHED):
# if required mode matched
result._append_data(file_data)
result_prev._append_data(file_data_prev)
# process regions separately
matcher = utils.FileRegionsMatcher(file_data,
file_data_prev)
prev_reg_ids = set()
for region in file_data.iterate_regions():
prev_id = matcher.get_prev_id(region.get_id())
if prev_id != None:
prev_reg_ids.add(prev_id)
if (matcher.is_matched(region.get_id()) == False
and (mode == Plugin.MODE_NEW
or mode == Plugin.MODE_TOUCHED)):
# new region
logging.debug("Processing region: " +
region.get_name() + " [new]")
result._append_data(region)
elif matcher.is_modified(region.get_id(
)) and mode == Plugin.MODE_TOUCHED:
# modified region
logging.debug("Processing region: " +
region.get_name() + " [modified]")
result._append_data(region)
result_prev._append_data(
file_data_prev.get_region(prev_id))
if mode == Plugin.MODE_TOUCHED:
for region_prev in file_data_prev.iterate_regions():
if region_prev.get_id() not in prev_reg_ids:
# deleted region
logging.debug("Processing region: " +
region_prev.get_name() +
" [deleted]")
result_prev._append_data(region_prev)
if mode == Plugin.MODE_TOUCHED:
file_data_prev_iterator = loader_prev.iterate_file_data(path)
if file_data_prev_iterator != None:
for file_data_prev in file_data_prev_iterator:
if file_data_prev.get_id() not in prev_file_ids:
# deleted file and required mode matched
logging.info("Processing: " +
file_data_prev.get_path() + " [deleted]")
result_prev._append_file_data(file_data_prev)
return (result, result_prev)
return (aggregated_data, aggregated_data_prev)
def _export_to_stdout(self, paths):
class StdoutWriter(object):
def write(self, *args, **kwargs):
print(args[0].strip())
exit_code = 0
columns = []
columnNames = [
"file", "region", "type", "modified", "line start", "line end"
]
for name in sorted(self.loader.iterate_namespace_names()):
namespace = self.loader.get_namespace(name)
for field in sorted(namespace.iterate_field_names()):
columns.append((name, field))
columnNames.append(name + ":" + field)
writer = StdoutWriter()
csvWriter = csv.writer(writer)
csvWriter.writerow(columnNames)
for path in paths:
path = utils.preprocess_path(path)
files = self.loader.iterate_file_data(path)
if files == None:
utils.report_bad_path(path)
exit_code += 1
continue
for file_data in files:
matcher = None
file_data_prev = self.loader_prev.load_file_data(
file_data.get_path())
if file_data_prev != None:
matcher = utils.FileRegionsMatcher(file_data,
file_data_prev)
for reg in file_data.iterate_regions():
per_reg_data = []
per_reg_data.append(api.Region.T().to_str(reg.get_type()))
if matcher != None and matcher.is_matched(reg.get_id()):
per_reg_data.append(matcher.is_modified(reg.get_id()))
else:
per_reg_data.append(None)
per_reg_data.append(reg.get_line_begin())
per_reg_data.append(reg.get_line_end())
for column in columns:
per_reg_data.append(reg.get_data(column[0], column[1]))
csvWriter.writerow([file_data.get_path(),
reg.get_name()] + per_reg_data)
per_file_data = []
per_file_data.append('file')
if file_data_prev != None:
per_file_data.append(file_data.get_checksum() !=
file_data_prev.get_checksum())
else:
per_file_data.append(None)
per_file_data.append(file_data.get_region(1).get_line_begin())
per_file_data.append(file_data.get_region(1).get_line_end())
for column in columns:
per_file_data.append(
file_data.get_data(column[0], column[1]))
csvWriter.writerow([file_data.get_path(), None] +
per_file_data)
return exit_code