def cout_txt_regions(path, regions, indent=0): for region in regions: details = [('Region name', region['info']['name']), ('Region type', region['info']['type']), ('Offsets', str(region['info']['offset_begin']) + "-" + str(region['info']['offset_end'])), ('Line numbers', str(region['info']['line_begin']) + "-" + str(region['info']['line_end'])), ('Modified', str(region['info']['modified']))] for namespace in sorted(list(region['data'].keys())): diff_data = {} if '__diff__' in list(region['data'][namespace].keys()): diff_data = region['data'][namespace]['__diff__'] for field in sorted(list(region['data'][namespace].keys())): diff_str = "" if field == '__diff__': continue if field in list(diff_data.keys()): diff_str = " [" + ("+" if diff_data[field] >= 0 else "") + str(diff_data[field]) + "]" details.append( (namespace + ":" + field, str(region['data'][namespace][field]) + diff_str)) cout.notify(path, region['info']['cursor'], cout.SEVERITY_INFO, "Metrics per '" + region['info']['name'] + "' region", details, indent=indent) if 'subregions' in list(region.keys()): cout_txt_regions(path, region['subregions'], indent=indent + 1)
def report_limit_exceeded(path, cursor, namespace, field, region_name, stat_level, trend_value, stat_limit, is_modified, is_suppressed): if region_name != None: message = "Metric '" + namespace + ":" + field + "' for region '" + region_name + "' exceeds the limit." else: message = "Metric '" + namespace + ":" + field + "' exceeds the limit." details = [("Metric name", namespace + ":" + field), ("Region name", region_name), ("Metric value", stat_level), ("Modified", is_modified), ("Change trend", '{0:{1}}'.format(trend_value, '+' if trend_value else '')), ("Limit", stat_limit), ("Suppressed", is_suppressed)] cout.notify(path, cursor, cout.SEVERITY_WARNING, message, details)
def run(self, args): loader = self.get_plugin('metrixpp.mpp.dbf').get_loader() # get previous db file loader loader_prev = self.get_plugin('metrixpp.mpp.dbf').get_loader_prev() exit_code = 0 for path in (args if len(args) > 0 else [""]): added_lines = 0 file_iterator = loader.iterate_file_data(path) if file_iterator == None: utils.report_bad_path(path) exit_code += 1 continue for file_data in file_iterator: added_lines += self._compare_file(file_data, loader, loader_prev) cout.notify(path, '', cout.SEVERITY_INFO, "Change trend report", [('Added lines', added_lines)]) return exit_code
def main(plugin, args): exit_code = 0 loader_prev = plugin.get_plugin('metrixpp.mpp.dbf').get_loader_prev() loader = plugin.get_plugin('metrixpp.mpp.dbf').get_loader() paths = None if len(args) == 0: paths = [""] else: paths = args # Try to optimise iterative change scans modified_file_ids = None if plugin.mode != plugin.MODE_ALL: modified_file_ids = get_list_of_modified_files(loader, loader_prev) for path in paths: path = utils.preprocess_path(path) for limit in plugin.iterate_limits(): warns_count = 0 logging.info("Applying limit: " + str(limit)) filters = [limit.filter] if modified_file_ids != None: filters.append(('file_id', 'IN', modified_file_ids)) sort_by = None limit_by = None limit_warnings = None if plugin.hotspots != None: sort_by = limit.field if limit.type == "max": sort_by = "-" + sort_by if plugin.mode == plugin.MODE_ALL: # if it is not ALL mode, the tool counts number of printed warnings below limit_by = plugin.hotspots limit_warnings = plugin.hotspots selected_data = loader.load_selected_data(limit.namespace, fields = [limit.field], path=path, filters = filters, sort_by=sort_by, limit_by=limit_by) if selected_data == None: utils.report_bad_path(path) exit_code += 1 continue for select_data in selected_data: if limit_warnings != None and limit_warnings <= 0: break is_modified = None diff = None file_data = loader.load_file_data(select_data.get_path()) file_data_prev = loader_prev.load_file_data(select_data.get_path()) if file_data_prev != None: if file_data.get_checksum() == file_data_prev.get_checksum(): diff = 0 is_modified = False else: matcher = utils.FileRegionsMatcher(file_data, file_data_prev) prev_id = matcher.get_prev_id(select_data.get_region().get_id()) if matcher.is_matched(select_data.get_region().get_id()): if matcher.is_modified(select_data.get_region().get_id()): is_modified = True else: is_modified = False diff = api.DiffData(select_data, file_data_prev.get_region(prev_id)).get_data(limit.namespace, limit.field) if (plugin.is_mode_matched(limit.limit, select_data.get_data(limit.namespace, limit.field), diff, is_modified) == False): continue is_sup = is_metric_suppressed(limit.namespace, limit.field, loader, select_data) if is_sup == True and plugin.no_suppress == False: continue region_cursor = 0 region_name = None if select_data.get_region() != None: if select_data.get_region().get_type() & limit.region_types == 0: continue region_cursor = select_data.get_region().cursor region_name = select_data.get_region().name warns_count += 1 exit_code += 1 report_limit_exceeded(select_data.get_path(), region_cursor, limit.namespace, limit.field, region_name, select_data.get_data(limit.namespace, limit.field), diff, limit.limit, is_modified, is_sup) if limit_warnings != None: limit_warnings -= 1 cout.notify(path, None, cout.SEVERITY_INFO, "{0} regions exceeded the limit {1}".format(warns_count, str(limit))) return exit_code
def cout_txt(data, loader): details = [] for key in list(data['file-data'].keys()): if key == 'regions': cout_txt_regions(data['info']['path'], data['file-data'][key]) else: namespace = key diff_data = {} if '__diff__' in list(data['file-data'][namespace].keys()): diff_data = data['file-data'][namespace]['__diff__'] for field in sorted(list(data['file-data'][namespace].keys())): diff_str = "" if field == '__diff__': continue if field in list(diff_data.keys()): diff_str = " [" + ("+" if diff_data[field] >= 0 else "") + str(diff_data[field]) + "]" details.append( (namespace + ":" + field, str(data['file-data'][namespace][field]) + diff_str)) if len(details) > 0: cout.notify(data['info']['path'], 0, cout.SEVERITY_INFO, "Metrics per file", details) attr_map = { 'total': 'Total', 'avg': 'Average', 'min': 'Minimum', 'max': 'Maximum', } for namespace in sorted(list(data['aggregated-data'].keys())): for field in sorted(list(data['aggregated-data'][namespace].keys())): details = [] diff_data = {} if '__diff__' in list( data['aggregated-data'][namespace][field].keys()): diff_data = data['aggregated-data'][namespace][field][ '__diff__'] for attr in ['avg', 'min', 'max', 'total']: diff_str = "" if attr in list(diff_data.keys()): if isinstance(diff_data[attr], float): diff_str = " [" + ( "+" if diff_data[attr] >= 0 else "") + str( round(diff_data[attr], DIGIT_COUNT)) + "]" else: diff_str = " [" + ("+" if diff_data[attr] >= 0 else "") + str(diff_data[attr]) + "]" if attr == 'avg' and data['aggregated-data'][namespace][field][ 'nonzero'] == True: diff_str += " (excluding zero metric values)" if isinstance(data['aggregated-data'][namespace][field][attr], float): # round the data to reach same results on platforms with different precision details.append( (attr_map[attr], str( round( data['aggregated-data'][namespace][field] [attr], DIGIT_COUNT)) + diff_str)) else: details.append( (attr_map[attr], str(data['aggregated-data'][namespace][field][attr]) + diff_str)) measured = data['aggregated-data'][namespace][field]['count'] if 'count' in list(diff_data.keys()): diff_str = ' [{0:{1}}]'.format( diff_data['count'], '+' if diff_data['count'] >= 0 else '') sup_diff_str = "" if 'sup' in list(diff_data.keys()): sup_diff_str = ' [{0:{1}}]'.format( diff_data['sup'], '+' if diff_data['sup'] >= 0 else '') elem_name = 'regions' if loader.get_namespace( namespace).are_regions_supported() == False: elem_name = 'files' details.append( ('Distribution', '{0}{1} {2} in total (including {3}{4} suppressed)'.format( measured, diff_str, elem_name, data['aggregated-data'][namespace][field]['sup'], sup_diff_str))) details.append( (' Metric value', 'Ratio : R-sum : Number of ' + elem_name)) count_str_len = len(str(measured)) sum_ratio = 0 for bar in data['aggregated-data'][namespace][field][ 'distribution-bars']: sum_ratio += bar['ratio'] diff_str = "" if '__diff__' in list(bar.keys()): if bar['__diff__'] >= 0: diff_str = ' [+{0:<{1}}]'.format( bar['__diff__'], count_str_len) else: diff_str = ' [{0:<{1}}]'.format( bar['__diff__'], count_str_len + 1) if isinstance(bar['metric'], float): metric_str = "{0:.4f}".format(bar['metric']) else: metric_str = str(bar['metric']) metric_str = ( " " * (cout.DETAILS_OFFSET - len(metric_str) - 1)) + metric_str count_str = str(bar['count']) count_str = ((" " * (count_str_len - len(count_str))) + count_str + diff_str + "\t") details.append((metric_str, "{0:.3f}".format(bar['ratio']) + " : " + "{0:.3f}".format(sum_ratio) + " : " + count_str + ('|' * int(bar['ratio'] * 100)))) cout.notify( data['info']['path'], '', # no line number cout.SEVERITY_INFO, "Overall metrics for '" + namespace + ":" + field + "' metric", details) details = [] for each in sorted(data['subdirs']): details.append(('Directory', each)) for each in sorted(data['subfiles']): details.append(('File', each)) if len(details) > 0: cout.notify( data['info']['path'], '', # no line number cout.SEVERITY_INFO, "Directory content:", details)
def parse(self, data): def reset_next_block(start): return { 'name': '', 'start': start, 'cursor': 0, 'type': '', 'confirmed': False } count_mismatched_brackets = 0 text = data.get_content() indent_current = 0 blocks = [{ 'name': '__global__', 'start': 0, 'cursor': 0, 'type': '__global__', 'indent_start': indent_current, 'children': [] }] curblk = 0 next_block = reset_next_block(0) cursor_last_pos = 0 cursor_current = 1 for m in re.finditer(self.regex_cpp, text): # Comment if text[m.start()] == '/': data.add_marker(m.start(), m.end(), api.Marker.T.COMMENT) # String elif text[m.start()] == '"' or text[m.start()] == '\'': data.add_marker(m.start() + 1, m.end() - 1, api.Marker.T.STRING) # Preprocessor (including internal comments) elif text[m.start()] == ' ' or text[m.start()] == '\t' or text[ m.start()] == '#': data.add_marker(m.start(), m.end(), api.Marker.T.PREPROCESSOR) # Statement end elif text[m.start()] == ';': # Reset next block name and start next_block['name'] = "" next_block['start'] = m.end() # potential region start # Template argument closing bracket elif text[m.start()] == '>': # Reset next block name (in order to skip class names in templates), if has not been confirmed before if next_block['confirmed'] == False and ( next_block['type'] == 'class' or next_block['type'] == 'struct'): next_block['name'] = "" # Template argument opening bracket or after class inheritance specification elif text[m.start()] == ':' or text[m.start()] == '<': # .. if goes after calss definition if next_block['type'] == 'class' or next_block[ 'type'] == 'struct': next_block['confirmed'] = True # Double end line elif text[m.start()] == '\n' or text[m.start()] == '\r': # Reset next block start, if has not been named yet if next_block['name'] == "": next_block['start'] = m.end() # potential region start # Block start... elif text[m.start()] == '{': # shift indent right indent_current += 1 # ... if name detected previously if next_block['name'] != '': # - Start of enclosed block blocks.append({ 'name': next_block['name'], 'start': next_block['start'], 'cursor': next_block['cursor'], 'type': next_block['type'], 'indent_start': indent_current, 'children': [] }) next_block = reset_next_block(m.end()) curblk += 1 # ... reset next block start, otherwise else: # - unknown type of block start next_block['start'] = m.end() # potential region start # Block end... elif text[m.start()] == '}': # ... if indent level matches the start if blocks[curblk]['indent_start'] == indent_current: next_block = reset_next_block(m.end()) if curblk == 0: cout.notify( data.get_path(), cursor_current + len( self.regex_ln.findall(text, cursor_last_pos, m.start())), cout.SEVERITY_WARNING, "Non-matching closing bracket '}' detected.") count_mismatched_brackets += 1 continue self.finalize_block(text, blocks[curblk], m.end()) assert (blocks[curblk]['type'] != '__global__') curblk -= 1 assert (curblk >= 0) blocks[curblk]['children'].append(blocks.pop()) # shift indent left indent_current -= 1 if indent_current < 0: cout.notify( data.get_path(), cursor_current + len( self.regex_ln.findall(text, cursor_last_pos, m.start())), cout.SEVERITY_WARNING, "Non-matching closing bracket '}' detected.") count_mismatched_brackets += 1 indent_current = 0 # Potential namespace, struct, class elif m.group('block_type') != None: if next_block['name'] == "": # - 'name' next_block['name'] = m.group('block_name').strip() if next_block['name'] == "": next_block['name'] = '__noname__' # - 'cursor' cursor_current += len( self.regex_ln.findall(text, cursor_last_pos, m.start('block_name'))) cursor_last_pos = m.start('block_name') next_block['cursor'] = cursor_current # - 'type' next_block['type'] = m.group('block_type').strip() # - 'start' detected earlier # Potential function name detected... elif m.group('fn_name') != None: # ... if outside of a function (do not detect enclosed functions, unless classes are matched) # wander why 'or next_block['type'] != 'function'' is in the condition? # - remove it, run the tests and will see if blocks[curblk]['type'] != 'function' and ( next_block['name'] == "" or next_block['type'] != 'function'): # - 'name' next_block['name'] = m.group('fn_name').strip() # - 'cursor' cursor_current += len( self.regex_ln.findall(text, cursor_last_pos, m.start('fn_name'))) cursor_last_pos = m.start('fn_name') # NOTE: cursor could be collected together with line_begin, line_end, # but we keep it here separately for easier debugging of file parsing problems next_block['cursor'] = cursor_current # - 'type' next_block['type'] = 'function' # - 'start' detected earlier else: assert (len("Unknown match by regular expression") == 0) while indent_current > 0: # log all cout.notify( data.get_path(), cursor_current + len(self.regex_ln.findall(text, cursor_last_pos, len(text))), cout.SEVERITY_WARNING, "Non-matching opening bracket '{' detected.") count_mismatched_brackets += 1 indent_current -= 1 for (ind, each) in enumerate(blocks): each = each # used block = blocks[len(blocks) - 1 - ind] self.finalize_block(text, block, len(text)) self.add_lines_data(text, blocks) self.add_regions(data, blocks) return count_mismatched_brackets
def parse(self, data): def reset_next_block(start): return { 'name': '', 'start': start, 'cursor': 0, 'type': '', 'inside_attribute': False } count_mismatched_brackets = 0 text = data.get_content() indent_current = 0 blocks = [{ 'name': '__global__', 'start': 0, 'cursor': 0, 'type': '__global__', 'indent_start': indent_current, 'children': [] }] curblk = 0 next_block = reset_next_block(0) cursor_last_pos = 0 cursor_current = 1 for m in re.finditer(self.regex_cpp, text): # Comment if text[m.start()] == '/': data.add_marker(m.start(), m.end(), api.Marker.T.COMMENT) # String elif text[m.start()] == '"' or text[m.start()] == '\'': data.add_marker(m.start() + 1, m.end() - 1, api.Marker.T.STRING) # Preprocessor (including internal comments) elif text[m.start()] == ' ' or text[m.start()] == '\t' or text[ m.start()] == '#': data.add_marker(m.start(), m.end(), api.Marker.T.PREPROCESSOR) # Statement end elif text[m.start()] == ';': # Reset next block name and start next_block['name'] = "" next_block['start'] = m.end() # potential region start # Block openned by '[' bracket... elif text[m.start()] == '[': # ... may include attributes, so do not capture function names inside next_block['inside_attribute'] = True # Block closed by ']' bracket... # note: do not care about nesting for simplicity - # because attribute's statement can not have symbol ']' inside elif text[m.start()] == ']': # ... may include attributes, so do not capture function names inside next_block['inside_attribute'] = False # Double end line elif text[m.start()] == '\n' or text[m.start()] == '\r': # Reset next block start, if has not been named yet if next_block['name'] == "": next_block['start'] = m.end() # potential region start # Block start... elif text[m.start()] == '{': # shift indent right indent_current += 1 # ... if name detected previously if next_block['name'] != '': # - Start of enclosed block blocks.append({ 'name': next_block['name'], 'start': next_block['start'], 'cursor': next_block['cursor'], 'type': next_block['type'], 'indent_start': indent_current, 'children': [] }) next_block = reset_next_block(m.end()) curblk += 1 # ... reset next block start, otherwise else: # - unknown type of block start next_block['start'] = m.end() # potential region start # Block end... elif text[m.start()] == '}': # ... if indent level matches the start if blocks[curblk]['indent_start'] == indent_current: next_block = reset_next_block(m.end()) if curblk == 0: cout.notify( data.get_path(), cursor_current + len( self.regex_ln.findall(text, cursor_last_pos, m.start())), cout.SEVERITY_WARNING, "Non-matching closing bracket '}' detected.") count_mismatched_brackets += 1 continue self.finalize_block(text, blocks[curblk], m.end()) assert (blocks[curblk]['type'] != '__global__') curblk -= 1 assert (curblk >= 0) blocks[curblk]['children'].append(blocks.pop()) # shift indent left indent_current -= 1 if indent_current < 0: cout.notify( data.get_path(), cursor_current + len( self.regex_ln.findall(text, cursor_last_pos, m.start())), cout.SEVERITY_WARNING, "Non-matching closing bracket '}' detected.") count_mismatched_brackets += 1 indent_current = 0 # Potential namespace, struct, class, interface elif m.group('block_type') != None: if next_block['name'] == "": # - 'name' clearance_pattern = re.compile(r'\s+') next_block['name'] = clearance_pattern.sub( '', m.group('block_name')) # - 'cursor' cursor_current += len( self.regex_ln.findall(text, cursor_last_pos, m.start('block_name'))) cursor_last_pos = m.start('block_name') next_block['cursor'] = cursor_current # - 'type' next_block['type'] = m.group('block_type').strip() # - 'start' detected earlier # Potential function name detected... elif m.group('fn_name') != None: # ... if outside of a function # (do not detect functions enclosed directly in a function, i.e. without classes) # ... and other name before has not been matched if blocks[curblk]['type'] != 'function' and (next_block['name'] == "") \ and next_block['inside_attribute'] == False: # - 'name' clearance_pattern = re.compile(r'\s+') next_block['name'] = clearance_pattern.sub( '', m.group('fn_name')) # - 'cursor' cursor_current += len( self.regex_ln.findall(text, cursor_last_pos, m.start('fn_name'))) cursor_last_pos = m.start('fn_name') # NOTE: cursor could be collected together with line_begin, line_end, # but we keep it here separately for easier debugging of file parsing problems next_block['cursor'] = cursor_current # - 'type' next_block['type'] = 'function' # - 'start' detected earlier else: assert (len("Unknown match by regular expression") == 0) while indent_current > 0: # log all cout.notify( data.get_path(), cursor_current + len(self.regex_ln.findall(text, cursor_last_pos, len(text))), cout.SEVERITY_WARNING, "Non-matching opening bracket '{' detected.") count_mismatched_brackets += 1 indent_current -= 1 for (ind, each) in enumerate(blocks): each = each # used block = blocks[len(blocks) - 1 - ind] self.finalize_block(text, block, len(text)) self.add_lines_data(text, blocks) self.add_regions(data, blocks) return count_mismatched_brackets
def run(self, args): exit_code = 0 loader_prev = self.get_plugin('metrixpp.mpp.dbf').get_loader_prev( none_if_empty=True) loader = self.get_plugin('metrixpp.mpp.dbf').get_loader() details = [] for each in loader.iterate_properties(): prev_value_str = "" if loader_prev != None: prev = loader_prev.get_property(each.name) if prev == None: prev_value_str = " [new]" elif prev != each.value: prev_value_str = " [modified (was: " + loader_prev.get_property( each.name) + ")]" details.append((each.name, each.value + prev_value_str)) path = self.get_plugin('metrixpp.mpp.dbf').get_dbfile_path() if ('METRIXPLUSPLUS_TEST_MODE' in list(os.environ.keys()) and os.environ['METRIXPLUSPLUS_TEST_MODE'] == "True"): # in tests, paths come as full paths, strip it for consistent gold files # TODO: if there are other path-like arguments, it is better to think about other solution path = os.path.basename(path) cout.notify(path, '', cout.SEVERITY_INFO, 'Created using plugins and settings:', details) details = [] for each in sorted(loader.iterate_namespace_names()): for field in sorted( loader.get_namespace(each).iterate_field_names()): prev_value_str = "" if loader_prev != None: prev = False prev_namespace = loader_prev.get_namespace(each) if prev_namespace != None: prev = prev_namespace.check_field(field) if prev == False: prev_value_str = " [new]" details.append((each + ':' + field, prev_value_str)) cout.notify(path, '', cout.SEVERITY_INFO, 'Collected metrics:', details) paths = None if len(args) == 0: paths = [""] else: paths = args for path in paths: details = [] path = utils.preprocess_path(path) file_iterator = loader.iterate_file_data(path=path) if file_iterator == None: utils.report_bad_path(path) exit_code += 1 continue for each in file_iterator: prev_value_str = "" if loader_prev != None: prev = loader_prev.load_file_data(each.get_path()) if prev == None: prev_value_str = " [new]" elif prev.get_checksum() != each.get_checksum(): prev_value_str = " [modified]" details.append( (each.get_path(), '{0:#x}'.format(each.get_checksum()) + prev_value_str)) cout.notify(path, '', cout.SEVERITY_INFO, 'Processed files and checksums:', details) return exit_code
def callback(self, parent, data, is_updated): is_updated = is_updated or self.is_updated if is_updated == True: text = data.get_content() file_count = 0 file_list_text = [] for region in data.iterate_regions(): count = 0 list_text = [] last_comment_end = None for marker in data.iterate_markers( filter_group=api.Marker.T.COMMENT, region_id=region.get_id(), exclude_children=True): if last_comment_end != None and len( text[last_comment_end:marker.get_offset_begin( )].strip()) > 0: # check continues comment blocks # stop searching, if this comment block is separated from the last by non-blank string break last_comment_end = marker.get_offset_end() matches = self.pattern.findall(text, marker.get_offset_begin(), marker.get_offset_end()) for m in matches: namespace_name, field = m.split(':') db_loader = self.get_plugin( 'metrixpp.mpp.dbf').get_loader() namespace = db_loader.get_namespace(namespace_name) if namespace == None or namespace.check_field( field) == False: cout.notify( data.get_path(), region.get_cursor(), cout.SEVERITY_WARNING, "Suppressed metric '" + namespace_name + ":" + field + "' is not being collected", [("Metric name", namespace_name + ":" + field), ("Region name", region.get_name())]) continue if namespace.are_regions_supported() == False: if region.get_id() != 1: cout.notify( data.get_path(), region.get_cursor(), cout.SEVERITY_WARNING, "Suppressed metric '" + namespace_name + ":" + field + "' is attributed to a file, not a region. " "Remove it or move to the beginning of the file.", [("Metric name", namespace_name + ":" + field), ("Region name", region.get_name())]) continue if m in file_list_text: cout.notify( data.get_path(), region.get_cursor(), cout.SEVERITY_WARNING, "Duplicate suppression of the metric '" + namespace_name + ":" + field + "'", [("Metric name", namespace_name + ":" + field), ("Region name", None)]) continue file_count += 1 file_list_text.append(m) continue if m in list_text: cout.notify( data.get_path(), region.get_cursor(), cout.SEVERITY_WARNING, "Duplicate suppression of the metric '" + namespace_name + ":" + field + "'", [("Metric name", namespace_name + ":" + field), ("Region name", region.get_name())]) continue count += 1 list_text.append(m) continue if count > 0: region.set_data(self.get_name(), 'count', count) region.set_data(self.get_name(), 'list', '[' + ']['.join(list_text) + ']') if file_count > 0: data.set_data(self.get_name() + '.file', 'count', file_count) data.set_data(self.get_name() + '.file', 'list', '[' + ']['.join(file_list_text) + ']')