for dir_name, _, file_list in os.walk(PATH_TO_METADATA_DIR): for filename in file_list: if (filename == 'histograms.xml' or filename == 'histogram_suffixes_list.xml'): # Compute the relative path of the histograms xml file. file_path = os.path.relpath(os.path.join(dir_name, filename), PATH_TO_METADATA_DIR) files.append( os.path.join('tools/metrics/histograms/metadata', file_path).replace(os.sep, '/').lower()) return sorted(files) ENUMS_XML_RELATIVE = 'tools/metrics/histograms/enums.xml' # The absolute path to the metadata folder. PATH_TO_METADATA_DIR = path_util.GetInputFile( 'tools/metrics/histograms/metadata') # In the middle state, histogram paths include both the large histograms.xml # file as well as the split up files. # TODO: Improve on the current design to avoid calling `os.walk()` at the time # of module import. HISTOGRAMS_XMLS_RELATIVE = (['tools/metrics/histograms/histograms.xml'] + _FindHistogramsXmlFiles()) OBSOLETE_XML_RELATIVE = ('tools/metrics/histograms/metadata/' 'obsolete_histograms.xml') ALL_XMLS_RELATIVE = [ENUMS_XML_RELATIVE, OBSOLETE_XML_RELATIVE ] + HISTOGRAMS_XMLS_RELATIVE HISTOGRAMS_PREFIX_LIST = [ os.path.basename(os.path.dirname(f)) for f in HISTOGRAMS_XMLS_RELATIVE ]
ENUMS_XML_RELATIVE = 'tools/metrics/histograms/enums.xml' # This file path accounts for cases where the script is executed from other # metrics-related directories. PATH_TO_HISTOGRAMS_XML_DIR = os.path.join('..', 'histograms/histograms_xml') # In the middle state, histogram paths include both the large histograms.xml # file as well as the split up files. # TODO: Improve on the current design to avoid calling `os.walk()` at the time # of module import. HISTOGRAMS_XMLS_RELATIVE = ( ['tools/metrics/histograms/histograms.xml'] + _FindHistogramsXmlFiles(PATH_TO_HISTOGRAMS_XML_DIR)) OBSOLETE_XML_RELATIVE = ('tools/metrics/histograms/histograms_xml/' 'obsolete_histograms.xml') ALL_XMLS_RELATIVE = [ENUMS_XML_RELATIVE, OBSOLETE_XML_RELATIVE ] + HISTOGRAMS_XMLS_RELATIVE ENUMS_XML = path_util.GetInputFile(ENUMS_XML_RELATIVE) HISTOGRAMS_XMLS = [path_util.GetInputFile(f) for f in HISTOGRAMS_XMLS_RELATIVE] OBSOLETE_XML = path_util.GetInputFile(OBSOLETE_XML_RELATIVE) ALL_XMLS = [ENUMS_XML, OBSOLETE_XML] + HISTOGRAMS_XMLS ALL_TEST_XMLS_RELATIVE = [ 'tools/metrics/histograms/test_data/enums.xml', 'tools/metrics/histograms/test_data/histograms.xml', 'tools/metrics/histograms/test_data/ukm.xml', ] ALL_TEST_XMLS = [path_util.GetInputFile(f) for f in ALL_TEST_XMLS_RELATIVE] TEST_ENUMS_XML, TEST_HISTOGRAMS_XML, TEST_UKM_XML = ALL_TEST_XMLS
def ReadHistogramValues(filename, start_marker, end_marker, strip_k_prefix): """Creates a dictionary of enum values, read from a C++ file. Args: filename: The unix-style path (relative to src/) of the file to open. start_marker: A regex that signifies the start of the enum values. end_marker: A regex that signifies the end of the enum values. strip_k_prefix: Set to True if enum values are declared as kFoo and the 'k' should be stripped. Returns: A dictionary from enum value to enum label. Raises: DuplicatedValue: An error when two enum labels share the same value. """ # Read the file as a list of lines with io.open(path_util.GetInputFile(filename)) as f: content = f.readlines() START_REGEX = re.compile(start_marker) ITEM_REGEX = re.compile(r'^(\w+)') ITEM_REGEX_WITH_INIT = re.compile(r'(\w+)\s*=\s*(\d*)') WRAPPED_INIT = re.compile(r'(\d+)') END_REGEX = re.compile(end_marker) iterator = iter(content) # Find the start of the enum for line in iterator: if START_REGEX.match(line.strip()): break enum_value = 0 result = {} for line in iterator: line = line.strip() # Exit condition: we reached last enum value if END_REGEX.match(line): break # Inside enum: generate new xml entry m = ITEM_REGEX_WITH_INIT.match(line) if m: label = m.group(1) if m.group(2): enum_value = int(m.group(2)) else: # Enum name is so long that the value wrapped to the next line next_line = next(iterator).strip() enum_value = int(WRAPPED_INIT.match(next_line).group(1)) else: m = ITEM_REGEX.match(line) if m: label = m.group(1) else: continue if strip_k_prefix: assert label.startswith('k'), "Enum " + label + " should start with 'k'." label = label[1:] # If two enum labels have the same value if enum_value in result: raise DuplicatedValue(result[enum_value], label) result[enum_value] = label enum_value += 1 return result
# Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Migrates histogram_suffixes to patterned histograms""" import argparse import logging import os from xml.dom import minidom import extract_histograms import histogram_configuration_model import histogram_paths import path_util HISTOGRAM_SUFFIXES_LIST_PATH = path_util.GetInputFile( 'tools/metrics/histograms/histograms_xml/histogram_suffixes_list.xml') def _ExtractObsoleteNode(node, recursive=True): """Extracts obsolete child from |node|. Returns None if not exists.""" if not recursive: obsolete = [ element for element in node.getElementsByTagName('obsolete') if element.parentNode == node ] else: obsolete = node.getElementsByTagName('obsolete') if not obsolete: return None assert len(obsolete) == 1, ( 'Node %s should at most contain one obsolete node.' %
# Copyright 2017 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Paths to description XML files in this directory.""" import os import sys sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common')) import path_util ALL_XMLS_RELATIVE = [ 'tools/metrics/histograms/enums.xml', 'tools/metrics/histograms/histograms.xml' ] ALL_XMLS = [path_util.GetInputFile(f) for f in ALL_XMLS_RELATIVE] ENUMS_XML, HISTOGRAMS_XML = ALL_XMLS ALL_TEST_XMLS_RELATIVE = [ 'tools/metrics/histograms/test_data/enums.xml', 'tools/metrics/histograms/test_data/histograms.xml', 'tools/metrics/histograms/test_data/ukm.xml', ] ALL_TEST_XMLS = [path_util.GetInputFile(f) for f in ALL_TEST_XMLS_RELATIVE] TEST_ENUMS_XML, TEST_HISTOGRAMS_XML, TEST_UKM_XML = ALL_TEST_XMLS
def main(): # Find default paths. default_root = path_util.GetInputFile('/') default_extra_histograms_path = path_util.GetInputFile( 'tools/histograms/histograms.xml') # Parse command line options parser = optparse.OptionParser() parser.add_option( '--root-directory', dest='root_directory', default=default_root, help='scan within DIRECTORY for histograms [optional, defaults to "%s"]' % default_root, metavar='DIRECTORY') parser.add_option( '--extra_histograms-file', dest='extra_histograms_file_location', default=default_extra_histograms_path, help='read additional histogram definitions from FILE (relative to ' '--root-directory) [optional, defaults to "%s"]' % default_extra_histograms_path, metavar='FILE') parser.add_option('--csv', action='store_true', dest='output_as_csv', default=False, help=('output as csv for ease of parsing ' + '[optional, defaults to %default]')) parser.add_option( '--verbose', action='store_true', dest='verbose', default=False, help=('print file position information with histograms ' + '[optional, defaults to %default]')) (options, args) = parser.parse_args() if args: parser.print_help() sys.exit(1) logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) try: os.chdir(options.root_directory) except EnvironmentError as e: logging.error("Could not change to root directory: %s", e) sys.exit(1) chromium_histograms, location_map = readChromiumHistograms() xml_histograms = readAllXmlHistograms() unmapped_histograms = chromium_histograms - xml_histograms if os.path.isfile(options.extra_histograms_file_location): xml_histograms2 = readXmlHistograms( options.extra_histograms_file_location) unmapped_histograms -= xml_histograms2 else: logging.warning('No such file: %s', options.extra_histograms_file_location) if options.output_as_csv: output_csv(unmapped_histograms, location_map) else: output_log(unmapped_histograms, location_map, options.verbose)
def main(): # Find default paths. default_root = path_util.GetInputFile('/') default_histograms_path = path_util.GetInputFile( 'tools/metrics/histograms/histograms.xml') default_extra_histograms_path = path_util.GetInputFile( 'tools/histograms/histograms.xml') # Parse command line options parser = optparse.OptionParser() parser.add_option( '--root-directory', dest='root_directory', default=default_root, help='scan within DIRECTORY for histograms [optional, defaults to "%s"]' % default_root, metavar='DIRECTORY') parser.add_option( '--histograms-file', dest='histograms_file_location', default=default_histograms_path, help= 'read histogram definitions from FILE (relative to --root-directory) ' '[optional, defaults to "%s"]' % default_histograms_path, metavar='FILE') parser.add_option( '--exrta_histograms-file', dest='extra_histograms_file_location', default=default_extra_histograms_path, help='read additional histogram definitions from FILE (relative to ' '--root-directory) [optional, defaults to "%s"]' % default_extra_histograms_path, metavar='FILE') (options, args) = parser.parse_args() if args: parser.print_help() sys.exit(1) logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) try: os.chdir(options.root_directory) except EnvironmentError as e: logging.error("Could not change to root directory: %s", e) sys.exit(1) chromium_histograms = readChromiumHistograms() xml_histograms = readXmlHistograms(options.histograms_file_location) unmapped_histograms = chromium_histograms - xml_histograms if os.path.isfile(options.extra_histograms_file_location): xml_histograms2 = readXmlHistograms( options.extra_histograms_file_location) unmapped_histograms -= xml_histograms2 else: logging.warning('No such file: %s', options.extra_histograms_file_location) if len(unmapped_histograms): logging.info('') logging.info('') logging.info('Histograms in Chromium but not in XML files:') logging.info('-------------------------------------------------') for histogram in sorted(unmapped_histograms): logging.info(' %s - %s', histogram, hashHistogramName(histogram)) else: logging.info('Success! No unmapped histograms found.')
#!/usr/bin/env python # Copyright 2019 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Verifies that the structured.xml file is well-formatted.""" import os import re import sys from xml.dom import minidom sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common')) import path_util STRUCTURED_XML = path_util.GetInputFile(('tools/metrics/structured/' 'structured.xml')) def checkElementOwners(config, element_tag): """Check that every element in the config has at least one owner.""" errors = [] for node in config.getElementsByTagName(element_tag): name = node.getAttribute('name') owner_nodes = node.getElementsByTagName('owner') # Check <owner> tag is present for each element. if not owner_nodes: errors.append("<owner> tag is required for %s '%s'." % (element_tag, name)) continue
"""Splits a large histograms.xml and writes out the split xmls. Args: output_base_dir: The output base directory. """ if not os.path.exists(output_base_dir): os.mkdir(output_base_dir) histogram_nodes, variants_nodes, histogram_suffixes_nodes = _ParseMergedXML( ) # Create separate XML file for histogram suffixes. _CreateXMLFile('histogram suffixes', 'histogram_suffixes_list', histogram_suffixes_nodes, output_base_dir, 'histogram_suffixes_list.xml') obsolete_nodes, non_obsolete_nodes = _SeparateObsoleteHistogram( histogram_nodes) # Create separate XML file for obsolete histograms. _CreateXMLFile('obsolete histograms', 'histograms', obsolete_nodes, output_base_dir, 'obsolete_histograms.xml') document_dict = _BuildDocumentDict(non_obsolete_nodes + variants_nodes, 0) _WriteDocumentDict(document_dict, output_base_dir) if __name__ == '__main__': SplitIntoMultipleHistogramXMLs( path_util.GetInputFile('tools/metrics/histograms/histograms_xml'))
# Copyright 2017 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Paths to description XML files in this directory.""" import os import sys sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common')) import path_util HISTOGRAMS_XML = path_util.GetInputFile( 'tools/metrics/histograms/histograms.xml') ENUMS_XML = path_util.GetInputFile('tools/metrics/histograms/enums.xml') ALL_XMLS = [HISTOGRAMS_XML, ENUMS_XML]