for dir_name, _, file_list in os.walk(PATH_TO_METADATA_DIR):
for filename in file_list:
if (filename == 'histograms.xml'
or filename == 'histogram_suffixes_list.xml'):
# Compute the relative path of the histograms xml file.
file_path = os.path.relpath(os.path.join(dir_name, filename),
PATH_TO_METADATA_DIR)
files.append(
os.path.join('tools/metrics/histograms/metadata',
file_path).replace(os.sep, '/').lower())
return sorted(files)
ENUMS_XML_RELATIVE = 'tools/metrics/histograms/enums.xml'
# The absolute path to the metadata folder.
PATH_TO_METADATA_DIR = path_util.GetInputFile(
'tools/metrics/histograms/metadata')
# In the middle state, histogram paths include both the large histograms.xml
# file as well as the split up files.
# TODO: Improve on the current design to avoid calling `os.walk()` at the time
# of module import.
HISTOGRAMS_XMLS_RELATIVE = (['tools/metrics/histograms/histograms.xml'] +
_FindHistogramsXmlFiles())
OBSOLETE_XML_RELATIVE = ('tools/metrics/histograms/metadata/'
'obsolete_histograms.xml')
ALL_XMLS_RELATIVE = [ENUMS_XML_RELATIVE, OBSOLETE_XML_RELATIVE
] + HISTOGRAMS_XMLS_RELATIVE
HISTOGRAMS_PREFIX_LIST = [
os.path.basename(os.path.dirname(f)) for f in HISTOGRAMS_XMLS_RELATIVE
]
ENUMS_XML_RELATIVE = 'tools/metrics/histograms/enums.xml'
# This file path accounts for cases where the script is executed from other
# metrics-related directories.
PATH_TO_HISTOGRAMS_XML_DIR = os.path.join('..', 'histograms/histograms_xml')
# In the middle state, histogram paths include both the large histograms.xml
# file as well as the split up files.
# TODO: Improve on the current design to avoid calling `os.walk()` at the time
# of module import.
HISTOGRAMS_XMLS_RELATIVE = (
['tools/metrics/histograms/histograms.xml'] +
_FindHistogramsXmlFiles(PATH_TO_HISTOGRAMS_XML_DIR))
OBSOLETE_XML_RELATIVE = ('tools/metrics/histograms/histograms_xml/'
'obsolete_histograms.xml')
ALL_XMLS_RELATIVE = [ENUMS_XML_RELATIVE, OBSOLETE_XML_RELATIVE
] + HISTOGRAMS_XMLS_RELATIVE
ENUMS_XML = path_util.GetInputFile(ENUMS_XML_RELATIVE)
HISTOGRAMS_XMLS = [path_util.GetInputFile(f) for f in HISTOGRAMS_XMLS_RELATIVE]
OBSOLETE_XML = path_util.GetInputFile(OBSOLETE_XML_RELATIVE)
ALL_XMLS = [ENUMS_XML, OBSOLETE_XML] + HISTOGRAMS_XMLS
ALL_TEST_XMLS_RELATIVE = [
'tools/metrics/histograms/test_data/enums.xml',
'tools/metrics/histograms/test_data/histograms.xml',
'tools/metrics/histograms/test_data/ukm.xml',
]
ALL_TEST_XMLS = [path_util.GetInputFile(f) for f in ALL_TEST_XMLS_RELATIVE]
TEST_ENUMS_XML, TEST_HISTOGRAMS_XML, TEST_UKM_XML = ALL_TEST_XMLS
def ReadHistogramValues(filename, start_marker, end_marker, strip_k_prefix):
"""Creates a dictionary of enum values, read from a C++ file.
Args:
filename: The unix-style path (relative to src/) of the file to open.
start_marker: A regex that signifies the start of the enum values.
end_marker: A regex that signifies the end of the enum values.
strip_k_prefix: Set to True if enum values are declared as kFoo and the
'k' should be stripped.
Returns:
A dictionary from enum value to enum label.
Raises:
DuplicatedValue: An error when two enum labels share the same value.
"""
# Read the file as a list of lines
with io.open(path_util.GetInputFile(filename)) as f:
content = f.readlines()
START_REGEX = re.compile(start_marker)
ITEM_REGEX = re.compile(r'^(\w+)')
ITEM_REGEX_WITH_INIT = re.compile(r'(\w+)\s*=\s*(\d*)')
WRAPPED_INIT = re.compile(r'(\d+)')
END_REGEX = re.compile(end_marker)
iterator = iter(content)
# Find the start of the enum
for line in iterator:
if START_REGEX.match(line.strip()):
break
enum_value = 0
result = {}
for line in iterator:
line = line.strip()
# Exit condition: we reached last enum value
if END_REGEX.match(line):
break
# Inside enum: generate new xml entry
m = ITEM_REGEX_WITH_INIT.match(line)
if m:
label = m.group(1)
if m.group(2):
enum_value = int(m.group(2))
else:
# Enum name is so long that the value wrapped to the next line
next_line = next(iterator).strip()
enum_value = int(WRAPPED_INIT.match(next_line).group(1))
else:
m = ITEM_REGEX.match(line)
if m:
label = m.group(1)
else:
continue
if strip_k_prefix:
assert label.startswith('k'), "Enum " + label + " should start with 'k'."
label = label[1:]
# If two enum labels have the same value
if enum_value in result:
raise DuplicatedValue(result[enum_value], label)
result[enum_value] = label
enum_value += 1
return result
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Migrates histogram_suffixes to patterned histograms"""
import argparse
import logging
import os
from xml.dom import minidom
import extract_histograms
import histogram_configuration_model
import histogram_paths
import path_util
HISTOGRAM_SUFFIXES_LIST_PATH = path_util.GetInputFile(
'tools/metrics/histograms/histograms_xml/histogram_suffixes_list.xml')
def _ExtractObsoleteNode(node, recursive=True):
"""Extracts obsolete child from |node|. Returns None if not exists."""
if not recursive:
obsolete = [
element for element in node.getElementsByTagName('obsolete')
if element.parentNode == node
]
else:
obsolete = node.getElementsByTagName('obsolete')
if not obsolete:
return None
assert len(obsolete) == 1, (
'Node %s should at most contain one obsolete node.' %
# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Paths to description XML files in this directory."""
import os
import sys
sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common'))
import path_util
ALL_XMLS_RELATIVE = [
'tools/metrics/histograms/enums.xml',
'tools/metrics/histograms/histograms.xml'
]
ALL_XMLS = [path_util.GetInputFile(f) for f in ALL_XMLS_RELATIVE]
ENUMS_XML, HISTOGRAMS_XML = ALL_XMLS
ALL_TEST_XMLS_RELATIVE = [
'tools/metrics/histograms/test_data/enums.xml',
'tools/metrics/histograms/test_data/histograms.xml',
'tools/metrics/histograms/test_data/ukm.xml',
]
ALL_TEST_XMLS = [path_util.GetInputFile(f) for f in ALL_TEST_XMLS_RELATIVE]
TEST_ENUMS_XML, TEST_HISTOGRAMS_XML, TEST_UKM_XML = ALL_TEST_XMLS
def main():
# Find default paths.
default_root = path_util.GetInputFile('/')
default_extra_histograms_path = path_util.GetInputFile(
'tools/histograms/histograms.xml')
# Parse command line options
parser = optparse.OptionParser()
parser.add_option(
'--root-directory',
dest='root_directory',
default=default_root,
help='scan within DIRECTORY for histograms [optional, defaults to "%s"]'
% default_root,
metavar='DIRECTORY')
parser.add_option(
'--extra_histograms-file',
dest='extra_histograms_file_location',
default=default_extra_histograms_path,
help='read additional histogram definitions from FILE (relative to '
'--root-directory) [optional, defaults to "%s"]' %
default_extra_histograms_path,
metavar='FILE')
parser.add_option('--csv',
action='store_true',
dest='output_as_csv',
default=False,
help=('output as csv for ease of parsing ' +
'[optional, defaults to %default]'))
parser.add_option(
'--verbose',
action='store_true',
dest='verbose',
default=False,
help=('print file position information with histograms ' +
'[optional, defaults to %default]'))
(options, args) = parser.parse_args()
if args:
parser.print_help()
sys.exit(1)
logging.basicConfig(format='%(levelname)s: %(message)s',
level=logging.INFO)
try:
os.chdir(options.root_directory)
except EnvironmentError as e:
logging.error("Could not change to root directory: %s", e)
sys.exit(1)
chromium_histograms, location_map = readChromiumHistograms()
xml_histograms = readAllXmlHistograms()
unmapped_histograms = chromium_histograms - xml_histograms
if os.path.isfile(options.extra_histograms_file_location):
xml_histograms2 = readXmlHistograms(
options.extra_histograms_file_location)
unmapped_histograms -= xml_histograms2
else:
logging.warning('No such file: %s',
options.extra_histograms_file_location)
if options.output_as_csv:
output_csv(unmapped_histograms, location_map)
else:
output_log(unmapped_histograms, location_map, options.verbose)
def main():
# Find default paths.
default_root = path_util.GetInputFile('/')
default_histograms_path = path_util.GetInputFile(
'tools/metrics/histograms/histograms.xml')
default_extra_histograms_path = path_util.GetInputFile(
'tools/histograms/histograms.xml')
# Parse command line options
parser = optparse.OptionParser()
parser.add_option(
'--root-directory',
dest='root_directory',
default=default_root,
help='scan within DIRECTORY for histograms [optional, defaults to "%s"]'
% default_root,
metavar='DIRECTORY')
parser.add_option(
'--histograms-file',
dest='histograms_file_location',
default=default_histograms_path,
help=
'read histogram definitions from FILE (relative to --root-directory) '
'[optional, defaults to "%s"]' % default_histograms_path,
metavar='FILE')
parser.add_option(
'--exrta_histograms-file',
dest='extra_histograms_file_location',
default=default_extra_histograms_path,
help='read additional histogram definitions from FILE (relative to '
'--root-directory) [optional, defaults to "%s"]' %
default_extra_histograms_path,
metavar='FILE')
(options, args) = parser.parse_args()
if args:
parser.print_help()
sys.exit(1)
logging.basicConfig(format='%(levelname)s: %(message)s',
level=logging.INFO)
try:
os.chdir(options.root_directory)
except EnvironmentError as e:
logging.error("Could not change to root directory: %s", e)
sys.exit(1)
chromium_histograms = readChromiumHistograms()
xml_histograms = readXmlHistograms(options.histograms_file_location)
unmapped_histograms = chromium_histograms - xml_histograms
if os.path.isfile(options.extra_histograms_file_location):
xml_histograms2 = readXmlHistograms(
options.extra_histograms_file_location)
unmapped_histograms -= xml_histograms2
else:
logging.warning('No such file: %s',
options.extra_histograms_file_location)
if len(unmapped_histograms):
logging.info('')
logging.info('')
logging.info('Histograms in Chromium but not in XML files:')
logging.info('-------------------------------------------------')
for histogram in sorted(unmapped_histograms):
logging.info(' %s - %s', histogram, hashHistogramName(histogram))
else:
logging.info('Success! No unmapped histograms found.')
#!/usr/bin/env python
# Copyright 2019 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Verifies that the structured.xml file is well-formatted."""
import os
import re
import sys
from xml.dom import minidom
sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common'))
import path_util
STRUCTURED_XML = path_util.GetInputFile(('tools/metrics/structured/'
'structured.xml'))
def checkElementOwners(config, element_tag):
"""Check that every element in the config has at least one owner."""
errors = []
for node in config.getElementsByTagName(element_tag):
name = node.getAttribute('name')
owner_nodes = node.getElementsByTagName('owner')
# Check <owner> tag is present for each element.
if not owner_nodes:
errors.append("<owner> tag is required for %s '%s'." %
(element_tag, name))
continue
"""Splits a large histograms.xml and writes out the split xmls.
Args:
output_base_dir: The output base directory.
"""
if not os.path.exists(output_base_dir):
os.mkdir(output_base_dir)
histogram_nodes, variants_nodes, histogram_suffixes_nodes = _ParseMergedXML(
)
# Create separate XML file for histogram suffixes.
_CreateXMLFile('histogram suffixes', 'histogram_suffixes_list',
histogram_suffixes_nodes, output_base_dir,
'histogram_suffixes_list.xml')
obsolete_nodes, non_obsolete_nodes = _SeparateObsoleteHistogram(
histogram_nodes)
# Create separate XML file for obsolete histograms.
_CreateXMLFile('obsolete histograms', 'histograms', obsolete_nodes,
output_base_dir, 'obsolete_histograms.xml')
document_dict = _BuildDocumentDict(non_obsolete_nodes + variants_nodes, 0)
_WriteDocumentDict(document_dict, output_base_dir)
if __name__ == '__main__':
SplitIntoMultipleHistogramXMLs(
path_util.GetInputFile('tools/metrics/histograms/histograms_xml'))
# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Paths to description XML files in this directory."""
import os
import sys
sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common'))
import path_util
HISTOGRAMS_XML = path_util.GetInputFile(
'tools/metrics/histograms/histograms.xml')
ENUMS_XML = path_util.GetInputFile('tools/metrics/histograms/enums.xml')
ALL_XMLS = [HISTOGRAMS_XML, ENUMS_XML]
