def ReadSymsHeader(sym_file):
"""Parse the header of the symbol file
The first line of the syms file will read like:
MODULE Linux arm F4F6FA6CCBDEF455039C8DE869C8A2F40 blkid
https://code.google.com/p/google-breakpad/wiki/SymbolFiles
Args:
sym_file: The symbol file to parse
Returns:
A SymbolHeader object
Raises:
ValueError if the first line of |sym_file| is invalid
"""
with cros_build_lib.Open(sym_file) as f:
header = f.readline().split()
if header[0] != 'MODULE' or len(header) != 5:
raise ValueError('header of sym file is invalid')
return SymbolHeader(os=header[1],
cpu=header[2],
id=header[3],
name=header[4])
def json(obj, fp: Optional[Union[str, os.PathLike, TextIO]] = None,
compact: bool = False) -> Optional[str]:
"""Convert an object to JSON with the right format.
Args:
obj: The object to serialize & format.
fp: By default, the JSON string is returned. The |fp| allows specifying a
file object (in text mode) to write to instead.
compact: Whether the output will be compact (flattened to one line), or
human readable (spread over multiple lines).
Returns:
A string if |fp| is not specified, else None.
"""
kwargs = {
# JSON style guide says Uunicode characters are fully allowed.
'ensure_ascii': False,
# We use 2 space indent to match JSON style guide.
'indent': None if compact else 2,
'separators': (',', ':') if compact else (',', ': '),
'sort_keys': True,
}
if fp:
with cros_build_lib.Open(fp, mode='w') as real_fp:
mod_json.dump(obj, real_fp, **kwargs)
if not compact:
real_fp.write('\n')
else:
ret = mod_json.dumps(obj, **kwargs)
if not compact:
ret += '\n'
return ret
def SortTests(tests, jobs=1, timing_cache_file=None):
"""Interleave the slowest & fastest
Hopefully we can pipeline the overall process better by queueing the slowest
tests first while also using half the slots for fast tests. We don't need
the timing info to be exact, just ballpark.
Args:
tests: The list of tests to sort.
jobs: How many jobs will we run in parallel.
timing_cache_file: Where to read test timing info.
Returns:
The tests ordered for best execution timing (we hope).
"""
if timing_cache_file is None:
timing_cache_file = TIMING_CACHE_FILE
# Usually |tests| will be a generator -- break it down.
tests = list(tests)
# If we have enough spare cpus to crunch the jobs, just do so.
if len(tests) <= jobs:
return tests
# Create a dict mapping tests to their timing information using the cache.
try:
with cros_build_lib.Open(timing_cache_file) as f:
cache = json.load(f)
except (IOError, ValueError):
cache = {}
# Sort the cached list of tests from slowest to fastest.
sorted_tests = [test for (test, _timing) in
sorted(cache.items(), key=lambda x: x[1], reverse=True)]
# Then extract the tests from the cache list that we care about -- remember
# that the cache could be stale and contain tests that no longer exist, or
# the user only wants to run a subset of tests.
ret = []
for test in sorted_tests:
if test in tests:
ret.append(test)
tests.remove(test)
# Any tests not in the cache we just throw on the end. No real way to
# predict their speed ahead of time, and we'll get useful data when they
# run the test a second time.
ret += tests
# Now interleave the fast & slow tests so every other one mixes.
# On systems with fewer cores, this can help out in two ways:
# (1) Better utilization of resources when some slow tests are I/O or time
# bound, so the other cores can spawn/fork fast tests faster (generally).
# (2) If there is common code that is broken, we get quicker feedback if we
# churn through the fast tests.
# Worse case, this interleaving doesn't slow things down overall.
fast = ret[:int(round(len(ret) / 2.0)) - 1:-1]
slow = ret[:-len(fast)]
ret[::2] = slow
ret[1::2] = fast
return ret
def WriteQueueToFile(listing, queue, relpath=None):
"""Write all the items in |queue| to the |listing|.
Note: The queue must have a sentinel None appended to the end.
Args:
listing: Where to write out the list of files.
queue: The queue of paths to drain.
relpath: If set, write out paths relative to this one.
"""
if not listing:
# Still drain the queue so we make sure the producer has finished
# before we return. Otherwise, the queue might get destroyed too
# quickly which will trigger a traceback in the producer.
while queue.get() is not None:
continue
return
with cros_build_lib.Open(listing, 'wb+') as f:
while True:
path = queue.get()
if path is None:
return
if relpath:
path = os.path.relpath(path, relpath)
f.write('%s\n' % path)
def _IsFilePathGPTDiskImage(file_path):
"""Determines if a file is a valid GPT disk.
Args:
file_path: Path to the file to test.
"""
if os.path.isfile(file_path):
with cros_build_lib.Open(file_path) as image_file:
image_file.seek(0x1fe)
if image_file.read(10) == '\x55\xaaEFI PART':
return True
return False
def LoadFromCSV(csv_file, name=None):
"""Create a new Table object by loading contents of |csv_file|."""
table = None
with cros_build_lib.Open(csv_file) as f:
for line in f:
if line[-1] == '\n':
line = line[0:-1]
vals = Table._SplitCSVLine(line)
if not table:
# Read headers
table = Table(vals, name=name)
else:
# Read data row
table.AppendRow(vals)
return table
def write_sizes(sizes: dict, required_paths: list, human_readable: bool,
output_format: str,
output_path: typing.Union(str, typing.TextIO)):
"""Writes the sizes in CSV format.
Args:
sizes: A dictionary of path -> size.
required_paths: list of paths to order results by
human_readable: set to True when user wants output in human readable format
output_format: output format (json or csv)
output_path: path to write output to
"""
def size_string(sz):
if human_readable:
return pformat.size(sz)
return sz
output = []
# If required_paths passed in, emit output in same order as passed in.
if required_paths:
for path in required_paths:
if path not in sizes:
size = -1
else:
size = size_string(sizes[path])
output.append({'path': path, 'size': size})
else:
for path, size in sorted(sizes.items()):
output.append({'path': path, 'size': size_string(sizes[path])})
with cros_build_lib.Open(output_path, mode='w') as f:
if output_format == 'csv':
writer = csv.DictWriter(f, ['path', 'size'])
writer.writeheader()
for row in output:
writer.writerow(row)
elif output_format == 'json':
pformat.json(output, f)
