def testArchive(self):
"""Test successful archive of goma logs without stats/counterz specified."""
self._CreateLogFile('compiler_proxy',
datetime.datetime(2017, 4, 26, 12, 0, 0))
self._CreateLogFile('compiler_proxy-subproc',
datetime.datetime(2017, 4, 26, 12, 0, 0))
self._CreateLogFile('gomacc', datetime.datetime(2017, 4, 26, 12, 1, 0))
self._CreateLogFile('gomacc', datetime.datetime(2017, 4, 26, 12, 2, 0))
archiver = goma_lib.LogsArchiver(self.goma_log_dir,
dest_dir=self.dest_dir)
archiver_tuple = archiver.Archive()
# Verify that no stats/counterz was returned in the tuple.
self.assertFalse(archiver_tuple.counterz_file)
self.assertFalse(archiver_tuple.stats_file)
archived_files = os.listdir(self.dest_dir)
# Verify that the list of files returned matched what we find in the
# dest_dir.
self.assertCountEqual(archiver_tuple.log_files, archived_files)
self.assertCountEqual(archived_files, [
'compiler_proxy.host.log.INFO.20170426-120000.000000.gz',
'gomacc.host.log.INFO.20170426-120100.000000.tar.gz',
'compiler_proxy-subproc.host.log.INFO.20170426-120000.000000.gz'
])
def testArchiveWithStatsAndCounterz(self):
"""Test successful archive of goma logs with stats and counterz."""
self._CreateLogFile('compiler_proxy',
datetime.datetime(2017, 4, 26, 12, 0, 0))
self._CreateLogFile('compiler_proxy-subproc',
datetime.datetime(2017, 4, 26, 12, 0, 0))
self._CreateLogFile('gomacc', datetime.datetime(2017, 4, 26, 12, 1, 0))
self._CreateLogFile('gomacc', datetime.datetime(2017, 4, 26, 12, 2, 0))
self._CreateFile('stats.binaryproto')
self._CreateFile('counterz.binaryproto')
archiver = goma_lib.LogsArchiver(self.goma_log_dir,
dest_dir=self.dest_dir,
stats_file='stats.binaryproto',
counterz_file='counterz.binaryproto')
archiver_tuple = archiver.Archive()
self.assertCountEqual(archiver_tuple.log_files, [
'compiler_proxy.host.log.INFO.20170426-120000.000000.gz',
'gomacc.host.log.INFO.20170426-120100.000000.tar.gz',
'compiler_proxy-subproc.host.log.INFO.20170426-120000.000000.gz'
])
self.assertEqual(archiver_tuple.stats_file, 'stats.binaryproto')
self.assertEqual(archiver_tuple.counterz_file, 'counterz.binaryproto')
def testNinjaLogArchive(self):
"""Test successful archive of ninja logs."""
self._CreateLogFile('compiler_proxy',
datetime.datetime(2017, 8, 21, 12, 0, 0))
self._CreateLogFile('compiler_proxy-subproc',
datetime.datetime(2017, 8, 21, 12, 0, 0))
ninja_log_path = os.path.join(self.goma_log_dir, 'ninja_log')
osutils.WriteFile(ninja_log_path, 'Ninja Log Content\n')
timestamp = datetime.datetime(2017, 8, 21, 12, 0, 0)
mtime = time.mktime(timestamp.timetuple())
os.utime(ninja_log_path, ((time.time(), mtime)))
osutils.WriteFile(os.path.join(self.goma_log_dir, 'ninja_command'),
'ninja_command')
osutils.WriteFile(os.path.join(self.goma_log_dir, 'ninja_cwd'),
'ninja_cwd')
osutils.WriteFile(os.path.join(self.goma_log_dir, 'ninja_env'),
'key1=value1\0key2=value2\0')
osutils.WriteFile(os.path.join(self.goma_log_dir, 'ninja_exit'), '0')
archiver = goma_lib.LogsArchiver(self.goma_log_dir,
dest_dir=self.dest_dir)
archived_tuple = archiver.Archive()
username = getpass.getuser()
pid = os.getpid()
hostname = cros_build_lib.GetHostName()
ninjalog_filename = 'ninja_log.%s.%s.20170821-120000.%d.gz' % (
username, hostname, pid)
# Verify the archived files in the dest_dir
archived_files = os.listdir(self.dest_dir)
self.assertCountEqual(archived_files, [
ninjalog_filename,
'compiler_proxy-subproc.host.log.INFO.20170821-120000.000000.gz',
'compiler_proxy.host.log.INFO.20170821-120000.000000.gz'
])
# Verify the archived_tuple result.
self.assertFalse(archived_tuple.counterz_file)
self.assertFalse(archived_tuple.stats_file)
self.assertCountEqual(archived_tuple.log_files, [
'compiler_proxy.host.log.INFO.20170821-120000.000000.gz',
'compiler_proxy-subproc.host.log.INFO.20170821-120000.000000.gz',
ninjalog_filename
])
# Verify content of ninja_log file.
ninjalog_path = os.path.join(self.dest_dir, ninjalog_filename)
with gzip.open(ninjalog_path, 'rt') as gzip_file:
ninja_log_content = gzip_file.read()
content, eof, metadata_json = ninja_log_content.split('\n', 3)
self.assertEqual('Ninja Log Content', content)
self.assertEqual('# end of ninja log', eof)
metadata = json.loads(metadata_json)
self.assertEqual(
metadata, {
'platform':
'chromeos',
'cmdline': ['ninja_command'],
'cwd':
'ninja_cwd',
'exit':
0,
'env': {
'key1': 'value1',
'key2': 'value2'
},
'compiler_proxy_info':
'compiler_proxy.host.log.INFO.20170821-120000.000000'
})
def InstallPackages(input_proto, output_proto, _config):
"""Install packages into a sysroot, building as necessary and permitted."""
compile_source = (input_proto.flags.compile_source
or input_proto.flags.toolchain_changed)
# Testing if Goma will support unknown compilers now.
use_goma = input_proto.flags.use_goma
target_sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path)
build_target = controller_util.ParseBuildTarget(
input_proto.sysroot.build_target)
# Get the package atom for each specified package. The field is optional, so
# error only when we cannot parse an atom for each of the given packages.
packages = [
controller_util.PackageInfoToCPV(x).cp for x in input_proto.packages
]
if input_proto.packages and not all(packages):
cros_build_lib.Die(
'Invalid package(s) specified. Unable to parse atom from all packages.'
)
package_indexes = [
binpkg.PackageIndexInfo.from_protobuf(x)
for x in input_proto.package_indexes
]
if not target_sysroot.IsToolchainInstalled():
cros_build_lib.Die('Toolchain must first be installed.')
_LogBinhost(build_target.name)
use_flags = [u.flag for u in input_proto.use_flags]
build_packages_config = sysroot.BuildPackagesRunConfig(
usepkg=not compile_source,
install_debug_symbols=True,
packages=packages,
package_indexes=package_indexes,
use_flags=use_flags,
use_goma=use_goma,
incremental_build=False)
try:
sysroot.BuildPackages(build_target, target_sysroot,
build_packages_config)
except sysroot_lib.PackageInstallError as e:
if not e.failed_packages:
# No packages to report, so just exit with an error code.
return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY
# We need to report the failed packages.
for package in e.failed_packages:
package_info = output_proto.failed_packages.add()
controller_util.CPVToPackageInfo(package, package_info)
return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE
# Copy goma logs to specified directory if there is a goma_config and
# it contains a log_dir to store artifacts.
if input_proto.goma_config.log_dir.dir:
# Get the goma log directory based on the GLOG_log_dir env variable.
# TODO(crbug.com/1045001): Replace environment variable with query to
# goma object after goma refactoring allows this.
log_source_dir = os.getenv('GLOG_log_dir')
if not log_source_dir:
cros_build_lib.Die('GLOG_log_dir must be defined.')
archiver = goma_lib.LogsArchiver(
log_source_dir,
dest_dir=input_proto.goma_config.log_dir.dir,
stats_file=input_proto.goma_config.stats_file,
counterz_file=input_proto.goma_config.counterz_file)
archiver_tuple = archiver.Archive()
if archiver_tuple.stats_file:
output_proto.goma_artifacts.stats_file = archiver_tuple.stats_file
if archiver_tuple.counterz_file:
output_proto.goma_artifacts.counterz_file = archiver_tuple.counterz_file
output_proto.goma_artifacts.log_files[:] = archiver_tuple.log_files
# Read metric events log and pipe them into output_proto.events.
deserialize_metrics_log(output_proto.events, prefix=build_target.name)