def main(argv):
parser = GetParser()
options = parser.parse_args(argv)
logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
# Check that we have no uncommitted files, and that our checkout's HEAD is
# contained in a remote branch. This is to ensure that we don't accidentally
# run uncommitted migrations.
uncommitted_files = git.RunGit(os.getcwd(), ['status', '-s']).output
if uncommitted_files:
cros_build_lib.Die('You appear to have uncommitted files. Aborting!')
remote_branches = git.RunGit(os.getcwd(),
['branch', '-r', '--contains']).output
if not remote_branches:
cros_build_lib.Die(
'You appear to be on a local branch of chromite. Aborting!')
if options.command == MIGRATE:
positive_confirmation = 'please modify my database'
warn = (
'This option will apply schema changes to your existing database. '
'You should not run this against the production database unless '
'your changes are thoroughly tested, and those tests included '
'in cidb_integration_test.py (including tests that old data is '
'sanely migrated forward). Database corruption could otherwise '
'result. Are you sure you want to proceed? If so, type "%s" '
'now.\n') % positive_confirmation
elif options.command == WIPE:
positive_confirmation = 'please delete my data'
warn = (
'This operation will wipe (i.e. DELETE!) the entire contents of '
'the database pointed at by %s. Are you sure you want to proceed? '
'If so, type "%s" now.\n') % (os.path.join(
options.cred_dir, 'host.txt'), positive_confirmation)
else:
cros_build_lib.Die('No command or unsupported command. Exiting.')
print(warn)
conf_string = cros_build_lib.GetInput('(%s)?: ' % positive_confirmation)
if conf_string != positive_confirmation:
cros_build_lib.Die('You changed your mind. Aborting.')
if options.command == MIGRATE:
print('OK, applying migrations...')
db = cidb.CIDBConnection(options.cred_dir)
db.ApplySchemaMigrations(maxVersion=options.migrate_version)
elif options.command == WIPE:
print('OK, wiping database...')
db = cidb.CIDBConnection(options.cred_dir)
db.DropDatabase()
print('Done.')
def LocalCIDBConnection(self, cidb_user):
"""Create a CIDBConnection with the local mysqld instance.
Args:
cidb_user: The mysql user to connect as.
Returns:
The created CIDBConnection object.
"""
creds_dir_path = os.path.join(self.tempdir, 'local_cidb_creds')
osutils.RmDir(creds_dir_path, ignore_missing=True)
osutils.SafeMakedirs(creds_dir_path)
osutils.WriteFile(os.path.join(creds_dir_path, 'host.txt'),
self.mysqld_host)
osutils.WriteFile(os.path.join(creds_dir_path, 'port.txt'),
str(self.mysqld_port))
osutils.WriteFile(os.path.join(creds_dir_path, 'user.txt'), cidb_user)
if cidb_user in self.CIDB_CREDS_DIR:
shutil.copy(os.path.join(self.CIDB_CREDS_DIR[cidb_user], 'password.txt'),
creds_dir_path)
return cidb.CIDBConnection(
creds_dir_path,
query_retry_args=cidb.SqlConnectionRetryArgs(4, 1, 1.1))
def Run(self):
"""Run cros uprevchrome.
Raises:
Exception if the PFQ build_id is not valid.
Exception if UprevChrome raises exceptions.
"""
self.options.Freeze()
# Delay import so sqlalchemy isn't pulled in until we need it.
from chromite.lib import cidb
cidb_creds = self.options.cred_dir
if cidb_creds is None:
try:
cidb_creds = cros_cidbcreds.CheckAndGetCIDBCreds()
except:
logging.error('Failed to download CIDB creds from gs.\n'
'Can try obtaining your credentials at '
'go/cros-cidb-admin and manually passing it in '
'with --cred-dir.')
raise
db = cidb.CIDBConnection(cidb_creds)
build_number = self.ValidatePFQBuild(self.options.pfq_build, db)
with osutils.TempDir(prefix='uprevchrome_',
delete=self.options.wipe) as work_dir:
self.UprevChrome(work_dir, self.options.pfq_build, build_number)
logging.info('Used working directory: %s', work_dir)
def main(argv):
# Parse command line arguments.
parser = GetParser()
options = parser.parse_args(argv)
# Set up clients.
credentials = options.cred_dir or cros_cidbcreds.CheckAndGetCIDBCreds()
db = cidb.CIDBConnection(credentials)
topology.FetchTopologyFromCIDB(db)
milo_client = milo.MiloClient(options.service_acct_json,
host=options.milo_host)
builds = []
# Add explicitly requested builds.
if options.build_ids:
for build_id in options.build_ids:
builds.append(
MakeBuildEntry(db,
milo_client,
build_id,
no_suites=options.no_suites))
# Search for builds by build config.
if options.build_config:
masters = db.GetBuildHistory(options.build_config,
options.num_builds,
final=True)
for master in masters:
builds.append(
MakeBuildEntry(db,
milo_client,
master['id'],
master,
no_suites=options.no_suites))
statuses = db.GetSlaveStatuses(master['id'])
for slave in statuses:
builds.append(
MakeBuildEntry(db,
milo_client,
slave['id'],
slave,
no_suites=options.no_suites))
if not options.allow_empty and not options.no_suites:
builds = [b for b in builds if len(b.get('suite_ids', []))]
# Output results.
with open(options.output, 'w') if options.output else sys.stdout as f:
if options.json:
output = {
'builds': builds,
}
json.dump(output, f)
else:
for b in builds:
f.write(StringifyBuildEntry(b))
f.write('\n')
def main(argv):
parser = GetParser()
options = parser.parse_args(argv)
if not _CheckOptions(options):
sys.exit(1)
credentials = options.cred_dir
if not credentials:
credentials = cros_cidbcreds.CheckAndGetCIDBCreds()
db = cidb.CIDBConnection(credentials)
if options.end_date:
end_date = options.end_date
else:
end_date = datetime.datetime.now().date()
# Determine the start date to use, which is required.
if options.start_date:
start_date = options.start_date
else:
assert options.past_month or options.past_week or options.past_day
# Database search results will include both the starting and ending
# days. So, the number of days to subtract is one less than the
# length of the search period.
#
# For instance, the starting day for the week ending 2014-04-21
# should be 2017-04-15 (date - 6 days).
if options.past_month:
start_date = end_date - datetime.timedelta(days=29)
elif options.past_week:
start_date = end_date - datetime.timedelta(days=6)
else:
start_date = end_date
if options.build_type == 'cq':
master_config = constants.CQ_MASTER
else:
master_config = constants.PFQ_MASTER
cl_stats_engine = CLStatsEngine(db)
cl_stats_engine.Gather(start_date, end_date, master_config)
summary = cl_stats_engine.Summarize(options.build_type,
options.bad_patch_candidates)
if options.report_file:
with open(options.report_file, "w") as f:
logging.info("Writing report to %s", options.report_file)
GenerateReport(f, summary)
def Run(self):
"""Run cros buildresult."""
self.options.Freeze()
commandline.RunInsideChroot(self)
credentials = self.options.cred_dir
if not credentials:
credentials = cros_cidbcreds.CheckAndGetCIDBCreds(
force_update=self.options.force_update)
# Delay import so sqlalchemy isn't pulled in until we need it.
from chromite.lib import cidb
db = cidb.CIDBConnection(credentials)
build_statuses = FetchBuildStatuses(db, self.options)
if build_statuses:
# Filter out builds that don't exist in CIDB, or which aren't finished.
build_statuses = [
b for b in build_statuses if IsBuildStatusFinished(b)
]
# If we found no builds at all, return a different exit code to help
# automated scripts know they should try waiting longer.
if not build_statuses:
logging.error('No build found. Perhaps not started?')
return 2
# Fixup all of the builds we have.
build_statuses = [FixUpBuildStatus(db, b) for b in build_statuses]
# Produce our final result.
if self.options.report == 'json':
report = ReportJson(build_statuses)
else:
report = Report(build_statuses)
print(report)
def main(argv):
parser = GetParser()
options = parser.parse_args(argv)
# Determine which hosts to connect to.
db = cidb.CIDBConnection(options.cred_dir)
topology.FetchTopologyFromCIDB(db)
if options.json_file:
# Use the specified alerts.
logging.info('Using JSON file %s', options.json_file)
with open(options.json_file) as f:
summary_json = f.read()
print(summary_json)
else:
builds = [tuple(x.split(',')) for x in options.builds]
if not builds:
builds = constants.SOM_BUILDS[options.som_tree]
# Generate the set of alerts to send.
logdog_client = logdog.LogdogClient(options.service_acct_json,
host=options.logdog_host)
milo_client = milo.MiloClient(options.service_acct_json,
host=options.milo_host)
summary_json = GenerateAlertsSummary(
db,
builds=builds,
logdog_client=logdog_client,
milo_client=milo_client,
allow_experimental=options.allow_experimental)
if options.output_json:
with open(options.output_json, 'w') as f:
logging.info('Writing JSON file %s', options.output_json)
f.write(summary_json)
# Authenticate and send the alerts.
som_client = som.SheriffOMaticClient(options.service_acct_json,
insecure=options.som_insecure,
host=options.som_host)
som_client.SendAlerts(summary_json, tree=options.som_tree)
def Run(self):
"""Run cros uprevchrome.
Raises:
Exception if the PFQ build_id is not valid.
Exception if UprevChrome raises exceptions.
"""
self.options.Freeze()
# Delay import so sqlalchemy isn't pulled in until we need it.
from chromite.lib import cidb
cidb_creds = self.options.cred_dir
if cidb_creds is None:
try:
cidb_creds = cros_cidbcreds.CheckAndGetCIDBCreds()
except:
logging.error('Failed to download CIDB creds from gs.\n'
'Can try obtaining your credentials at '
'go/cros-cidb-admin and manually passing it in '
'with --cred-dir.')
raise
db = cidb.CIDBConnection(cidb_creds)
build_number = self.ValidatePFQBuild(self.options.pfq_build, db)
chroot_tmp = os.path.join(constants.SOURCE_ROOT,
constants.DEFAULT_CHROOT_DIR, 'tmp')
tmp_override = None if cros_build_lib.IsInsideChroot() else chroot_tmp
work_dir = tempfile.mkdtemp(prefix='uprevchrome_', dir=tmp_override)
try:
self.UprevChrome(work_dir, self.options.pfq_build, build_number)
finally:
if self.options.wipe:
osutils.RmDir(work_dir)
logging.info('Removed work_dir %s', work_dir)
else:
logging.info('Leaving working directory at %s', work_dir)
def main(argv):
parser = GetParser()
options = parser.parse_args(argv)
if not _CheckOptions(options):
sys.exit(1)
db = cidb.CIDBConnection(options.cred_dir)
if options.end_date:
end_date = options.end_date
else:
end_date = datetime.datetime.now().date()
# Determine the start date to use, which is required.
if options.start_date:
start_date = options.start_date
else:
assert options.past_month or options.past_week or options.past_day
if options.past_month:
start_date = end_date - datetime.timedelta(days=30)
elif options.past_week:
start_date = end_date - datetime.timedelta(days=7)
else:
start_date = end_date - datetime.timedelta(days=1)
if options.build_type == 'cq':
master_config = constants.CQ_MASTER
else:
master_config = constants.PFQ_MASTER
cl_stats_engine = CLStatsEngine(db)
cl_stats_engine.Gather(start_date,
end_date,
master_config,
starting_build_number=options.starting_build,
ending_build_number=options.ending_build)
cl_stats_engine.Summarize(options.build_type, options.bad_patch_candidates)
def InitializeClients(self):
"""Check if underlying clients are initialized.
Returns:
A boolean indicating the client statuses.
"""
pid_mismatch = (self.process_id != os.getpid())
if self._IsCIDBClientMissing() or pid_mismatch:
self.process_id = os.getpid()
if self.cidb_creds:
for_service = self.for_service if self.for_service else False
self.cidb_conn = cidb.CIDBConnection(self.cidb_creds,
for_service=for_service)
elif not cidb.CIDBConnectionFactory.IsCIDBSetup():
self.cidb_conn = None
else:
self.cidb_conn = (
cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder())
if self._IsBuildbucketClientMissing() or pid_mismatch:
self.bb_client = buildbucket_v2.BuildbucketV2()
return not (self._IsCIDBClientMissing()
or self._IsBuildbucketClientMissing())
def testValidatePFQBuild(self):
"""Test ValidatePFQBuild."""
self.SetupCommandMock(self.cmd_args)
db = cidb.CIDBConnection('cred_dir')
self.assertEqual('100', self.cmd_mock.inst.ValidatePFQBuild(100, db))
def Run(self):
"""Run cros build."""
self.options.Freeze()
commandline.RunInsideChroot(self)
credentials = self.options.cred_dir
if not credentials:
credentials = cros_cidbcreds.CheckAndGetCIDBCreds()
# Delay import so sqlalchemy isn't pulled in until we need it.
from chromite.lib import cidb
db = cidb.CIDBConnection(credentials)
# Timeframe for discovering builds, if options.build_id not used.
start_date, end_date = self.OptionsToStartEndDates(self.options)
# Trending is sufficiently different to be handled on it's own.
if not self.options.trending and self.options.report != 'success':
assert not self.options.csv, (
'--csv can only be used with --trending or --report success.')
# Data about a single build (optional).
focus_build = None
if self.options.build_id:
logging.info('Gathering data for %s', self.options.build_id)
focus_status = build_time_stats.BuildIdToBuildStatus(
db, self.options.build_id)
focus_build = build_time_stats.GetBuildTimings(focus_status)
build_config = focus_status['build_config']
builds_statuses = build_time_stats.BuildConfigToStatuses(
db, build_config, start_date, end_date)
description = 'Focus %d - %s' % (self.options.build_id,
build_config)
elif self.options.build_config:
builds_statuses = build_time_stats.BuildConfigToStatuses(
db, self.options.build_config, start_date, end_date)
description = 'Config %s' % self.options.build_config
elif self.options.build_type:
builds_statuses = build_time_stats.MasterConfigToStatuses(
db, BUILD_TYPE_MAP[self.options.build_type], start_date,
end_date)
description = 'Type %s' % self.options.build_type
if not builds_statuses:
logging.critical('No Builds Found For: %s', description)
return 1
if self.options.report == 'success':
# Calculate per-build success rates and per-stage success rates.
build_success_rates = build_time_stats.GetBuildSuccessRates(
builds_statuses)
stage_success_rates = (
build_time_stats.GetStageSuccessRates(builds_statuses)
if self.options.stages else {})
if self.options.csv:
build_time_stats.SuccessReportCsv(sys.stdout,
build_success_rates,
stage_success_rates)
else:
build_time_stats.SuccessReport(sys.stdout, description,
build_success_rates,
stage_success_rates)
return 0
# Compute per-build timing.
builds_timings = [
build_time_stats.GetBuildTimings(status)
for status in builds_statuses
]
if not builds_timings:
logging.critical('No timing results For: %s', description)
return 1
# Report results.
if self.options.report == 'standard':
build_time_stats.Report(sys.stdout, description, focus_build,
builds_timings, self.options.stages,
self.options.trending, self.options.csv)
elif self.options.report == 'stability':
build_time_stats.StabilityReport(sys.stdout, description,
builds_timings)
