def _InsertPreCQBuildRequests(self, bot_db, current_ts):
new_timestamp = current_ts.strftime('%Y-%m-%d %H:%M:%S')
old_ts = current_ts - datetime.timedelta(hours=2)
old_timestamp = old_ts.strftime('%Y-%m-%d %H:%M:%S')
b_id = bot_db.InsertBuild(
'pre_cq_launcher', _random(),
'pre_cq_launcher', 'bot_hostname')
build_req_1 = build_requests.BuildRequest(
id=None, build_id=b_id,
request_build_config='test_pre_cq_1',
request_build_args='test_build_args_1',
request_buildbucket_id='test_bb_id_1',
request_reason=build_requests.REASON_SANITY_PRE_CQ,
timestamp=old_timestamp)
build_req_2 = build_requests.BuildRequest(
id=None, build_id=b_id,
request_build_config='test_pre_cq_1',
request_build_args='test_build_args_2',
request_buildbucket_id='test_bb_id_2',
request_reason=build_requests.REASON_SANITY_PRE_CQ,
timestamp=new_timestamp)
build_req_3 = build_requests.BuildRequest(
id=None, build_id=b_id,
request_build_config='test_pre_cq_2',
request_build_args='test_build_args_3',
request_buildbucket_id='test_bb_id_3',
request_reason=build_requests.REASON_SANITY_PRE_CQ,
timestamp=new_timestamp)
bot_db.InsertBuildRequests([build_req_1, build_req_2, build_req_3])
return b_id
def _InsertCQBuildRequests(self, bot_db):
b_id = bot_db.InsertBuild(
'master-paladin', _random(),
'master-paladin', 'bot_hostname')
build_req_1 = build_requests.BuildRequest(
id=None, build_id=b_id,
request_build_config='test1-paladin',
request_build_args='test_build_args_1',
request_buildbucket_id='test_bb_id_1',
request_reason=build_requests.REASON_IMPORTANT_CQ_SLAVE,
timestamp=None)
build_req_2 = build_requests.BuildRequest(
id=None, build_id=b_id,
request_build_config='test1-paladin',
request_build_args='test_build_args_2',
request_buildbucket_id='test_bb_id_2',
request_reason=build_requests.REASON_IMPORTANT_CQ_SLAVE,
timestamp=None)
build_req_3 = build_requests.BuildRequest(
id=None, build_id=b_id,
request_build_config='test1-paladin',
request_build_args='test_build_args_3',
request_buildbucket_id='test_bb_id_3',
request_reason=build_requests.REASON_EXPERIMENTAL_CQ_SLAVE,
timestamp=None)
bot_db.InsertBuildRequests([build_req_1, build_req_2, build_req_3])
return b_id
def GetBuildRequestsForRequesterBuild(self,
requester_build_id,
request_reason=None):
"""Get the build_requests associated to the requester build.
Args:
requester_build_id: The build id of the requester build.
request_reason: If provided, only return the build_request of the given
request reason. Default to None.
Returns:
A list of build_request.BuildRequest instances.
"""
results = []
for value in self.buildRequestTable.values():
if (value['build_id'] == requester_build_id
and request_reason is None
or request_reason == value['request_reason']):
results.append(
build_requests.BuildRequest(
value['id'], value['build_id'],
value['request_build_config'],
value['request_build_args'],
value['request_buildbucket_id'],
value['request_reason'], value['timestamp']))
return results
def GetBuildRequestsForBuildConfigs(self,
request_build_configs,
num_results=-1,
start_time=None):
"""Get BuildRequests for a list build_configs.
Args:
request_build_configs: build configs (string) to request.
num_results: number of results to return, default to -1.
start_time: get build requests sent after start_time.
Returns:
A list of BuildRequest instances sorted by id in descending order.
"""
results = []
for value in self.buildRequestTable.values():
if start_time is not None and value['timestamp'] < start_time:
continue
if value['request_build_config'] in request_build_configs:
results.append(
build_requests.BuildRequest(
value['id'], value['build_id'],
value['request_build_config'],
value['request_build_args'],
value['request_buildbucket_id'],
value['request_reason'], value['timestamp']))
requests = sorted(results, key=lambda r: r.id, reverse=True)
if num_results != -1:
return requests[:num_results]
else:
return requests
def _RetryBuilds(self, builds):
"""Retry builds with Buildbucket.
Args:
builds: config names of the builds to retry with Buildbucket.
Returns:
A set of retried builds.
"""
assert builds is not None
new_scheduled_important_slaves = []
new_scheduled_build_reqs = []
for build in builds:
try:
buildbucket_id = self.new_buildbucket_info_dict[
build].buildbucket_id
build_retry = self.new_buildbucket_info_dict[build].retry
logging.info(
'Going to retry build %s buildbucket_id %s '
'with retry # %d', build, buildbucket_id, build_retry + 1)
if not self.dry_run:
fields = {
'build_type': self.config.build_type,
'build_name': self.config.name
}
metrics.Counter(
constants.MON_BB_RETRY_BUILD_COUNT).increment(
fields=fields)
content = self.buildbucket_client.RetryBuildRequest(
buildbucket_id, dryrun=self.dry_run)
new_buildbucket_id = buildbucket_lib.GetBuildId(content)
new_created_ts = buildbucket_lib.GetBuildCreated_ts(content)
new_scheduled_important_slaves.append(
(build, new_buildbucket_id, new_created_ts))
new_scheduled_build_reqs.append(
build_requests.BuildRequest(
None, self.master_build_id, build, None,
new_buildbucket_id,
build_requests.REASON_IMPORTANT_CQ_SLAVE, None))
logging.info(
'Retried build %s buildbucket_id %s created_ts %s', build,
new_buildbucket_id, new_created_ts)
except buildbucket_lib.BuildbucketResponseException as e:
logging.error('Failed to retry build %s buildbucket_id %s: %s',
build, buildbucket_id, e)
if new_scheduled_important_slaves:
self.metadata.ExtendKeyListWithList(
constants.METADATA_SCHEDULED_IMPORTANT_SLAVES,
new_scheduled_important_slaves)
return set([build for build, _, _ in new_scheduled_important_slaves])
def testGetBuildRequestsForRequesterBuild(self):
"""Test GetBuildRequestsForRequesterBuild."""
self._PrepareDatabase()
bot_db = self.LocalCIDBConnection(self.CIDB_USER_BOT)
b_id_1 = bot_db.InsertBuild(
'master-paladin', _random(),
'master-paladin', 'bot_hostname')
build_req_1 = build_requests.BuildRequest(
id=None, build_id=b_id_1,
request_build_config='test1-paladin',
request_build_args='test_build_args_1',
request_buildbucket_id='test_bb_id_1',
request_reason=build_requests.REASON_IMPORTANT_CQ_SLAVE,
timestamp=None)
build_req_2 = build_requests.BuildRequest(
id=None, build_id=b_id_1,
request_build_config='test2-paladin',
request_build_args='test_build_args_2',
request_buildbucket_id='test_bb_id_2',
request_reason=build_requests.REASON_IMPORTANT_CQ_SLAVE,
timestamp=None)
build_req_3 = build_requests.BuildRequest(
id=None, build_id=b_id_1,
request_build_config='test3-paladin',
request_build_args='test_build_args_3',
request_buildbucket_id='test_bb_id_3',
request_reason=build_requests.REASON_EXPERIMENTAL_CQ_SLAVE,
timestamp=None)
bot_db.InsertBuildRequests([build_req_1, build_req_2, build_req_3])
requests = bot_db.GetBuildRequestsForRequesterBuild(b_id_1)
self.assertCountEqual([r.request_build_config for r in requests],
['test1-paladin', 'test2-paladin', 'test3-paladin'])
requests = bot_db.GetBuildRequestsForRequesterBuild(
b_id_1, request_reason=build_requests.REASON_IMPORTANT_CQ_SLAVE)
self.assertCountEqual([r.request_build_config for r in requests],
['test1-paladin', 'test2-paladin'])
requests = bot_db.GetBuildRequestsForRequesterBuild(
b_id_1, request_reason=build_requests.REASON_EXPERIMENTAL_CQ_SLAVE)
self.assertCountEqual([r.request_build_config for r in requests],
['test3-paladin'])
def GetLatestBuildRequestsForReason(self,
request_reason,
status=None,
num_results=NUM_RESULTS_NO_LIMIT,
n_days_back=7):
"""Gets the latest build_requests associated with the request_reason.
Args:
request_reason: The reason to filter by
status: Whether to filter on status
num_results: Number of results to return, default to
self.NUM_RESULTS_NO_LIMIT.
n_days_back: How many days back to look for build requests.
Returns:
A list of build_request.BuildRequest instances.
"""
def _MatchesStatus(value):
return status is None or value['status'] == status
def _MatchesTimeConstraint(value):
if n_days_back is None:
return True
# MySQL doesn't support timestamps with microsecond resolution
now = datetime.datetime.now().replace(microsecond=0)
then = now - datetime.timedelta(days=n_days_back)
return then < value['timestamp']
by_build_config = {}
for value in self.buildRequestTable.values():
if (value['request_reason'] == request_reason
and _MatchesStatus(value)
and _MatchesTimeConstraint(value)):
by_build_config.setdefault(value['request_build_config'],
[]).append(value)
max_in_group = [
build_requests.BuildRequest(
**max(group, key=lambda value: value['timestamp']))
for group in by_build_config.values()
]
limit = None
if num_results != self.NUM_RESULTS_NO_LIMIT:
limit = num_results
return max_in_group[:limit]
def ScheduleSlaveBuildsViaBuildbucket(self,
important_only=False,
dryrun=False):
"""Schedule slave builds by sending PUT requests to Buildbucket.
Args:
important_only: Whether only schedule important slave builds, default to
False.
dryrun: Whether a dryrun, default to False.
"""
if self.buildbucket_client is None:
logging.info('No buildbucket_client. Skip scheduling slaves.')
return
build_identifier, db = self._run.GetCIDBHandle()
build_id = build_identifier.cidb_id
if build_id is None:
logging.info('No build id. Skip scheduling slaves.')
return
# May be None. This is okay.
master_buildbucket_id = self._run.options.buildbucket_id
if self._run.options.cbb_snapshot_revision:
logging.info('Parent has cbb_snapshot_rev=%s',
self._run.options.cbb_snapshot_revision)
scheduled_important_slave_builds = []
scheduled_experimental_slave_builds = []
unscheduled_slave_builds = []
scheduled_build_reqs = []
# Get all active slave build configs.
slave_config_map = self._GetSlaveConfigMap(important_only)
for slave_config_name, slave_config in sorted(slave_config_map.items()):
try:
buildbucket_id, created_ts = self.PostSlaveBuildToBuildbucket(
slave_config_name,
slave_config,
build_id,
master_buildbucket_id,
dryrun=dryrun)
request_reason = None
if slave_config.important:
scheduled_important_slave_builds.append((slave_config_name,
buildbucket_id, created_ts))
request_reason = build_requests.REASON_IMPORTANT_CQ_SLAVE
else:
scheduled_experimental_slave_builds.append(
(slave_config_name, buildbucket_id, created_ts))
request_reason = build_requests.REASON_EXPERIMENTAL_CQ_SLAVE
scheduled_build_reqs.append(
build_requests.BuildRequest(None, build_id, slave_config_name, None,
buildbucket_id, request_reason, None))
except buildbucket_lib.BuildbucketResponseException as e:
# Use 16-digit ts to be consistent with the created_ts from Buildbucket
current_ts = int(round(time.time() * 1000000))
unscheduled_slave_builds.append((slave_config_name, None, current_ts))
if important_only or slave_config.important:
raise
else:
logging.warning('Failed to schedule %s current timestamp %s: %s',
slave_config_name, current_ts, e)
if config_lib.IsMasterCQ(self._run.config) and db and scheduled_build_reqs:
db.InsertBuildRequests(scheduled_build_reqs)
self._run.attrs.metadata.ExtendKeyListWithList(
constants.METADATA_SCHEDULED_IMPORTANT_SLAVES,
scheduled_important_slave_builds)
self._run.attrs.metadata.ExtendKeyListWithList(
constants.METADATA_SCHEDULED_EXPERIMENTAL_SLAVES,
scheduled_experimental_slave_builds)
self._run.attrs.metadata.ExtendKeyListWithList(
constants.METADATA_UNSCHEDULED_SLAVES, unscheduled_slave_builds)