def test_get_docs_until_pass_with_pass(self):
doc_list = [
{
"status": "foo",
"id": 1
},
{
"status": "foo",
"id": 2
},
{
"status": "PASS",
"id": 3
},
{
"status": "foo",
"id": 4
},
{
"status": "foo",
"id": 5
},
]
retrieved_list = [doc for doc in bcommon.get_docs_until_pass(doc_list)]
self.assertEqual(len(retrieved_list), 3)
self.assertListEqual(doc_list[:3], retrieved_list)
def test_get_docs_until_pass_no_pass(self):
doc_list = [
{
"status": "foo",
"id": 1
},
{
"status": "foo",
"id": 2
},
{
"status": "foo",
"id": 3
},
{
"status": "foo",
"id": 4
},
{
"status": "foo",
"id": 5
},
]
retrieved_list = [doc for doc in bcommon.get_docs_until_pass(doc_list)]
self.assertListEqual(doc_list, retrieved_list)
def test_get_docs_until_pass_with_pass_last(self):
doc_list = [
{"status": "foo", "id": 1},
{"status": "foo", "id": 2},
{"status": "foo", "id": 3},
{"status": "foo", "id": 4},
{"status": "PASS", "id": 5},
]
retrieved_list = [doc for doc in bcommon.get_docs_until_pass(doc_list)]
self.assertListEqual(doc_list, retrieved_list)
def _find_boot_bisect_data(obj_id, start_doc, database, db_options):
"""Execute the real bisect logic.
This is where the BisectDocument is created and returned.
:param obj_id: The `bson.objectid.ObjectId` of the starting point.
:type obj_id: bson.objectid.ObjectId
:param start_doc: The starting document.
:type start_doc: dictionary
:param database: The connection to the database.
:param db_options: The options for the database connection.
:type db_options: dictionary
:return A BisectDocument instance.
"""
start_doc_get = start_doc.get
board = start_doc_get(models.BOARD_KEY)
job = start_doc_get(models.JOB_KEY)
defconfig = start_doc_get(models.DEFCONFIG_KEY)
defconfig_full = start_doc_get(models.DEFCONFIG_FULL_KEY) or defconfig
created_on = start_doc_get(models.CREATED_KEY)
arch = start_doc_get(
models.ARCHITECTURE_KEY) or models.ARM_ARCHITECTURE_KEY
lab_name = start_doc_get(models.LAB_NAME_KEY)
bisect_doc = mbisect.BootBisectDocument(obj_id)
bisect_doc.boot_id = obj_id
bisect_doc.version = "1.0"
bisect_doc.job = job
bisect_doc.job_id = start_doc_get(models.JOB_ID_KEY, None)
bisect_doc.build_id = start_doc_get(models.BUILD_ID_KEY, None)
bisect_doc.created_on = datetime.datetime.now(tz=bson.tz_util.utc)
bisect_doc.board = board
bisect_doc.arch = arch
bisect_doc.defconfig = defconfig
bisect_doc.defconfig_full = defconfig_full
spec = {
models.LAB_NAME_KEY: lab_name,
models.BOARD_KEY: board,
models.DEFCONFIG_KEY: defconfig,
models.DEFCONFIG_FULL_KEY: defconfig_full,
models.JOB_KEY: start_doc_get(models.JOB_KEY),
models.ARCHITECTURE_KEY: arch,
models.CREATED_KEY: {
"$lt": created_on
},
models.GIT_BRANCH_KEY: start_doc_get(models.GIT_BRANCH_KEY)
}
# The function to apply to each boot document to find its build
# one and combine the values.
func = bcommon.combine_defconfig_values
bad_doc = func(start_doc, db_options)
bad_doc_get = bad_doc.get
bisect_doc.bad_commit_date = bad_doc_get(
models.BISECT_DEFCONFIG_CREATED_KEY)
bisect_doc.bad_commit = bad_doc_get(models.GIT_COMMIT_KEY)
bisect_doc.bad_commit_url = bad_doc_get(models.GIT_URL_KEY)
all_valid_docs = [bad_doc]
# Search through all the previous boot reports, until one that
# passed is found, and combine them with their build document.
all_prev_docs = utils.db.find(database[models.BOOT_COLLECTION],
0,
0,
spec=spec,
fields=BOOT_SEARCH_FIELDS,
sort=BOOT_SORT)
if all_prev_docs:
all_valid_docs.extend([
func(doc, db_options)
for doc in bcommon.get_docs_until_pass(all_prev_docs)
])
# The last doc should be the good one, in case it is, add the
# values to the bisect_doc.
good_doc = all_valid_docs[-1]
if (good_doc[models.BISECT_BOOT_STATUS_KEY] == models.PASS_STATUS):
good_doc_get = good_doc.get
bisect_doc.good_commit = good_doc_get(models.GIT_COMMIT_KEY)
bisect_doc.good_commit_url = good_doc_get(models.GIT_URL_KEY)
bisect_doc.good_commit_date = good_doc_get(
models.BISECT_DEFCONFIG_CREATED_KEY)
# Store everything in the bisect_data list of the bisect_doc.
bisect_doc.bisect_data = all_valid_docs
return bisect_doc
def execute_build_bisection(doc_id, db_options, fields=None):
"""Calculate bisect data for the provided build report.
It searches all the previous builds starting from the provided one
until it finds one that passed. After that, it combines the value into a
single data structure.
:param doc_id: The build document ID.
:type doc_id: str
:param db_options: The mongodb database connection parameters.
:type db_options: dict
:param fields: A `fields` data structure with the fields to return or
exclude. Default to None.
:type fields: list or dict
:return A numeric value for the result status and a list of dictionaries.
"""
database = utils.db.get_db_connection(db_options)
result = []
code = 200
obj_id = bson.objectid.ObjectId(doc_id)
start_doc = utils.db.find_one(
database[models.BUILD_COLLECTION],
[obj_id],
fields=BUILD_SEARCH_FIELDS
)
if start_doc and isinstance(start_doc, types.DictionaryType):
start_doc_get = start_doc.get
if start_doc_get(models.STATUS_KEY) == models.PASS_STATUS:
code = 400
result = None
else:
bisect_doc = mbisect.DefconfigBisectDocument(obj_id)
bisect_doc.version = "1.0"
bisect_doc.arch = start_doc_get(models.ARCHITECTURE_KEY, None)
bisect_doc.job = start_doc_get(models.JOB_KEY, None)
bisect_doc.job_id = start_doc_get(models.JOB_ID_KEY, None)
bisect_doc.build_id = start_doc_get(models.ID_KEY)
bisect_doc.defconfig = start_doc_get(models.DEFCONFIG_KEY, None)
bisect_doc.defconfig_full = start_doc_get(
models.DEFCONFIG_FULL_KEY, None)
bisect_doc.git_branch = start_doc_get(models.GIT_BRANCH_KEY)
bisect_doc.created_on = datetime.datetime.now(tz=bson.tz_util.utc)
bisect_doc.bad_commit_date = start_doc_get(models.CREATED_KEY)
bisect_doc.bad_commit = start_doc_get(models.GIT_COMMIT_KEY)
bisect_doc.bad_commit_url = start_doc_get(models.GIT_URL_KEY)
spec = {
models.ARCHITECTURE_KEY: start_doc_get(
models.ARCHITECTURE_KEY),
models.DEFCONFIG_FULL_KEY: start_doc_get(
models.DEFCONFIG_FULL_KEY),
models.DEFCONFIG_KEY: start_doc_get(models.DEFCONFIG_KEY),
models.JOB_KEY: start_doc_get(models.JOB_KEY),
models.GIT_BRANCH_KEY: start_doc_get(models.GIT_BRANCH_KEY)
}
all_valid_docs = [start_doc]
# Search for the first passed build so that we can limit the
# next search. Doing this to cut down search and load time on
# mongodb side: there are a lot of build documents to search
# for and the mongodb Cursor can get quite big.
# Tweak the spec to search for PASS status and limit also the
# result found: we are only interested in the first found one.
# Need to use copy.deepcoy here since for some strange reasons,
# just adding and removing the keys from the spec is not working
# as expected.
pass_spec = copy.deepcopy(spec)
pass_spec[models.STATUS_KEY] = models.PASS_STATUS
# Only interested in older builds.
pass_spec[models.CREATED_KEY] = \
{"$lt": start_doc_get(models.CREATED_KEY)}
passed_builds = utils.db.find(
database[models.BUILD_COLLECTION],
10,
0,
spec=pass_spec,
fields=BUILD_SEARCH_FIELDS,
sort=BUILD_SORT
)
# In case we have a passed doc, tweak the spec to search between
# the valid dates.
passed_build = None
if passed_builds.count() > 0:
passed_build = _search_passed_doc(passed_builds)
if passed_build is not None:
spec[models.CREATED_KEY] = {
"$gte": passed_build.get(models.CREATED_KEY),
"$lt": start_doc_get(models.CREATED_KEY)
}
else:
utils.LOG.warn("No passed build found for '%s'", obj_id)
spec[models.CREATED_KEY] = {
"$lt": start_doc_get(models.CREATED_KEY)
}
all_prev_docs = utils.db.find(
database[models.BUILD_COLLECTION],
0,
0,
spec=spec,
fields=BUILD_SEARCH_FIELDS,
sort=BUILD_SORT
)
if all_prev_docs:
all_valid_docs.extend(
[
doc for doc in
bcommon.get_docs_until_pass(all_prev_docs)
]
)
if passed_build and all_valid_docs[-1] != passed_build:
all_valid_docs.append(passed_build)
# The last doc should be the good one, in case it is, add the
# values to the bisect_doc.
good_doc = all_valid_docs[-1]
if good_doc[models.STATUS_KEY] == models.PASS_STATUS:
good_doc_get = good_doc.get
bisect_doc.good_commit = good_doc_get(
models.GIT_COMMIT_KEY)
bisect_doc.good_commit_url = good_doc_get(
models.GIT_URL_KEY)
bisect_doc.good_commit_date = good_doc_get(
models.CREATED_KEY)
# Store everything in the bisect data.
bisect_doc.bisect_data = all_valid_docs
bcommon.save_bisect_doc(database, bisect_doc, doc_id)
bisect_doc = bcommon.update_doc_fields(bisect_doc, fields)
result = [bisect_doc]
else:
code = 404
result = None
return code, result
def _find_boot_bisect_data(obj_id, start_doc, database, db_options):
"""Execute the real bisect logic.
This is where the BisectDocument is created and returned.
:param obj_id: The `bson.objectid.ObjectId` of the starting point.
:type obj_id: bson.objectid.ObjectId
:param start_doc: The starting document.
:type start_doc: dictionary
:param database: The connection to the database.
:param db_options: The options for the database connection.
:type db_options: dictionary
:return A BisectDocument instance.
"""
start_doc_get = start_doc.get
board = start_doc_get(models.BOARD_KEY)
job = start_doc_get(models.JOB_KEY)
defconfig = start_doc_get(models.DEFCONFIG_KEY)
defconfig_full = start_doc_get(models.DEFCONFIG_FULL_KEY) or defconfig
created_on = start_doc_get(models.CREATED_KEY)
arch = start_doc_get(models.ARCHITECTURE_KEY)
lab_name = start_doc_get(models.LAB_NAME_KEY)
bisect_doc = mbisect.BootBisectDocument(obj_id)
bisect_doc.boot_id = obj_id
bisect_doc.version = "1.0"
bisect_doc.job = job
bisect_doc.job_id = start_doc_get(models.JOB_ID_KEY, None)
bisect_doc.build_id = start_doc_get(models.BUILD_ID_KEY, None)
bisect_doc.created_on = datetime.datetime.now(tz=bson.tz_util.utc)
bisect_doc.board = board
bisect_doc.arch = arch
bisect_doc.defconfig = defconfig
bisect_doc.defconfig_full = defconfig_full
spec = {
models.LAB_NAME_KEY: lab_name,
models.BOARD_KEY: board,
models.DEFCONFIG_KEY: defconfig,
models.DEFCONFIG_FULL_KEY: defconfig_full,
models.JOB_KEY: start_doc_get(models.JOB_KEY),
models.ARCHITECTURE_KEY: arch,
models.CREATED_KEY: {"$lt": created_on},
models.GIT_BRANCH_KEY: start_doc_get(models.GIT_BRANCH_KEY)
}
# The function to apply to each boot document to find its build
# one and combine the values.
func = bcommon.combine_defconfig_values
bad_doc = func(start_doc, db_options)
bad_doc_get = bad_doc.get
bisect_doc.bad_commit_date = bad_doc_get(
models.BISECT_DEFCONFIG_CREATED_KEY)
bisect_doc.bad_commit = bad_doc_get(models.GIT_COMMIT_KEY)
bisect_doc.bad_commit_url = bad_doc_get(models.GIT_URL_KEY)
all_valid_docs = [bad_doc]
# Search through all the previous boot reports, until one that
# passed is found, and combine them with their build document.
all_prev_docs = utils.db.find(
database[models.BOOT_COLLECTION],
0,
0,
spec=spec,
fields=BOOT_SEARCH_FIELDS,
sort=BOOT_SORT
)
if all_prev_docs:
all_valid_docs.extend(
[
func(doc, db_options)
for doc in bcommon.get_docs_until_pass(all_prev_docs)
]
)
# The last doc should be the good one, in case it is, add the
# values to the bisect_doc.
good_doc = all_valid_docs[-1]
if (good_doc[models.BISECT_BOOT_STATUS_KEY] ==
models.PASS_STATUS):
good_doc_get = good_doc.get
bisect_doc.good_commit = good_doc_get(
models.GIT_COMMIT_KEY)
bisect_doc.good_commit_url = good_doc_get(
models.GIT_URL_KEY)
bisect_doc.good_commit_date = good_doc_get(
models.BISECT_DEFCONFIG_CREATED_KEY)
# Store everything in the bisect_data list of the bisect_doc.
bisect_doc.bisect_data = all_valid_docs
return bisect_doc
def execute_defconfig_bisection(doc_id, db_options, fields=None):
"""Calculate bisect data for the provided defconfig report.
It searches all the previous defconfig built starting from the provided one
until it finds one that passed. After that, it combines the value into a
single data structure.
:param doc_id: The boot document ID.
:type doc_id: str
:param db_options: The mongodb database connection parameters.
:type db_options: dict
:param fields: A `fields` data structure with the fields to return or
exclude. Default to None.
:type fields: list or dict
:return A numeric value for the result status and a list of dictionaries.
"""
database = utils.db.get_db_connection(db_options)
result = []
code = 200
obj_id = bson.objectid.ObjectId(doc_id)
start_doc = utils.db.find_one(
database[models.DEFCONFIG_COLLECTION],
[obj_id],
fields=DEFCONFIG_SEARCH_FIELDS
)
if all([start_doc, isinstance(start_doc, types.DictionaryType)]):
start_doc_get = start_doc.get
if start_doc_get(models.STATUS_KEY) == models.PASS_STATUS:
code = 400
result = None
else:
bisect_doc = mbisect.DefconfigBisectDocument(obj_id)
bisect_doc.version = "1.0"
bisect_doc.arch = start_doc_get(models.ARCHITECTURE_KEY, None)
bisect_doc.job = start_doc_get(models.JOB_KEY, None)
bisect_doc.job_id = start_doc_get(models.JOB_ID_KEY, None)
bisect_doc.defconfig_id = start_doc_get(models.ID_KEY)
bisect_doc.defconfig = start_doc_get(models.DEFCONFIG_KEY, None)
bisect_doc.defconfig_full = start_doc_get(
models.DEFCONFIG_FULL_KEY, None)
bisect_doc.created_on = datetime.datetime.now(tz=bson.tz_util.utc)
bisect_doc.bad_commit_date = start_doc_get(models.CREATED_KEY)
bisect_doc.bad_commit = start_doc_get(models.GIT_COMMIT_KEY)
bisect_doc.bad_commit_url = start_doc_get(models.GIT_URL_KEY)
spec = {
models.ARCHITECTURE_KEY: start_doc_get(
models.ARCHITECTURE_KEY),
models.DEFCONFIG_FULL_KEY: start_doc_get(
models.DEFCONFIG_FULL_KEY),
models.DEFCONFIG_KEY: start_doc_get(models.DEFCONFIG_KEY),
models.JOB_KEY: start_doc_get(models.JOB_KEY),
}
all_valid_docs = [start_doc]
# Search for the first passed defconfig so that we can limit the
# next search. Doing this to cut down search and load time on
# mongodb side: there are a lot of defconfig documents to search
# for and the mongodb Cursor can get quite big.
# Tweak the spec to search for PASS status and limit also the
# result found: we are only interested in the first found one.
# Need to use copy.deepcoy here since for some strange reasons,
# just adding and removing the keys from the spec is not working
# as expected.
pass_spec = copy.deepcopy(spec)
pass_spec[models.STATUS_KEY] = models.PASS_STATUS
# Only interested in older builds.
pass_spec[models.CREATED_KEY] = \
{"$lt": start_doc_get(models.CREATED_KEY)}
passed_builds = utils.db.find(
database[models.DEFCONFIG_COLLECTION],
10,
0,
spec=pass_spec,
fields=DEFCONFIG_SEARCH_FIELDS,
sort=DEFCONFIG_SORT
)
# In case we have a passed doc, tweak the spec to search between
# the valid dates.
passed_build = None
if passed_builds.count() > 0:
passed_build = _search_passed_doc(passed_builds)
if passed_build is not None:
spec[models.CREATED_KEY] = {
"$gte": passed_build.get(models.CREATED_KEY),
"$lt": start_doc_get(models.CREATED_KEY)
}
else:
utils.LOG.warn("No passed build found for '%s'", obj_id)
spec[models.CREATED_KEY] = {
"$lt": start_doc_get(models.CREATED_KEY)
}
all_prev_docs = utils.db.find(
database[models.DEFCONFIG_COLLECTION],
0,
0,
spec=spec,
fields=DEFCONFIG_SEARCH_FIELDS,
sort=DEFCONFIG_SORT
)
if all_prev_docs:
all_valid_docs.extend(
[
doc for doc in
bcommon.get_docs_until_pass(all_prev_docs)
]
)
if all([passed_build, all_valid_docs[-1] != passed_build]):
all_valid_docs.append(passed_build)
# The last doc should be the good one, in case it is, add the
# values to the bisect_doc.
good_doc = all_valid_docs[-1]
if good_doc[models.STATUS_KEY] == models.PASS_STATUS:
good_doc_get = good_doc.get
bisect_doc.good_commit = good_doc_get(
models.GIT_COMMIT_KEY)
bisect_doc.good_commit_url = good_doc_get(
models.GIT_URL_KEY)
bisect_doc.good_commit_date = good_doc_get(
models.CREATED_KEY)
# Store everything in the bisect data.
bisect_doc.bisect_data = all_valid_docs
bcommon.save_bisect_doc(database, bisect_doc, doc_id)
bisect_doc = bcommon.update_doc_fields(bisect_doc, fields)
result = [bisect_doc]
else:
code = 404
result = None
return code, result