def test_update_doc_fields_no_fields_type(self): bisect_doc = mbisect.BisectDocument("foo") bisect_doc.id = "bar" self.assertDictEqual(bisect_doc.to_dict(), bcommon.update_doc_fields(bisect_doc, "None")) self.assertDictEqual( bisect_doc.to_dict(), bcommon.update_doc_fields(bisect_doc, ("None", None)))
def test_update_doc_fields_no_fields_type(self): bisect_doc = mbisect.BisectDocument("foo") bisect_doc.id = "bar" self.assertDictEqual( bisect_doc.to_dict(), bcommon.update_doc_fields(bisect_doc, "None") ) self.assertDictEqual( bisect_doc.to_dict(), bcommon.update_doc_fields(bisect_doc, ("None", None)) )
def test_update_doc_fields_list(self): bisect_doc = mbisect.BisectDocument("foo") bisect_doc.id = "bar" fields = ["bisect_data", "good_commit", "foo", "bar"] expected = {"_id": "bar", "bisect_data": [], "good_commit": None} self.assertDictEqual(expected, bcommon.update_doc_fields(bisect_doc, fields))
def test_update_doc_fields_list(self): bisect_doc = mbisect.BisectDocument("foo") bisect_doc.id = "bar" fields = ["bisect_data", "good_commit", "foo", "bar"] expected = { "_id": "bar", "bisect_data": [], "good_commit": None } self.assertDictEqual( expected, bcommon.update_doc_fields(bisect_doc, fields))
def test_update_doc_fields_dict(self): bisect_doc = mbisect.BisectDocument() bisect_doc.id = "bar" fields = { "bad_commit": True, "_id": False, "good_commit": False, "foo": False, "bar": True } expected = {"bad_commit": None} self.assertDictEqual(expected, bcommon.update_doc_fields(bisect_doc, fields))
def execute_boot_bisection(doc_id, db_options, fields=None): """Perform a bisect-like on the provided boot report. It searches all the previous boot reports starting from the provided one until it finds one whose boot passed. After that, it looks for all the build reports and combines the value into a single data structure. :param doc_id: The boot document ID. :type doc_id: str :param db_options: The mongodb database connection parameters. :type db_options: dict :param fields: A `fields` data structure with the fields to return or exclude. Default to None. :type fields: list or dict :return A numeric value for the result status and a list dictionaries. """ database = utils.db.get_db_connection(db_options) result = [] code = 200 obj_id = bson.objectid.ObjectId(doc_id) start_doc = utils.db.find_one2(database[models.BOOT_COLLECTION], obj_id, fields=BOOT_SEARCH_FIELDS) if start_doc and isinstance(start_doc, types.DictionaryType): start_doc_get = start_doc.get if start_doc_get(models.STATUS_KEY) == models.PASS_STATUS: code = 400 result = None else: bisect_doc = _find_boot_bisect_data(obj_id, start_doc, database, db_options) bcommon.save_bisect_doc(database, bisect_doc, doc_id) bisect_doc = bcommon.update_doc_fields(bisect_doc, fields) result = [bisect_doc] else: code = 404 result = None return code, result
def test_update_doc_fields_dict(self): bisect_doc = mbisect.BisectDocument("foo") bisect_doc.id = "bar" fields = { "bisect_data": True, "bad_commit": True, "_id": False, "good_commit": False, "foo": False, "bar": True } expected = { "bisect_data": [], "bad_commit": None } self.assertDictEqual( expected, bcommon.update_doc_fields(bisect_doc, fields))
def execute_boot_bisection(doc_id, db_options, fields=None): """Perform a bisect-like on the provided boot report. It searches all the previous boot reports starting from the provided one until it finds one whose boot passed. After that, it looks for all the build reports and combines the value into a single data structure. :param doc_id: The boot document ID. :type doc_id: str :param db_options: The mongodb database connection parameters. :type db_options: dict :param fields: A `fields` data structure with the fields to return or exclude. Default to None. :type fields: list or dict :return A numeric value for the result status and a list dictionaries. """ database = utils.db.get_db_connection(db_options) result = [] code = 200 obj_id = bson.objectid.ObjectId(doc_id) start_doc = utils.db.find_one2( database[models.BOOT_COLLECTION], obj_id, fields=BOOT_SEARCH_FIELDS) if start_doc and isinstance(start_doc, types.DictionaryType): start_doc_get = start_doc.get if start_doc_get(models.STATUS_KEY) == models.PASS_STATUS: code = 400 result = None else: bisect_doc = _find_boot_bisect_data( obj_id, start_doc, database, db_options) bcommon.save_bisect_doc(database, bisect_doc, doc_id) bisect_doc = bcommon.update_doc_fields(bisect_doc, fields) result = [bisect_doc] else: code = 404 result = None return code, result
def execute_boot_bisection_compared_to(doc_id, compare_to, db_options, fields=None): """Execute a bisect for one tree compared to another one. :param doc_id: The ID of the boot report we want compared. :type doc_id: string :param compare_to: The tree name to compare against. :type compare_to: string :param db_options: The options for the database connection. :type db_options: dictionary :param fields: A `fields` data structure with the fields to return or exclude. Default to None. :type fields: list or dict :return A numeric value for the result status and a list dictionaries. """ database = utils.db.get_db_connection(db_options) result = [] code = 200 obj_id = bson.objectid.ObjectId(doc_id) start_doc = utils.db.find_one2(database[models.BOOT_COLLECTION], obj_id, fields=BOOT_SEARCH_FIELDS) if start_doc and isinstance(start_doc, types.DictionaryType): start_doc_get = start_doc.get if start_doc_get(models.STATUS_KEY) == models.PASS_STATUS: code = 400 result = None else: # TODO: we need to know the baseline tree commit in order not to # search too much in the past. end_date, limit = bcommon.search_previous_bisect( database, {models.BOOT_ID_KEY: obj_id}, models.BISECT_BOOT_CREATED_KEY) board = start_doc_get(models.BOARD_KEY) job = start_doc_get(models.JOB_KEY) defconfig = start_doc_get(models.DEFCONFIG_KEY) defconfig_full = start_doc_get( models.DEFCONFIG_FULL_KEY) or defconfig created_on = start_doc_get(models.CREATED_KEY) arch = start_doc_get( models.ARCHITECTURE_KEY) or models.ARM_ARCHITECTURE_KEY lab_name = start_doc_get(models.LAB_NAME_KEY) bisect_doc = mbisect.BootBisectDocument(obj_id) bisect_doc.compare_to = compare_to bisect_doc.version = "1.0" bisect_doc.job = job bisect_doc.job_id = start_doc_get(models.JOB_ID_KEY, None) bisect_doc.build_id = start_doc_get(models.BUILD_ID_KEY, None) bisect_doc.boot_id = obj_id bisect_doc.created_on = datetime.datetime.now(tz=bson.tz_util.utc) bisect_doc.board = board bisect_doc.defconfig_full = defconfig_full bisect_doc.defconfig = defconfig bisect_doc.arch = arch if end_date: date_range = {"$lt": created_on, "$gte": end_date} else: date_range = {"$lt": created_on} spec = { models.LAB_NAME_KEY: lab_name, models.BOARD_KEY: board, models.DEFCONFIG_KEY: defconfig, models.DEFCONFIG_FULL_KEY: defconfig_full, models.JOB_KEY: compare_to, models.ARCHITECTURE_KEY: arch, models.GIT_BRANCH_KEY: start_doc_get(models.GIT_BRANCH_KEY), models.CREATED_KEY: date_range } prev_docs = utils.db.find(database[models.BOOT_COLLECTION], limit, 0, spec=spec, fields=BOOT_SEARCH_FIELDS, sort=BOOT_SORT) all_valid_docs = [] if prev_docs: # The function to apply to each boot document to find its # defconfig one and combine the values. func = bcommon.combine_defconfig_values all_valid_docs.extend( [func(doc, db_options) for doc in prev_docs]) bisect_doc.bisect_data = all_valid_docs bcommon.save_bisect_doc(database, bisect_doc, doc_id) bisect_doc = bcommon.update_doc_fields(bisect_doc, fields) result = [bisect_doc] else: code = 404 result = None return code, result
def execute_build_bisection(doc_id, db_options, fields=None): """Calculate bisect data for the provided build report. It searches all the previous builds starting from the provided one until it finds one that passed. After that, it combines the value into a single data structure. :param doc_id: The build document ID. :type doc_id: str :param db_options: The mongodb database connection parameters. :type db_options: dict :param fields: A `fields` data structure with the fields to return or exclude. Default to None. :type fields: list or dict :return A numeric value for the result status and a list of dictionaries. """ database = utils.db.get_db_connection(db_options) result = [] code = 200 obj_id = bson.objectid.ObjectId(doc_id) start_doc = utils.db.find_one( database[models.BUILD_COLLECTION], [obj_id], fields=BUILD_SEARCH_FIELDS ) if start_doc and isinstance(start_doc, types.DictionaryType): start_doc_get = start_doc.get if start_doc_get(models.STATUS_KEY) == models.PASS_STATUS: code = 400 result = None else: bisect_doc = mbisect.DefconfigBisectDocument(obj_id) bisect_doc.version = "1.0" bisect_doc.arch = start_doc_get(models.ARCHITECTURE_KEY, None) bisect_doc.job = start_doc_get(models.JOB_KEY, None) bisect_doc.job_id = start_doc_get(models.JOB_ID_KEY, None) bisect_doc.build_id = start_doc_get(models.ID_KEY) bisect_doc.defconfig = start_doc_get(models.DEFCONFIG_KEY, None) bisect_doc.defconfig_full = start_doc_get( models.DEFCONFIG_FULL_KEY, None) bisect_doc.git_branch = start_doc_get(models.GIT_BRANCH_KEY) bisect_doc.created_on = datetime.datetime.now(tz=bson.tz_util.utc) bisect_doc.bad_commit_date = start_doc_get(models.CREATED_KEY) bisect_doc.bad_commit = start_doc_get(models.GIT_COMMIT_KEY) bisect_doc.bad_commit_url = start_doc_get(models.GIT_URL_KEY) spec = { models.ARCHITECTURE_KEY: start_doc_get( models.ARCHITECTURE_KEY), models.DEFCONFIG_FULL_KEY: start_doc_get( models.DEFCONFIG_FULL_KEY), models.DEFCONFIG_KEY: start_doc_get(models.DEFCONFIG_KEY), models.JOB_KEY: start_doc_get(models.JOB_KEY), models.GIT_BRANCH_KEY: start_doc_get(models.GIT_BRANCH_KEY) } all_valid_docs = [start_doc] # Search for the first passed build so that we can limit the # next search. Doing this to cut down search and load time on # mongodb side: there are a lot of build documents to search # for and the mongodb Cursor can get quite big. # Tweak the spec to search for PASS status and limit also the # result found: we are only interested in the first found one. # Need to use copy.deepcoy here since for some strange reasons, # just adding and removing the keys from the spec is not working # as expected. pass_spec = copy.deepcopy(spec) pass_spec[models.STATUS_KEY] = models.PASS_STATUS # Only interested in older builds. pass_spec[models.CREATED_KEY] = \ {"$lt": start_doc_get(models.CREATED_KEY)} passed_builds = utils.db.find( database[models.BUILD_COLLECTION], 10, 0, spec=pass_spec, fields=BUILD_SEARCH_FIELDS, sort=BUILD_SORT ) # In case we have a passed doc, tweak the spec to search between # the valid dates. passed_build = None if passed_builds.count() > 0: passed_build = _search_passed_doc(passed_builds) if passed_build is not None: spec[models.CREATED_KEY] = { "$gte": passed_build.get(models.CREATED_KEY), "$lt": start_doc_get(models.CREATED_KEY) } else: utils.LOG.warn("No passed build found for '%s'", obj_id) spec[models.CREATED_KEY] = { "$lt": start_doc_get(models.CREATED_KEY) } all_prev_docs = utils.db.find( database[models.BUILD_COLLECTION], 0, 0, spec=spec, fields=BUILD_SEARCH_FIELDS, sort=BUILD_SORT ) if all_prev_docs: all_valid_docs.extend( [ doc for doc in bcommon.get_docs_until_pass(all_prev_docs) ] ) if passed_build and all_valid_docs[-1] != passed_build: all_valid_docs.append(passed_build) # The last doc should be the good one, in case it is, add the # values to the bisect_doc. good_doc = all_valid_docs[-1] if good_doc[models.STATUS_KEY] == models.PASS_STATUS: good_doc_get = good_doc.get bisect_doc.good_commit = good_doc_get( models.GIT_COMMIT_KEY) bisect_doc.good_commit_url = good_doc_get( models.GIT_URL_KEY) bisect_doc.good_commit_date = good_doc_get( models.CREATED_KEY) # Store everything in the bisect data. bisect_doc.bisect_data = all_valid_docs bcommon.save_bisect_doc(database, bisect_doc, doc_id) bisect_doc = bcommon.update_doc_fields(bisect_doc, fields) result = [bisect_doc] else: code = 404 result = None return code, result
def execute_boot_bisection_compared_to( doc_id, compare_to, db_options, fields=None): """Execute a bisect for one tree compared to another one. :param doc_id: The ID of the boot report we want compared. :type doc_id: string :param compare_to: The tree name to compare against. :type compare_to: string :param db_options: The options for the database connection. :type db_options: dictionary :param fields: A `fields` data structure with the fields to return or exclude. Default to None. :type fields: list or dict :return A numeric value for the result status and a list dictionaries. """ database = utils.db.get_db_connection(db_options) result = [] code = 200 obj_id = bson.objectid.ObjectId(doc_id) start_doc = utils.db.find_one2( database[models.BOOT_COLLECTION], obj_id, fields=BOOT_SEARCH_FIELDS) if start_doc and isinstance(start_doc, types.DictionaryType): start_doc_get = start_doc.get if start_doc_get(models.STATUS_KEY) == models.PASS_STATUS: code = 400 result = None else: # TODO: we need to know the baseline tree commit in order not to # search too much in the past. end_date, limit = bcommon.search_previous_bisect( database, {models.BOOT_ID_KEY: obj_id}, models.BISECT_BOOT_CREATED_KEY ) board = start_doc_get(models.BOARD_KEY) job = start_doc_get(models.JOB_KEY) defconfig = start_doc_get(models.DEFCONFIG_KEY) defconfig_full = start_doc_get( models.DEFCONFIG_FULL_KEY) or defconfig created_on = start_doc_get(models.CREATED_KEY) arch = start_doc_get(models.ARCHITECTURE_KEY) lab_name = start_doc_get(models.LAB_NAME_KEY) bisect_doc = mbisect.BootBisectDocument(obj_id) bisect_doc.compare_to = compare_to bisect_doc.version = "1.0" bisect_doc.job = job bisect_doc.job_id = start_doc_get(models.JOB_ID_KEY, None) bisect_doc.build_id = start_doc_get(models.BUILD_ID_KEY, None) bisect_doc.boot_id = obj_id bisect_doc.created_on = datetime.datetime.now(tz=bson.tz_util.utc) bisect_doc.board = board bisect_doc.defconfig_full = defconfig_full bisect_doc.defconfig = defconfig bisect_doc.arch = arch if end_date: date_range = { "$lt": created_on, "$gte": end_date } else: date_range = {"$lt": created_on} spec = { models.LAB_NAME_KEY: lab_name, models.BOARD_KEY: board, models.DEFCONFIG_KEY: defconfig, models.DEFCONFIG_FULL_KEY: defconfig_full, models.JOB_KEY: compare_to, models.ARCHITECTURE_KEY: arch, models.GIT_BRANCH_KEY: start_doc_get(models.GIT_BRANCH_KEY), models.CREATED_KEY: date_range } prev_docs = utils.db.find( database[models.BOOT_COLLECTION], limit, 0, spec=spec, fields=BOOT_SEARCH_FIELDS, sort=BOOT_SORT) all_valid_docs = [] if prev_docs: # The function to apply to each boot document to find its # defconfig one and combine the values. func = bcommon.combine_defconfig_values all_valid_docs.extend( [ func(doc, db_options) for doc in prev_docs ] ) bisect_doc.bisect_data = all_valid_docs bcommon.save_bisect_doc(database, bisect_doc, doc_id) bisect_doc = bcommon.update_doc_fields(bisect_doc, fields) result = [bisect_doc] else: code = 404 result = None return code, result
def test_update_doc_fields_no_fields(self): bisect_doc = mbisect.BisectDocument() bisect_doc.id = "bar" self.assertDictEqual(bisect_doc.to_dict(), bcommon.update_doc_fields(bisect_doc, None))
def execute_defconfig_bisection(doc_id, db_options, fields=None): """Calculate bisect data for the provided defconfig report. It searches all the previous defconfig built starting from the provided one until it finds one that passed. After that, it combines the value into a single data structure. :param doc_id: The boot document ID. :type doc_id: str :param db_options: The mongodb database connection parameters. :type db_options: dict :param fields: A `fields` data structure with the fields to return or exclude. Default to None. :type fields: list or dict :return A numeric value for the result status and a list of dictionaries. """ database = utils.db.get_db_connection(db_options) result = [] code = 200 obj_id = bson.objectid.ObjectId(doc_id) start_doc = utils.db.find_one( database[models.DEFCONFIG_COLLECTION], [obj_id], fields=DEFCONFIG_SEARCH_FIELDS ) if all([start_doc, isinstance(start_doc, types.DictionaryType)]): start_doc_get = start_doc.get if start_doc_get(models.STATUS_KEY) == models.PASS_STATUS: code = 400 result = None else: bisect_doc = mbisect.DefconfigBisectDocument(obj_id) bisect_doc.version = "1.0" bisect_doc.arch = start_doc_get(models.ARCHITECTURE_KEY, None) bisect_doc.job = start_doc_get(models.JOB_KEY, None) bisect_doc.job_id = start_doc_get(models.JOB_ID_KEY, None) bisect_doc.defconfig_id = start_doc_get(models.ID_KEY) bisect_doc.defconfig = start_doc_get(models.DEFCONFIG_KEY, None) bisect_doc.defconfig_full = start_doc_get( models.DEFCONFIG_FULL_KEY, None) bisect_doc.created_on = datetime.datetime.now(tz=bson.tz_util.utc) bisect_doc.bad_commit_date = start_doc_get(models.CREATED_KEY) bisect_doc.bad_commit = start_doc_get(models.GIT_COMMIT_KEY) bisect_doc.bad_commit_url = start_doc_get(models.GIT_URL_KEY) spec = { models.ARCHITECTURE_KEY: start_doc_get( models.ARCHITECTURE_KEY), models.DEFCONFIG_FULL_KEY: start_doc_get( models.DEFCONFIG_FULL_KEY), models.DEFCONFIG_KEY: start_doc_get(models.DEFCONFIG_KEY), models.JOB_KEY: start_doc_get(models.JOB_KEY), } all_valid_docs = [start_doc] # Search for the first passed defconfig so that we can limit the # next search. Doing this to cut down search and load time on # mongodb side: there are a lot of defconfig documents to search # for and the mongodb Cursor can get quite big. # Tweak the spec to search for PASS status and limit also the # result found: we are only interested in the first found one. # Need to use copy.deepcoy here since for some strange reasons, # just adding and removing the keys from the spec is not working # as expected. pass_spec = copy.deepcopy(spec) pass_spec[models.STATUS_KEY] = models.PASS_STATUS # Only interested in older builds. pass_spec[models.CREATED_KEY] = \ {"$lt": start_doc_get(models.CREATED_KEY)} passed_builds = utils.db.find( database[models.DEFCONFIG_COLLECTION], 10, 0, spec=pass_spec, fields=DEFCONFIG_SEARCH_FIELDS, sort=DEFCONFIG_SORT ) # In case we have a passed doc, tweak the spec to search between # the valid dates. passed_build = None if passed_builds.count() > 0: passed_build = _search_passed_doc(passed_builds) if passed_build is not None: spec[models.CREATED_KEY] = { "$gte": passed_build.get(models.CREATED_KEY), "$lt": start_doc_get(models.CREATED_KEY) } else: utils.LOG.warn("No passed build found for '%s'", obj_id) spec[models.CREATED_KEY] = { "$lt": start_doc_get(models.CREATED_KEY) } all_prev_docs = utils.db.find( database[models.DEFCONFIG_COLLECTION], 0, 0, spec=spec, fields=DEFCONFIG_SEARCH_FIELDS, sort=DEFCONFIG_SORT ) if all_prev_docs: all_valid_docs.extend( [ doc for doc in bcommon.get_docs_until_pass(all_prev_docs) ] ) if all([passed_build, all_valid_docs[-1] != passed_build]): all_valid_docs.append(passed_build) # The last doc should be the good one, in case it is, add the # values to the bisect_doc. good_doc = all_valid_docs[-1] if good_doc[models.STATUS_KEY] == models.PASS_STATUS: good_doc_get = good_doc.get bisect_doc.good_commit = good_doc_get( models.GIT_COMMIT_KEY) bisect_doc.good_commit_url = good_doc_get( models.GIT_URL_KEY) bisect_doc.good_commit_date = good_doc_get( models.CREATED_KEY) # Store everything in the bisect data. bisect_doc.bisect_data = all_valid_docs bcommon.save_bisect_doc(database, bisect_doc, doc_id) bisect_doc = bcommon.update_doc_fields(bisect_doc, fields) result = [bisect_doc] else: code = 404 result = None return code, result