Пример #1
0
    def update_test_bucket(self, conf_info):
        """
        update the conf bucket with most test info
        """
        bucket = Bucket('couchbase://%s/conf' % self.repo_cluster)

        for info in conf_info:
            conf = info.get('confFile', None)
            component = info.get('component', "")
            subcomponent = info.get('subcomponent', "")
            test_type = info.get('type', "")
            if conf is None:
                continue
            conf = CG.rm_leading_slash(conf)
            tests = info.get('tests', None)
            if tests is None:
                tests = CG.parse_conf_file(conf)
            bucket.upsert(
                conf,
                {'tests': tests,
                 'component': component,
                 'subcomponent': subcomponent,
                 'type': test_type})
Пример #2
0
    def update_history_bucket(self, conf_info):
        """
        update the history bucket with changes between git repo and conf bucket
        """

        url = 'couchbase://%s' % self.repo_cluster
        history_bucket = Bucket('%s/history' % url)
        test_bucket = Bucket('%s/conf' % url)
        timestamp = CG.timestamp()
        total_new = total_removed = total_changed = 0

        for info in conf_info:
            conf = info.get('confFile', None)
            component = info.get('component', "")
            subcomponent = info.get('subcomponent', "")
            test_type = info.get('type', "")
            if conf is None or conf in self.conf_history:
                # no conf or duplicate
                continue
            else:
                # update conf history
                self.conf_history.append(conf)

            conf = CG.rm_leading_slash(conf)
            doc = self.safe_get_doc(test_bucket, conf)

            # get last known status of tests for this conf
            if doc is not None and doc.rc == 0:
                cb_tests = doc.value.get('tests')
                if cb_tests is not None:
                    repo_tests = info.get('tests', None)
                    if repo_tests is None:
                        repo_tests = [str(t) for t in CG.parse_conf_file(conf)]

                    # array comparison
                    if cb_tests == repo_tests:
                        continue # same same

                    curr_new = curr_removed = curr_changed = 0
                    change_doc = {'ts': timestamp,
                                  'new': [],
                                  'removed': [],
                                  'changed': [],
                                  'component': component,
                                  'subcomponent': subcomponent,
                                  'type': test_type,
                                  'conf': conf}

                    # sort tests
                    cb_tests.sort()
                    repo_tests.sort()

                    # detect new/removed tests
                    base_repo_tests = CG.split_test_base(repo_tests)
                    base_cb_tests = CG.split_test_base(cb_tests)
                    new_tests = set(base_repo_tests) - set(base_cb_tests)
                    removed_tests = set(base_cb_tests) - set(base_repo_tests)

                    # record which test changed
                    for test in new_tests:
                        # get params
                        params_index = base_repo_tests.index(test)
                        param_parts = repo_tests[params_index].split(",")
                        params = ""
                        if len(param_parts) > 1:
                            params = ",%s" % param_parts[1]
                        test = "%s%s" % (test, params)
                        print "[new] %s" % test
                        change_doc['new'].append(test)
                        curr_new += 1

                    for test in removed_tests:
                        # get params
                        params_index = base_cb_tests.index(test)
                        param_parts = cb_tests[params_index].split(",")
                        params = ""
                        if len(param_parts) > 1:
                            params = ",%s" % param_parts[1]
                        test = "%s%s" % (test, params)
                        print "[removed] %s" % test
                        change_doc['removed'].append(test)
                        curr_removed += 1

                    # detect param changes
                    base_repo_params = [CG.parse_params(t) for t in repo_tests]
                    base_cb_params = [CG.parse_params(t) for t in cb_tests]
                    changed_params = set(base_repo_params) - set(base_cb_params)

                    # record which params changed
                    #     as determined by params in which
                    #     no params have been added or removed
                    for params in changed_params:

                        # given the set comparison
                        # the change will be in base_repo_params
                        change_index = base_repo_params.index(params)
                        changed_test = base_repo_tests[change_index]

                        # check if test already processed
                        if  changed_test in list(new_tests):
                            continue # already processed as new
                        if  changed_test in list(removed_tests):
                            continue # already processed as removed

                        # get original test info and params
                        original_test_index = base_cb_tests.index(changed_test)
                        original_test = cb_tests[original_test_index]
                        original_params = CG.parse_params(original_test)

                        # determine if real change or is just an add
                        original_params_keys = set(
                            CG.split_test_param_keys(original_params))
                        changed_params_keys = set(
                            CG.split_test_param_keys(params))

                        if original_params_keys != changed_params_keys:
                            test_new = "%s,%s" % (changed_test, params)
                            print "[new] %s" % test_new
                            change_doc['new'].append(test_new)
                            curr_new += 1
                            continue

                        # detect which value param actually changed
                        original_params_kv = CG.split_test_params(original_params)
                        changed_params_kv = CG.split_test_params(params)

                        to_diff_params = set(original_params_kv) - set(changed_params_kv)
                        from_diff_params = set(changed_params_kv) - set(original_params_kv)

                        # eval diff
                        param_to_str = ",".join(list(to_diff_params))
                        param_from_str = ",".join(list(from_diff_params))

                        print "[change] %s:  (%s) -> (%s)" % (
                            changed_test,
                            param_to_str,
                            param_from_str)
                        change_doc['changed'].append(
                            {'test': changed_test,
                             'to': param_to_str,
                             'from': param_from_str})
                        curr_changed += 1

                    curr_total = curr_new + curr_removed + curr_changed
                    if  curr_total > 0:
                        # push change
                        timestamp_sec = CG.timestamp_sec()
                        key = "%s:%s_%s" % (component, subcomponent, timestamp_sec)
                        history_bucket.upsert(key, change_doc)
                        total_new += curr_new
                        total_removed += curr_removed
                        total_changed += curr_changed
            else:
                # handle brand new conf
                return

        print "New: %d, Removed: %d, Changed: %d" % (
            total_new,
            total_removed,
            total_changed)