class TestProjectTestsWorkflow(Base):
    """ Functional tests to verify the configuration of a project test
    """
    @classmethod
    def setUpClass(cls):
        cls.msu = ManageSfUtils(config.GATEWAY_URL)
        cls.sample_project_dir = \
            os.path.join(config.SF_TESTS_DIR, "sample_project/")

    @classmethod
    def tearDownClass(cls):
        pass

    def setUp(self):
        self.projects = []
        self.dirs_to_delete = []
        self.un = config.ADMIN_USER
        self.gu = GerritUtils(
            config.GATEWAY_URL,
            auth_cookie=config.USERS[self.un]['auth_cookie'])
        self.gu2 = GerritUtils(
            config.GATEWAY_URL,
            auth_cookie=config.USERS[config.USER_2]['auth_cookie'])
        self.ju = JenkinsUtils()
        self.gu.add_pubkey(config.USERS[self.un]["pubkey"])
        priv_key_path = set_private_key(config.USERS[self.un]["privkey"])
        self.gitu_admin = GerritGitUtils(self.un,
                                         priv_key_path,
                                         config.USERS[self.un]['email'])
        # Clone the config repo and make change to it
        # in order to test the new sample_project
        self.config_clone_dir = self.clone_as_admin("config")
        self.original_layout = file(os.path.join(
            self.config_clone_dir, "zuul/layout.yaml")).read()
        self.original_zuul_projects = file(os.path.join(
            self.config_clone_dir, "zuul/projects.yaml")).read()
        self.original_project = file(os.path.join(
            self.config_clone_dir, "jobs/projects.yaml")).read()
        # Put USER_2 as core for config project
        self.gu.add_group_member(config.USER_2, "config-core")

    def tearDown(self):
        self.restore_config_repo(self.original_layout,
                                 self.original_project,
                                 self.original_zuul_projects)
        for name in self.projects:
            self.msu.deleteProject(name,
                                   config.ADMIN_USER)
        for dirs in self.dirs_to_delete:
            shutil.rmtree(dirs)

    def assert_reviewer_approvals(self, change_id, value):
        approvals = {}
        for _ in range(90):
            approvals = self.gu.get_reviewer_approvals(change_id,
                                                       'jenkins')
            if approvals and approvals.get('Verified') == value:
                break
            time.sleep(1)
        self.assertEqual(value, approvals.get('Verified'))

    def clone_as_admin(self, pname):
        url = "ssh://%s@%s:29418/%s" % (self.un, config.GATEWAY_HOST,
                                        pname)
        clone_dir = self.gitu_admin.clone(url, pname)
        if os.path.dirname(clone_dir) not in self.dirs_to_delete:
            self.dirs_to_delete.append(os.path.dirname(clone_dir))
        return clone_dir

    def restore_config_repo(self, layout, project, zuul):
        file(os.path.join(
            self.config_clone_dir, "zuul/layout.yaml"), 'w').write(
            layout)
        file(os.path.join(
            self.config_clone_dir, "zuul/projects.yaml"), 'w').write(
            zuul)
        file(os.path.join(
            self.config_clone_dir, "jobs/projects.yaml"), 'w').write(
            project)
        self.commit_direct_push_as_admin(
            self.config_clone_dir,
            "Restore layout.yaml and projects.yaml")

    def commit_direct_push_as_admin(self, clone_dir, msg):
        # Stage, commit and direct push the additions on master
        self.gitu_admin.add_commit_for_all_new_additions(clone_dir, msg)
        self.gitu_admin.direct_push_branch(clone_dir, 'master')

    def push_review_as_admin(self, clone_dir, msg):
        # Stage, commit and direct push the additions on master
        self.gitu_admin.add_commit_for_all_new_additions(clone_dir, msg)
        self.gitu_admin.review_push_branch(clone_dir, 'master')

    def create_project(self, name, user,
                       options=None):
        self.msu.createProject(name, user,
                               options)
        self.projects.append(name)

    def test_check_project_test_workflow(self):
        """ Validate new project to test via zuul layout.yaml
        """
        # We want to create a project, provide project source
        # code with tests. We then configure zuul/jjb to handle the
        # run of the test cases. We then validate Gerrit has been
        # updated about the test results
        # We use the sample-project (that already exists)

        pname = 'test_workflow_%s' % create_random_str()
        # Be sure the project does not exist
        self.msu.deleteProject(pname,
                               config.ADMIN_USER)
        # Create it
        self.create_project(pname, config.ADMIN_USER)

        # Add the sample-project to the empty repository
        clone_dir = self.clone_as_admin(pname)
        copytree(self.sample_project_dir, clone_dir)
        self.commit_direct_push_as_admin(clone_dir, "Add the sample project")

        # Change to config/zuul/layout.yaml and jobs/projects.yaml
        # in order to test the new project
        ycontent = file(os.path.join(
            self.config_clone_dir, "zuul/projects.yaml")).read()
        file(os.path.join(
            self.config_clone_dir, "zuul/projects.yaml"), 'w').write(
            ycontent.replace("zuul-demo", pname),
        )
        ycontent2 = load(file(os.path.join(
            self.config_clone_dir, "jobs/projects.yaml")).read())
        sp2 = copy.deepcopy(
            [p for p in ycontent2 if 'project' in p and
                p['project']['name'] == 'zuul-demo'][0])
        sp2['project']['name'] = pname
        ycontent2.append(sp2)
        file(os.path.join(
            self.config_clone_dir, "jobs/projects.yaml"), 'w').write(
            dump(ycontent2))

        # Retrieve the previous build number for config-check
        last_success_build_num_ch = \
            self.ju.get_last_build_number("config-check",
                                          "lastSuccessfulBuild")
        # Retrieve the previous build number for config-update
        last_success_build_num_cu = \
            self.ju.get_last_build_number("config-update",
                                          "lastSuccessfulBuild")

        # Send review (config-check) will be triggered
        self.push_review_as_admin(
            self.config_clone_dir,
            "Add config definition in Zuul/JJB config for %s" % pname)

        # Wait for config-check to finish and verify the success
        self.ju.wait_till_job_completes("config-check",
                                        last_success_build_num_ch,
                                        "lastSuccessfulBuild")

        last_build_num_ch, last_success_build_num_ch = 0, 1
        attempt = 0
        while last_build_num_ch != last_success_build_num_ch:
            if attempt >= 90:
                break
            time.sleep(1)
            last_build_num_ch = \
                self.ju.get_last_build_number("config-check",
                                              "lastBuild")
            last_success_build_num_ch = \
                self.ju.get_last_build_number("config-check",
                                              "lastSuccessfulBuild")
            attempt += 1

        self.assertEqual(last_build_num_ch, last_success_build_num_ch)
        # let some time to Zuul to update the test result to Gerrit.
        time.sleep(2)

        # Get the change id
        change_ids = self.gu.get_my_changes_for_project("config")
        self.assertGreater(len(change_ids), 0)
        change_id = change_ids[0]

        # Check whether zuul sets verified to +1 after running the tests
        # let some time to Zuul to update the test result to Gerrit.
        self.assert_reviewer_approvals(change_id, '+1')

        # review the change
        self.gu2.submit_change_note(change_id, "current", "Code-Review", "2")
        self.gu2.submit_change_note(change_id, "current", "Workflow", "1")

        # now zuul processes gate pipeline and runs config-check job
        # Wait for config-check to finish and verify the success
        self.ju.wait_till_job_completes("config-check",
                                        last_success_build_num_ch,
                                        "lastSuccessfulBuild")

        last_build_num_ch, last_success_build_num_ch = 0, 1
        attempt = 0
        while last_build_num_ch != last_success_build_num_ch:
            if attempt >= 90:
                break
            time.sleep(1)
            last_build_num_ch = \
                self.ju.get_last_build_number("config-check",
                                              "lastBuild")
            last_success_build_num_ch = \
                self.ju.get_last_build_number("config-check",
                                              "lastSuccessfulBuild")
            attempt += 1

        self.assertEqual(last_build_num_ch, last_success_build_num_ch)

        # Check whether zuul sets verified to +2 after running the tests
        # let some time to Zuul to update the test result to Gerrit.
        self.assert_reviewer_approvals(change_id, '+2')

        # verify whether zuul merged the patch
        change = self.gu.get_change('config', 'master', change_id)
        change_status = change['status']
        attempt = 0
        while change_status != 'MERGED':
            if attempt >= 90:
                break
            time.sleep(1)
            change = self.gu.get_change('config', 'master', change_id)
            change_status = change['status']
            attempt += 1
        self.assertEqual(change_status, 'MERGED')

        # Test post pipe line
        # as the patch is merged, post pieline should run config-update job
        # Wait for config-update to finish and verify the success
        self.ju.wait_till_job_completes("config-update",
                                        last_success_build_num_cu,
                                        "lastSuccessfulBuild")
        last_build_num_cu = \
            self.ju.get_last_build_number("config-update",
                                          "lastBuild")
        last_success_build_num_cu = \
            self.ju.get_last_build_number("config-update",
                                          "lastSuccessfulBuild")
        self.assertEqual(last_build_num_cu, last_success_build_num_cu)

        # Retrieve the prev build number for pname-unit-tests
        # Retrieve the prev build number for pname-functional-tests
        last_success_build_num_sp_ut = \
            self.ju.get_last_build_number("%s-unit-tests" % pname,
                                          "lastSuccessfulBuild")
        last_success_build_num_sp_ft = \
            self.ju.get_last_build_number("%s-functional-tests" % pname,
                                          "lastSuccessfulBuild")
        # Test config-update
        # config-update should have created jobs for pname
        # Trigger tests on pname
        # Send a review and check tests has been run
        self.gitu_admin.add_commit_and_publish(
            clone_dir, 'master', "Add useless file",
            self.un)
        # Wait for pname-unit-tests to finish and verify the success
        self.ju.wait_till_job_completes("%s-unit-tests" % pname,
                                        last_success_build_num_sp_ut,
                                        "lastSuccessfulBuild")
        # Wait for pname-functional-tests to end and check the success
        self.ju.wait_till_job_completes("%s-functional-tests" % pname,
                                        last_success_build_num_sp_ft,
                                        "lastSuccessfulBuild")
        # Check the unit tests succeed
        last_build_num_sp_ut = \
            self.ju.get_last_build_number("%s-unit-tests" % pname,
                                          "lastBuild")
        last_success_build_num_sp_ut = \
            self.ju.get_last_build_number("%s-unit-tests" % pname,
                                          "lastSuccessfulBuild")
        self.assertEqual(last_build_num_sp_ut, last_success_build_num_sp_ut)
        # Check the functional tests succeed
        last_build_num_sp_ft = \
            self.ju.get_last_build_number("%s-functional-tests" % pname,
                                          "lastBuild")
        last_success_build_num_sp_ft = \
            self.ju.get_last_build_number("%s-functional-tests" % pname,
                                          "lastSuccessfulBuild")
        self.assertEqual(last_build_num_sp_ft, last_success_build_num_sp_ft)

        # Get the change id
        change_ids = self.gu.get_my_changes_for_project(pname)
        self.assertGreater(len(change_ids), 0)
        change_id = change_ids[0]

        # let some time to Zuul to update the test result to Gerrit.
        for i in range(90):
            if "jenkins" in self.gu.get_reviewers(change_id):
                break
            time.sleep(1)

        self.assert_reviewer_approvals(change_id, '+1')
Beispiel #2
0
    def test_check_zuul_operations(self):
        """ Test if zuul verifies project correctly through zuul-demo project
        """
        # zuul-demo - test project used exclusively to test zuul installation
        # The necessary project descriptions are already declared in Jenkins
        # and zuul
        pname = 'zuul-demo'
        self.create_project(pname, config.ADMIN_USER)
        un = config.ADMIN_USER
        gu = GerritUtils(
            config.GATEWAY_URL,
            auth_cookie=config.USERS[un]['auth_cookie'])
        ju = JenkinsUtils()
        k_index = gu.add_pubkey(config.USERS[un]["pubkey"])
        # Gerrit part
        self.assertTrue(gu.project_exists(pname))
        priv_key_path = set_private_key(config.USERS[un]["privkey"])
        gitu = GerritGitUtils(un,
                              priv_key_path,
                              config.USERS[un]['email'])
        url = "ssh://%s@%s:29418/%s" % (un, config.GATEWAY_HOST,
                                        pname)
        clone_dir = gitu.clone(url, pname)
        self.dirs_to_delete.append(os.path.dirname(clone_dir))

        last_fail_build_num_ft = \
            ju.get_last_build_number("zuul-demo-functional-tests",
                                     "lastFailedBuild")
        last_fail_build_num_ut = \
            ju.get_last_build_number("zuul-demo-unit-tests",
                                     "lastFailedBuild")
        last_succeed_build_num_ft = \
            ju.get_last_build_number("zuul-demo-functional-tests",
                                     "lastSuccessfulBuild")
        last_succeed_build_num_ut = \
            ju.get_last_build_number("zuul-demo-unit-tests",
                                     "lastSuccessfulBuild")

        gitu.add_commit_and_publish(clone_dir, "master", "Test commit")

        change_ids = gu.get_my_changes_for_project(pname)
        self.assertEqual(len(change_ids), 1)
        change_id = change_ids[0]

        # Give some time for jenkins to work
        ju.wait_till_job_completes("zuul-demo-functional-tests",
                                   last_fail_build_num_ft, "lastFailedBuild")
        ju.wait_till_job_completes("zuul-demo-unit-tests",
                                   last_fail_build_num_ut, "lastFailedBuild")

        attempt = 0
        while "jenkins" not in gu.get_reviewers(change_id):
            if attempt >= 90:
                break
            time.sleep(1)
            attempt += 1

        attempt = 0
        while gu.get_reviewer_approvals(change_id, 'jenkins')['Verified'] \
                != '-1':
            if attempt >= 90:
                break
            time.sleep(1)
            attempt += 1

        self.assertEqual(
            gu.get_reviewer_approvals(change_id, 'jenkins')['Verified'], '-1')

        # Add the test case files and resubmit for review
        data = "echo Working"
        files = ["run_functional-tests.sh", "run_tests.sh"]

        for f in files:
            file(os.path.join(clone_dir, f), 'w').write(data)
            os.chmod(os.path.join(clone_dir, f), 0755)

        gitu.add_commit_and_publish(clone_dir, "master", None, fnames=files)

        # Give some time for jenkins to work
        ju.wait_till_job_completes("zuul-demo-functional-tests",
                                   last_succeed_build_num_ft,
                                   "lastSuccessfulBuild")
        ju.wait_till_job_completes("zuul-demo-unit-tests",
                                   last_succeed_build_num_ut,
                                   "lastSuccessfulBuild")

        attempt = 0
        while "jenkins" not in gu.get_reviewers(change_id):
            if attempt >= 90:
                break
            time.sleep(1)
            attempt += 1

        attempt = 0
        while gu.get_reviewer_approvals(change_id, 'jenkins')['Verified'] \
                != '+1':
            if attempt >= 90:
                break
            time.sleep(1)
            attempt += 1

        self.assertEqual(
            gu.get_reviewer_approvals(change_id, 'jenkins')['Verified'], '+1')

        gu.del_pubkey(k_index)
    def test_check_zuul_operations(self):
        """ Test if zuul verifies project correctly through zuul-demo project
        """
        # zuul-demo - test project used exclusively to test zuul installation
        # The necessary project descriptions are already declared in Jenkins
        # and zuul

        pname = 'demo/zuul-demo'

        self.create_project(pname, config.ADMIN_USER)
        un = config.ADMIN_USER
        gu = GerritUtils(
            config.GATEWAY_URL,
            auth_cookie=config.USERS[un]['auth_cookie'])
        ju = JenkinsUtils()
        k_index = gu.add_pubkey(config.USERS[un]["pubkey"])
        # Gerrit part
        self.assertTrue(gu.project_exists(pname))
        priv_key_path = set_private_key(config.USERS[un]["privkey"])
        gitu = GerritGitUtils(un,
                              priv_key_path,
                              config.USERS[un]['email'])
        url = "ssh://%s@%s:29418/%s" % (un, config.GATEWAY_HOST,
                                        pname)
        clone_dir = gitu.clone(url, pname)
        self.dirs_to_delete.append(os.path.dirname(clone_dir))

        last_fail_build_num_ft = \
            ju.get_last_build_number("zuul-demo-functional-tests",
                                     "lastFailedBuild")
        last_fail_build_num_ut = \
            ju.get_last_build_number("zuul-demo-unit-tests",
                                     "lastFailedBuild")
        last_succeed_build_num_ft = \
            ju.get_last_build_number("zuul-demo-functional-tests",
                                     "lastSuccessfulBuild")
        last_succeed_build_num_ut = \
            ju.get_last_build_number("zuul-demo-unit-tests",
                                     "lastSuccessfulBuild")

        gitu.add_commit_and_publish(clone_dir, "master", "Test commit")

        change_ids = gu.get_my_changes_for_project(pname)
        self.assertGreater(len(change_ids), 0)
        change_id = change_ids[0]

        # Give some time for jenkins to work
        ju.wait_till_job_completes("zuul-demo-functional-tests",
                                   last_fail_build_num_ft, "lastFailedBuild")
        ju.wait_till_job_completes("zuul-demo-unit-tests",
                                   last_fail_build_num_ut, "lastFailedBuild")

        attempt = 0
        while "jenkins" not in gu.get_reviewers(change_id):
            if attempt >= 90:
                break
            time.sleep(1)
            attempt += 1

        attempt = 0
        while gu.get_reviewer_approvals(change_id, 'jenkins')['Verified'] \
                != '-1':
            if attempt >= 90:
                break
            time.sleep(1)
            attempt += 1

        self.assertEqual(
            gu.get_reviewer_approvals(change_id, 'jenkins')['Verified'], '-1')

        # Add the test case files and resubmit for review
        data = "echo Working"
        files = ["run_functional-tests.sh", "run_tests.sh"]

        for f in files:
            file(os.path.join(clone_dir, f), 'w').write(data)
            os.chmod(os.path.join(clone_dir, f), 0755)

        gitu.add_commit_and_publish(clone_dir, "master", None, fnames=files)

        # Give some time for jenkins to work
        ju.wait_till_job_completes("zuul-demo-functional-tests",
                                   last_succeed_build_num_ft,
                                   "lastSuccessfulBuild")
        ju.wait_till_job_completes("zuul-demo-unit-tests",
                                   last_succeed_build_num_ut,
                                   "lastSuccessfulBuild")

        attempt = 0
        while "jenkins" not in gu.get_reviewers(change_id):
            if attempt >= 90:
                break
            time.sleep(1)
            attempt += 1

        attempt = 0
        while gu.get_reviewer_approvals(change_id, 'jenkins')['Verified'] \
                != '+1':
            if attempt >= 90:
                break
            time.sleep(1)
            attempt += 1

        self.assertEqual(
            gu.get_reviewer_approvals(change_id, 'jenkins')['Verified'], '+1')

        gu.del_pubkey(k_index)
Beispiel #4
0
class TestProjectTestsWorkflow(Base):
    """ Functional tests to verify the configuration of a project test
    """
    @classmethod
    def setUpClass(cls):
        cls.ru = ResourcesUtils()
        cls.sample_project_dir = \
            os.path.join(config.SF_TESTS_DIR, "sample_project/")

    @classmethod
    def tearDownClass(cls):
        pass

    def setUp(self):
        super(TestProjectTestsWorkflow, self).setUp()
        self.projects = []
        self.dirs_to_delete = []
        self.un = config.ADMIN_USER
        self.gu = GerritUtils(
            config.GATEWAY_URL,
            auth_cookie=config.USERS[self.un]['auth_cookie'])
        self.gu2 = GerritUtils(
            config.GATEWAY_URL,
            auth_cookie=config.USERS[config.USER_2]['auth_cookie'])
        self.ju = JenkinsUtils()
        self.gu.add_pubkey(config.USERS[self.un]["pubkey"])
        priv_key_path = set_private_key(config.USERS[self.un]["privkey"])
        self.gitu_admin = GerritGitUtils(self.un,
                                         priv_key_path,
                                         config.USERS[self.un]['email'])
        # Clone the config repo and keep job/zuul config content
        self.config_clone_dir = self.clone_as_admin("config")
        self.original_zuul_projects = file(os.path.join(
            self.config_clone_dir, "zuul/projects.yaml")).read()
        self.original_project = file(os.path.join(
            self.config_clone_dir, "jobs/projects.yaml")).read()
        self.need_restore_config_repo = False
        # Put USER_2 as core for config project
        self.gu.add_group_member(config.USER_2, "config-core")

    def tearDown(self):
        super(TestProjectTestsWorkflow, self).tearDown()
        if self.need_restore_config_repo:
            self.restore_config_repo(self.original_project,
                                     self.original_zuul_projects)
        for name in self.projects:
            self.ru.direct_delete_repo(name)
        for dirs in self.dirs_to_delete:
            shutil.rmtree(dirs)

    def assert_reviewer_approvals(self, change_id, value):
        approvals = {}
        for _ in range(300):
            approvals = self.gu.get_reviewer_approvals(change_id,
                                                       'jenkins')
            if approvals and approvals.get('Verified') == value:
                break
            time.sleep(1)
        self.assertEqual(value, approvals.get('Verified'))

    def clone_as_admin(self, pname):
        url = "ssh://%s@%s:29418/%s" % (self.un, config.GATEWAY_HOST,
                                        pname)
        clone_dir = self.gitu_admin.clone(url, pname)
        if os.path.dirname(clone_dir) not in self.dirs_to_delete:
            self.dirs_to_delete.append(os.path.dirname(clone_dir))
        return clone_dir

    def restore_config_repo(self, project, zuul):
        logger.info("Restore {zuul,jobs}/projects.yaml")
        file(os.path.join(
            self.config_clone_dir, "zuul/projects.yaml"), 'w').write(
            zuul)
        file(os.path.join(
            self.config_clone_dir, "jobs/projects.yaml"), 'w').write(
            project)
        change_sha = self.commit_direct_push_as_admin(
            self.config_clone_dir,
            "Restore {zuul,jobs}/projects.yaml")
        logger.info("Waiting for config-update on %s" % change_sha)
        self.ju.wait_for_config_update(change_sha)

    def commit_direct_push_as_admin(self, clone_dir, msg):
        # Stage, commit and direct push the additions on master
        self.gitu_admin.add_commit_for_all_new_additions(clone_dir, msg)
        return self.gitu_admin.direct_push_branch(clone_dir, 'master')

    def push_review_as_admin(self, clone_dir, msg):
        # Stage, commit and direct push the additions on master
        self.gitu_admin.add_commit_for_all_new_additions(clone_dir, msg)
        return self.gitu_admin.review_push_branch(clone_dir, 'master')

    def create_project(self, name):
        self.ru.direct_create_repo(name)
        self.projects.append(name)

    def test_timestamped_logs(self):
        """Test that jenkins timestamps logs"""
        # Done here to make sure a config-update job was run and to avoid
        # duplicating code
        timestamp_re = re.compile('\d{2}:\d{2}:\d{2}.\d{0,3}')
        n = self.ju.get_last_build_number("config-update",
                                          "lastBuild")
        cu_logs = self.ju.get_job_logs("config-update",
                                       n)
        self.assertTrue(cu_logs is not None)
        for l in cu_logs.split('\n'):
            if l:
                self.assertRegexpMatches(l, timestamp_re, msg=l)

    def test_check_project_test_workflow(self):
        """ Validate new project to test via zuul
        """
        # We want to create a project, provide project source
        # code with tests. We then configure zuul/jjb to handle the
        # run of the test cases. We then validate Gerrit has been
        # updated about the test results
        # We use the sample-project (that already exists)

        pname = 'test_workflow_%s' % create_random_str()
        logger.info("Creating project %s" % pname)

        # Create it
        self.create_project(pname)

        logger.info("Populating the project with %s" %
                    self.sample_project_dir)
        # Add the sample-project to the empty repository
        clone_dir = self.clone_as_admin(pname)
        copytree(self.sample_project_dir, clone_dir)
        self.commit_direct_push_as_admin(clone_dir, "Add the sample project")

        # Change to config/{zuul,jobs}/projects.yaml
        # in order to test the new project
        logger.info("Adding config-repo configuration")
        ycontent = file(os.path.join(
            self.config_clone_dir, "zuul/projects.yaml")).read()
        file(os.path.join(
            self.config_clone_dir, "zuul/projects.yaml"), 'w').write(
            ycontent.replace("zuul-demo", pname),
        )
        ycontent2 = load(file(os.path.join(
            self.config_clone_dir, "jobs/projects.yaml")).read())
        sp2 = copy.deepcopy(
            [p for p in ycontent2 if 'project' in p and
                p['project']['name'] == 'zuul-demo'][0])
        sp2['project']['name'] = pname
        ycontent2.append(sp2)
        file(os.path.join(
            self.config_clone_dir, "jobs/projects.yaml"), 'w').write(
            dump(ycontent2))

        # Send review (config-check) will be triggered
        logger.info("Submitting the config review")
        change_sha = self.push_review_as_admin(
            self.config_clone_dir,
            "Add config definition in Zuul/JJB config for %s" % pname)

        change_nr = self.gu.get_change_number(change_sha)

        logger.info("Waiting for verify +1 on change %d" % change_nr)
        self.assertEquals(self.gu.wait_for_verify(change_nr), 1)

        # review the config change as a member from the config-core group
        logger.info("Approving and waiting for verify +2")
        self.gu2.submit_change_note(change_nr, "current", "Code-Review", "2")
        self.gu2.submit_change_note(change_nr, "current", "Workflow", "1")

        for retry in xrange(60):
            jenkins_vote = self.gu.get_vote(change_nr, "Verified")
            if jenkins_vote == 2:
                break
            time.sleep(1)
        self.assertEquals(jenkins_vote, 2)

        # verify whether zuul merged the patch
        logger.info("Waiting for change to be merged")
        for retry in xrange(60):
            change_status = self.gu.get_info(change_nr)['status']
            if change_status == "MERGED":
                break
            time.sleep(1)
        self.assertEqual(change_status, 'MERGED')
        self.need_restore_config_repo = True

        logger.info("Waiting for config-update")
        config_update_log = self.ju.wait_for_config_update(change_sha)
        self.assertIn("Finished: SUCCESS", config_update_log)

        # Propose a change on a the repo and expect a Verified +1
        logger.info("Submiting a test change to %s" % pname)
        change_sha = self.gitu_admin.add_commit_and_publish(
            clone_dir, 'master', "Add useless file",
            self.un)

        change_nr = self.gu.get_change_number(change_sha)

        logger.info("Waiting for verify +1 on change %d" % change_nr)
        self.assertEquals(self.gu.wait_for_verify(change_nr), 1)

        # Update the change on a the repo and expect a Verified -1
        logger.info("Submiting a test change to %s suppose to fail" % pname)
        data = "#!/bin/bash\nexit 1\n"
        file(os.path.join(clone_dir, "run_tests.sh"), 'w').write(data)
        os.chmod(os.path.join(clone_dir, "run_tests.sh"), 0755)
        self.gitu_admin.add_commit_and_publish(
            clone_dir, "master", None, fnames=["run_tests.sh"])

        logger.info("Waiting for verify -1 on change %d" % change_nr)
        self.assertEquals(self.gu.wait_for_verify(change_nr), -1)

        logger.info("Validate jobs ran via the job api %s" % pname)
        # This piece of code is there by convenience ...
        # TODO: Should be moved in the job api tests file.
        # Test the manageSF jobs API: query per patch & revision
        change_ids = self.gu.get_my_changes_for_project(pname)
        self.assertGreater(len(change_ids), 0)
        change_id = change_ids[0]
        patch = self.gu.get_change_last_patchset(change_id)['_number']
        cookie = get_cookie(config.ADMIN_USER, config.ADMIN_PASSWORD)
        cookies = {"auth_pubtkt": cookie}
        base_url = config.GATEWAY_URL + "/manage/jobs/"
        for j in ["%s-functional-tests" % pname, "%s-unit-tests" % pname]:
            job = requests.get(base_url + '%s/?change=%s' % (j, patch),
                               cookies=cookies).json()
            self.assertTrue("jenkins" in job.keys(),
                            job)
            self.assertTrue(len(job["jenkins"]) > 1,
                            job)