def setUp(self): super(TestManageSF, self).setUp() self.projects = [] self.dirs_to_delete = [] self.ru = ResourcesUtils() self.msu = ManageSfUtils(config.GATEWAY_URL) self.gu = GerritUtils( config.GATEWAY_URL, auth_cookie=config.USERS[config.ADMIN_USER]['auth_cookie'])
def __init__(self): with open("%s/resources.yaml" % pwd, 'r') as rsc: self.resources = yaml.load(rsc) config.USERS[config.ADMIN_USER]['auth_cookie'] = get_cookie( config.ADMIN_USER, config.USERS[config.ADMIN_USER]['password']) self.msu = ManageSfUtils(config.GATEWAY_URL) self.ru = ResourcesUtils() self.ggu = GerritGitUtils(config.ADMIN_USER, config.ADMIN_PRIV_KEY_PATH, config.USERS[config.ADMIN_USER]['email']) self.stb_client = SFStoryboard( config.GATEWAY_URL + "/storyboard_api", config.USERS[config.ADMIN_USER]['auth_cookie'])
def setUp(self): super(TestProjectReplication, self).setUp() self.ru = ResourcesUtils() self.un = config.ADMIN_USER self.ju = JenkinsUtils() self.gu = GerritUtils(config.GATEWAY_URL, auth_cookie=config.USERS[self.un]['auth_cookie']) self.gu2 = GerritUtils( config.GATEWAY_URL, auth_cookie=config.USERS[config.USER_2]['auth_cookie']) self.k_idx = self.gu2.add_pubkey(config.USERS[config.USER_2]["pubkey"]) priv_key_path = set_private_key(config.USERS[self.un]["privkey"]) self.gitu_admin = GerritGitUtils(self.un, priv_key_path, config.USERS[self.un]['email']) # Prepare environment for git clone on mirror repo self.mt = Tool() self.mt_tempdir = tempfile.mkdtemp() # Copy the service private key in a flat file priv_key = file(config.SERVICE_PRIV_KEY_PATH, 'r').read() priv_key_path = os.path.join(self.mt_tempdir, 'user.priv') file(priv_key_path, 'w').write(priv_key) os.chmod(priv_key_path, stat.S_IREAD | stat.S_IWRITE) # Prepare the ssh wrapper script ssh_wrapper = "ssh -o StrictHostKeyChecking=no -i %s \"$@\"" % ( priv_key_path) wrapper_path = os.path.join(self.mt_tempdir, 'ssh_wrapper.sh') file(wrapper_path, 'w').write(ssh_wrapper) os.chmod(wrapper_path, stat.S_IRWXU) # Set the wrapper as GIT_SSH env variable self.mt.env['GIT_SSH'] = wrapper_path self.config_clone_dir = None # Project we are going to configure the replication for self.pname = 'test/replication' # Remove artifacts of previous run if any self.delete_config_section(self.un, self.pname) self.delete_mirror_repo(self.pname)
class TestManageSF(Base): """ Functional tests that validate managesf features. """ def setUp(self): super(TestManageSF, self).setUp() self.projects = [] self.dirs_to_delete = [] self.ru = ResourcesUtils() self.msu = ManageSfUtils(config.GATEWAY_URL) self.gu = GerritUtils( config.GATEWAY_URL, auth_cookie=config.USERS[config.ADMIN_USER]['auth_cookie']) def tearDown(self): super(TestManageSF, self).tearDown() for name in self.projects: self.ru.direct_delete_repo(name) for dirs in self.dirs_to_delete: shutil.rmtree(dirs) def create_project(self, name): self.ru.direct_create_repo(name) self.projects.append(name) def test_api_key_auth_with_sfmanager(self): """Test the api key auth workflow""" user2_cookies = dict( auth_pubtkt=config.USERS[config.USER_2]['auth_cookie']) url = "https://%s%s" % (config.GATEWAY_HOST, "/auth/apikey/") create_key = requests.post(url, cookies=user2_cookies) self.assertIn(create_key.status_code, (201, 409)) key = requests.get(url, cookies=user2_cookies).json().get('api_key') # call a simple command that needs authentication cmd = "sfmanager --url %s --auth-server-url " \ "%s --api-key %s sf_user list" % (config.GATEWAY_URL, config.GATEWAY_URL, key) users = self.msu.exe(cmd) self.assertTrue(config.USER_2 in users, "'%s' returned %s" % (cmd, users))
class SFProvisioner(object): """ This provider is only intended for testing SF backup/restore and update. It provisions some user datas in a SF installation based on a resourses.yaml file. Later those data can be checked by its friend the SFChecker. Provisioned data should remain really simple. """ def __init__(self): with open("%s/resources.yaml" % pwd, 'r') as rsc: self.resources = yaml.load(rsc) config.USERS[config.ADMIN_USER]['auth_cookie'] = get_cookie( config.ADMIN_USER, config.USERS[config.ADMIN_USER]['password']) self.msu = ManageSfUtils(config.GATEWAY_URL) self.ru = ResourcesUtils() self.ggu = GerritGitUtils(config.ADMIN_USER, config.ADMIN_PRIV_KEY_PATH, config.USERS[config.ADMIN_USER]['email']) self.stb_client = SFStoryboard( config.GATEWAY_URL + "/storyboard_api", config.USERS[config.ADMIN_USER]['auth_cookie']) def create_resources(self): print " Creating resources ..." if cmp_version(os.environ.get("PROVISIONED_VERSION", "0.0"), "2.4.0"): # Remove review-dashboard for p in self.resources['resources']['projects'].values(): del p['review-dashboard'] self.ru.create_resources("provisioner", {'resources': self.resources['resources']}) # Create review for the first few repositories for project in self.resources['resources']['repos'].keys()[:3]: self.clone_project(project) self.create_review(project, "Test review for %s" % project) def create_project(self, name): print " Creating project %s ..." % name self.ru.create_repo(name) def clone_project(self, name): # TODO(fbo); use gateway host instead of gerrit host self.url = "ssh://%s@%s:29418/%s" % (config.ADMIN_USER, config.GATEWAY_HOST, name) self.clone_dir = self.ggu.clone(self.url, name, config_review=False) def push_files_in_project(self, name, files): print " Add files(%s) in a commit ..." % ",".join(files) self.clone_project(name) for f in files: file(os.path.join(self.clone_dir, f), 'w').write('data') self.ggu.git_add(self.clone_dir, (f, )) self.ggu.add_commit_for_all_new_additions(self.clone_dir) self.ggu.direct_push_branch(self.clone_dir, 'master') def create_storyboard_issue(self, name, issue_name): project = self.stb_client.projects.get(name) story = self.stb_client.stories.create(title=issue_name) task = self.stb_client.tasks.create(story_id=story.id, project_id=project.id, title=issue_name) return task.id, story.id def create_issues_on_project(self, name, issues): print " Create %s issue(s) for that project ..." % len(issues) for i in issues: if is_present('storyboard'): issue = self.create_storyboard_issue(name, i['name']) else: issue = (random.randint(1, 100), random.randint(1, 100)) yield issue, i['review'] def create_pads(self, amount): # TODO pass def create_pasties(self, amount): # TODO pass def simple_login(self, user, password): """log as user to make the user listable""" get_cookie(user, password) def create_review(self, project, commit_message, branch='master'): """Very basic review creator for statistics and restore tests purposes.""" self.ggu.config_review(self.clone_dir) self.ggu.add_commit_in_branch(self.clone_dir, branch, commit=commit_message) self.ggu.review_push_branch(self.clone_dir, branch) def create_review_for_issue(self, project, issue): self.create_review( project, 'test\n\nTask: #%s\nStory: #%s' % (issue[0], issue[1]), 'branch_%s' % str(issue[0])) def create_local_user(self, username, password, email): self.msu.create_user(username, password, email) def command(self, cmd): return ssh_run_cmd(os.path.expanduser("~/.ssh/id_rsa"), "root", config.GATEWAY_HOST, shlex.split(cmd)) def compute_checksum(self, f): out = self.command("md5sum %s" % f)[0] if out: return out.split()[0] def read_file(self, f): return self.command("cat %s" % f)[0] def provision(self): for cmd in self.resources['commands']: print "Execute command %s" % cmd['cmd'] print self.command(cmd['cmd']) checksum_list = {} for checksum in self.resources['checksum']: print "Compute checksum for file %s" % checksum['file'] checksum_list[checksum['file']] = self.compute_checksum( checksum['file']) checksum_list['content_' + checksum['file']] = self.read_file( checksum['file']) yaml.dump(checksum_list, file('pc_checksums.yaml', 'w'), default_flow_style=False) for user in self.resources['local_users']: print "Create local user %s" % user['username'] self.create_local_user(user['username'], user['password'], user['email']) self.simple_login(user['username'], user['password']) for u in self.resources['users']: print "log in as %s" % u['name'] self.simple_login(u['name'], config.USERS[u['name']]['password']) for project in self.resources['projects']: print "Create user datas for %s" % project['name'] self.create_project(project['name']) self.push_files_in_project(project['name'], [f['name'] for f in project['files']]) for i, review in self.create_issues_on_project( project['name'], project['issues']): if review: print "Create review for bug %s in %s" % (i, project['name']) self.create_review_for_issue(project['name'], i) self.create_resources() self.create_pads(2) self.create_pasties(2)
class TestProjectReplication(Base): """ Functional tests to verify the gerrit replication feature """ def setUp(self): super(TestProjectReplication, self).setUp() self.ru = ResourcesUtils() self.un = config.ADMIN_USER self.ju = JenkinsUtils() self.gu = GerritUtils(config.GATEWAY_URL, auth_cookie=config.USERS[self.un]['auth_cookie']) self.gu2 = GerritUtils( config.GATEWAY_URL, auth_cookie=config.USERS[config.USER_2]['auth_cookie']) self.k_idx = self.gu2.add_pubkey(config.USERS[config.USER_2]["pubkey"]) priv_key_path = set_private_key(config.USERS[self.un]["privkey"]) self.gitu_admin = GerritGitUtils(self.un, priv_key_path, config.USERS[self.un]['email']) # Prepare environment for git clone on mirror repo self.mt = Tool() self.mt_tempdir = tempfile.mkdtemp() # Copy the service private key in a flat file priv_key = file(config.SERVICE_PRIV_KEY_PATH, 'r').read() priv_key_path = os.path.join(self.mt_tempdir, 'user.priv') file(priv_key_path, 'w').write(priv_key) os.chmod(priv_key_path, stat.S_IREAD | stat.S_IWRITE) # Prepare the ssh wrapper script ssh_wrapper = "ssh -o StrictHostKeyChecking=no -i %s \"$@\"" % ( priv_key_path) wrapper_path = os.path.join(self.mt_tempdir, 'ssh_wrapper.sh') file(wrapper_path, 'w').write(ssh_wrapper) os.chmod(wrapper_path, stat.S_IRWXU) # Set the wrapper as GIT_SSH env variable self.mt.env['GIT_SSH'] = wrapper_path self.config_clone_dir = None # Project we are going to configure the replication for self.pname = 'test/replication' # Remove artifacts of previous run if any self.delete_config_section(self.un, self.pname) self.delete_mirror_repo(self.pname) def tearDown(self): super(TestProjectReplication, self).tearDown() self.delete_config_section(self.un, self.pname) self.delete_mirror_repo(self.pname) self.ru.direct_delete_repo(self.pname) self.gu2.del_pubkey(self.k_idx) def clone(self, uri, target): self.assertTrue(uri.startswith('ssh://')) cmd = "git clone %s %s" % (uri, target) clone = os.path.join(self.mt_tempdir, target) if os.path.isdir(clone): shutil.rmtree(clone) self.mt.exe(cmd, self.mt_tempdir) return clone def create_project(self, name): logger.info("Create repo to for testing replication %s" % name) self.ru.direct_create_repo(name) def ssh_run_cmd(self, sshkey_priv_path, user, host, subcmd): host = '%s@%s' % (user, host) sshcmd = [ 'ssh', '-o', 'LogLevel=ERROR', '-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=/dev/null', '-i', sshkey_priv_path, host ] cmd = sshcmd + subcmd p = Popen(cmd, stdout=PIPE) return p.communicate(), p.returncode def delete_mirror_repo(self, name): logger.info("Delete mirror repo created by the replication") mirror_path = '/var/lib/gerrit/tmp/%s.git' % name cmd = [ 'ssh', 'gerrit.%s' % config.GATEWAY_HOST, 'rm', '-rf', mirror_path ] self.ssh_run_cmd(config.SERVICE_PRIV_KEY_PATH, 'root', config.GATEWAY_HOST, cmd) def create_config_section(self, project): logger.info("Add the replication config section") host = '%s@%s' % (config.GERRIT_USER, config.GATEWAY_HOST) mirror_repo_path = '/var/lib/gerrit/tmp/\${name}.git' url = '%s:%s' % (host, mirror_repo_path) path = os.path.join(self.config_clone_dir, 'gerrit/replication.config') call("git config -f %s --remove-section remote.test_project" % path, shell=True) call("git config -f %s --add remote.test_project.projects %s" % (path, project), shell=True) call("git config -f %s --add remote.test_project.url %s" % (path, url), shell=True) self.gitu_admin.add_commit_for_all_new_additions( self.config_clone_dir, "Add replication test section") # The direct push will trigger the config-update job # as we commit through 29418 change_sha = self.gitu_admin.direct_push_branch( self.config_clone_dir, 'master') logger.info("Waiting for config-update on %s" % change_sha) self.ju.wait_for_config_update(change_sha) cmd = [ 'ssh', 'gerrit.%s' % config.GATEWAY_HOST, 'grep', 'test_project', '/etc/gerrit/replication.config' ] logger.info("Wait for the replication config section to land") _, code = self.ssh_run_cmd(config.SERVICE_PRIV_KEY_PATH, 'root', config.GATEWAY_HOST, cmd) if code == 0: return raise Exception('replication.config file has not been updated (add)') def delete_config_section(self, user, project): logger.info("Remove the replication config section") url = "ssh://%s@%s:29418/config" % (self.un, config.GATEWAY_HOST) self.config_clone_dir = self.gitu_admin.clone(url, 'config', config_review=True) sha = open("%s/.git/refs/heads/master" % self.config_clone_dir).read().strip() path = os.path.join(self.config_clone_dir, 'gerrit/replication.config') call("git config -f %s --remove-section remote.test_project" % path, shell=True) change_sha = self.gitu_admin.add_commit_for_all_new_additions( self.config_clone_dir, "Remove replication test section") # The direct push will trigger the config-update job # as we commit through 29418 if change_sha == sha: # Nothing have been changed/Nothing to publish return change_sha = self.gitu_admin.direct_push_branch( self.config_clone_dir, 'master') logger.info("Waiting for config-update on %s" % change_sha) self.ju.wait_for_config_update(change_sha) cmd = [ 'ssh', 'gerrit.%s' % config.GATEWAY_HOST, 'grep', 'test_project', '/etc/gerrit/replication.config' ] _, code = self.ssh_run_cmd(config.SERVICE_PRIV_KEY_PATH, 'root', config.GATEWAY_HOST, cmd) if code != 0: return raise Exception('replication.config has not been updated (rm)') def mirror_clone_and_check_files(self, url, pname): for retry in xrange(50): clone = self.clone(url, pname) # clone may fail, as mirror repo is not yet ready(i.e gerrit not # yet replicated the project) if os.path.isdir(clone): logger.info("Files in the mirror repo: %s" % os.listdir(clone)) if os.path.isdir(clone) and \ os.path.isfile(os.path.join(clone, '.gitreview')): break else: time.sleep(3) self.assertTrue(os.path.exists(os.path.join(clone, '.gitreview'))) def test_replication(self): """ Test gerrit replication for review process """ # Create the project self.create_project(self.pname) # Be sure sftests.com host key is inside the known_hosts cmds = [[ 'ssh', 'gerrit.%s' % config.GATEWAY_HOST, 'ssh-keyscan', 'sftests.com', '>', '/var/lib/gerrit/.ssh/known_hosts' ]] for cmd in cmds: self.ssh_run_cmd(config.SERVICE_PRIV_KEY_PATH, 'root', config.GATEWAY_HOST, cmd) # Create new section for this project in replication.config self.create_config_section(self.pname) # Verify if gerrit replicated the repo self.managesf_repo_path = "ssh://%s@%s/var/lib/gerrit/tmp/" % ( 'root', config.GATEWAY_HOST) repo_url = self.managesf_repo_path + '%s.git' % self.pname logger.info("Wait for the replication to happen") self.mirror_clone_and_check_files(repo_url, self.pname)
def setUp(self): super(TestGerrit, self).setUp() self.projects = [] self.clone_dirs = [] self.dirs_to_delete = [] self.ru = ResourcesUtils()
class TestGerrit(Base): """ Functional tests that validate some gerrit behaviors. """ @classmethod def setUpClass(cls): pass @classmethod def tearDownClass(cls): pass def setUp(self): super(TestGerrit, self).setUp() self.projects = [] self.clone_dirs = [] self.dirs_to_delete = [] self.ru = ResourcesUtils() def tearDown(self): super(TestGerrit, self).tearDown() for name in self.projects: self.ru.direct_delete_repo(name) for dirs in self.dirs_to_delete: shutil.rmtree(dirs) def create_project(self, name): self.ru.direct_create_repo(name) self.projects.append(name) def _prepare_review_submit_testing(self, data=None): pname = 'p_%s' % create_random_str() self.create_project(pname) gu = GerritUtils( config.GATEWAY_URL, auth_cookie=config.USERS[config.ADMIN_USER]['auth_cookie']) k_index = gu.add_pubkey(config.USERS[config.ADMIN_USER]["pubkey"]) self.assertTrue(gu.project_exists(pname)) priv_key_path = set_private_key( config.USERS[config.ADMIN_USER]["privkey"]) gitu = GerritGitUtils(config.ADMIN_USER, priv_key_path, config.USERS[config.ADMIN_USER]['email']) url = "ssh://%s@%s:29418/%s" % (config.ADMIN_USER, config.GATEWAY_HOST, pname) clone_dir = gitu.clone(url, pname) self.dirs_to_delete.append(os.path.dirname(clone_dir)) if not data: gitu.add_commit_and_publish(clone_dir, "master", "Test commit") else: file(os.path.join(clone_dir, "file"), 'w').write(data[1]) gitu.add_commit_and_publish(clone_dir, "master", "Test commit", fnames=[data[0]]) change_ids = gu.get_my_changes_for_project(pname) self.assertEqual(len(change_ids), 1) change_id = change_ids[0] return change_id, gu, k_index, pname def test_review_labels(self): """ Test if list of review labels are as expected """ change_id, gu, k_index, _ = self._prepare_review_submit_testing() logger.info("Looking for labels for change %s" % change_id) labels = gu.get_labels_list_for_change(change_id) self.assertIn('Workflow', labels) self.assertIn('Code-Review', labels) self.assertIn('Verified', labels) self.assertEqual(len(labels.keys()), 3) gu.del_pubkey(k_index) def test_review_submit_approval(self): """ Test submit criteria - CR(+2s), V(+1), W(+1) """ change_id, gu, k_index, _ = self._prepare_review_submit_testing() gu.submit_change_note(change_id, "current", "Code-Review", "1") self.assertFalse(gu.submit_patch(change_id, "current")) gu.submit_change_note(change_id, "current", "Verified", "2") self.assertFalse(gu.submit_patch(change_id, "current")) gu.submit_change_note(change_id, "current", "Workflow", "1") self.assertFalse(gu.submit_patch(change_id, "current")) gu.submit_change_note(change_id, "current", "Code-Review", "2") self.assertTrue(gu.submit_patch(change_id, "current")) gu.del_pubkey(k_index) def test_plugins_installed(self): """ Test if plugins are present """ gu = GerritUtils( config.GATEWAY_URL, auth_cookie=config.USERS[config.ADMIN_USER]['auth_cookie']) plugins = gu.list_plugins() self.assertIn('download-commands', plugins) self.assertIn('avatars-gravatar', plugins) self.assertIn('reviewers-by-blame', plugins) def test_check_download_commands(self): """ Test if download commands plugin works """ change_id, gu, k_index, _ = self._prepare_review_submit_testing() resp = gu.get_change_last_patchset(change_id) self.assertIn("current_revision", resp) self.assertIn("revisions", resp) current_rev = resp["current_revision"] fetch = resp["revisions"][current_rev]["fetch"] self.assertGreater(fetch.keys(), 0) # disable and check if the fetch has anything gu.e_d_plugin("download-commands", 'disable') resp = gu.get_change_last_patchset(change_id) fetch = resp["revisions"][current_rev]["fetch"] self.assertEqual(len(fetch.keys()), 0) # enable the plugin and check if the fetch information is valid gu.e_d_plugin("download-commands", 'enable') resp = gu.get_change_last_patchset(change_id) fetch = resp["revisions"][current_rev]["fetch"] self.assertGreater(len(fetch.keys()), 0) gu.del_pubkey(k_index) def test_check_add_automatic_reviewers(self): """ Test if reviewers-by-blame plugin works """ data = "this\nis\na\ncouple\nof\nlines" change_id, gu, k1_index, pname = self._prepare_review_submit_testing( ('file', data)) # Merge the change gu.submit_change_note(change_id, "current", "Code-Review", "2") gu.submit_change_note(change_id, "current", "Verified", "2") gu.submit_change_note(change_id, "current", "Workflow", "1") self.assertTrue(gu.submit_patch(change_id, "current")) gu2 = GerritUtils( config.GATEWAY_URL, auth_cookie=config.USERS[config.USER_2]['auth_cookie']) # Change the file we have commited with Admin user k2_index = gu2.add_pubkey(config.USERS[config.USER_2]["pubkey"]) priv_key_path = set_private_key(config.USERS[config.USER_2]["privkey"]) gitu2 = GerritGitUtils(config.USER_2, priv_key_path, config.USERS[config.USER_2]['email']) url = "ssh://%s@%s:29418/%s" % (config.USER_2, config.GATEWAY_HOST, pname) clone_dir = gitu2.clone(url, pname) self.dirs_to_delete.append(os.path.dirname(clone_dir)) data = ['this', 'is', 'some', 'lines'] file(os.path.join(clone_dir, "file"), 'w').write("\n".join(data)) gitu2.add_commit_and_publish(clone_dir, "master", "Test commit", fnames=["file"]) # Get the change id change_ids = gu2.get_my_changes_for_project(pname) self.assertEqual(len(change_ids), 1) change_id = change_ids[0] # Verify first_u has been automatically added to reviewers for retry in xrange(3): if len(gu2.get_reviewers(change_id)) > 0: break time.sleep(1) reviewers = gu2.get_reviewers(change_id) self.assertEqual(len(reviewers), 1) self.assertEqual(reviewers[0], config.ADMIN_USER) gu.del_pubkey(k1_index) gu2.del_pubkey(k2_index) def test_gerrit_version(self): """ Test if correct Gerrit version is running """ url = config.GATEWAY_URL + "/r/config/server/version" resp = requests.get( url, cookies=dict( auth_pubtkt=config.USERS[config.USER_1]['auth_cookie'])) self.assertTrue('"2.11.9"' in resp.text)
def setUpClass(cls): cls.ru = ResourcesUtils() cls.sample_project_dir = \ os.path.join(config.SF_TESTS_DIR, "sample_project/")