def pre_start_hook(self): """Pull a mysql docker image and prepare the command to run it.""" cmd('docker pull mysql/mysql-server:5.7') self._start_cmd = [ 'docker', 'run', '--name', self._name, '-p', '{}:3306'.format(self.port), '-e', 'MYSQL_ROOT_PASSWORD={}'.format(MYSQL_ROOT_PASSWORD), '-e', 'MYSQL_DATABASE={}'.format(MYSQL_DATABASE), '-e', 'MYSQL_USER={}'.format(MYSQL_USER), '-e', 'MYSQL_PASSWORD={}'.format(MYSQL_PASSWORD), 'mysql/mysql-server:5.7' ]
def external_ip(self): """Return the external IP address of the cluster. Returns: str: '172.17.0.1' On Linux. Try to find the address that connects the machine to the Internet on Mac. """ if self._ext_ip: return self._ext_ip if platform.system() == 'Darwin': self._ext_ip = cmd( # The following expression warns due to only one `\` character # used, but this is actually voluntary. # pylint: disable=anomalous-backslash-in-string "ifconfig | grep -E '([0-9]{1,3}\.){3}[0-9]{1,3}'" # noqa: W605, E501 " | grep -v 127.0.0.1" " | awk '{ print $2 }' | cut -f2 -d:" " | head -n1").strip() assert self._ext_ip, 'Not connected to the internet ?' return self._ext_ip else: self._ext_ip = '172.17.0.1' # Should work on Linux systems return self._ext_ip
def stop(self): """Send a SIGTERM to the daemon and wait until it finishes.""" print('terminating {} PID: {}...'.format(self._name, self._process.pid)) if self._stop_cmd is not None: cmd(self._stop_cmd.format(name=self._name), cwd=self._base_path) # self.wait_for_it() else: self._process.terminate() for _ in range(30): if self._process.poll() is not None: time.sleep(1) # allow logs to flush to disk return print('process still here, retry in 1s') time.sleep(1) raise Exception('{} never stopped {}'.format(self._name, self._process.pid))
def loglines(self): """Return a list of loglines from the git log command. Returns: list: The lines resulting from git log command `git log --pretty=format:"%an %ae|%s|%H|%cd" --date=iso`. """ res = cmd('git log --pretty=format:"%an %ae|%s|%H|%cd" --date=iso', cwd=self._dir) return reversed(res.splitlines())
def test_git_repo(self): """Test that a fake git repo works correctly. Steps: - Create a fake remote git repo. - Clone it with the LocalGitRepo class. - Push a specified yaml file on a specified branch. - Check that the file is on that branch. - Check the branch has been pushed to the remote. """ remote = tempfile.mkdtemp(prefix='eve_remote_') cmd('git init --bare', cwd=remote) local = LocalGitRepo(remote=remote) command = 'do something' branch = 'feature/1' local.push(SingleCommandYaml(command), branch=branch) self.assertIn(command, cmd('cat eve/main.yml', cwd=local._dir)) self.assertIn(branch, cmd('git branch', cwd=local._dir)) self.assertIn(branch, cmd('git branch', cwd=remote))
def __init__(self, remote): self.remote = remote self._dir = tempfile.mkdtemp(prefix='eve_local_') cmd('git clone {} .'.format(self.remote), cwd=self._dir) cmd('git config user.email "*****@*****.**"', cwd=self._dir) cmd('git config user.name "John Doe"', cwd=self._dir) self.branch = None
def push(self, yaml=None, dirs=(), branch=None): """Create a new commit to trigger a test build. Args: yaml (YamlFactory or str): The yaml file to be pushed. dirs (list): Additional folders to be pushed to the git repo root. branch (str): The branch name to push to. """ if branch is None: branch = 'bugfix/heal_the_world_{}'.format(uuid4()) if yaml is None: yaml = SingleCommandYaml() self.branch = branch try: cmd('git checkout %s' % branch, cwd=self._dir) except CalledProcessError: cmd('git checkout -b %s' % branch, cwd=self._dir) try: mkdir(join(self._dir, 'eve')) except OSError as error: if not error.errno == errno.EEXIST: raise if isinstance(yaml, RawYaml): yaml.filedump(join(self._dir, 'eve', 'main.yml')) else: shutil.copyfile(yaml, join(self._dir, 'eve', 'main.yml')) for src in dirs: shutil.copytree(src, join(self._dir, basename(src))) cmd('git add -A', cwd=self._dir) cmd('git commit -m "add yaml file"', cwd=self._dir) cmd('git push -u origin HEAD:%s' % branch, cwd=self._dir) return self
def cmd(self, *args, **kwargs): """Run a command in the git repo directory.""" kwargs['cwd'] = self._dir return cmd(*args, **kwargs)
def revision(self): return cmd('git rev-parse %s' % self.branch, cwd=self._dir).strip()
def _log(self): """Return the logs of the Vault docker container.""" return cmd('docker logs {}'.format(self._name))
def pre_start_hook(self): """Build an eve docker image and prepare the environment to run it.""" cmd('docker build --build-arg INTERNAL_DOCKER=1 -t eve_master .') self._env = self.environ self.dump(self._env_file)
def test_simple_build(self): """Test a simple build with registry support. Steps: - Check that the tag doesn't belong to the registry. - Build the project. - Check all steps for correctness, including image built and uploaded to the registry. - Check that the tagged image was added to the registry. - Build the project again. - Check all steps again, make sure image was not built, and reused straight from the local images. - Remove all local images. - Build the project again. - Check the image is pulled from the registry. """ local_repo = self.cluster.clone() local_repo.push(yaml=SingleCommandYaml('exit 0', worker={ 'type': 'docker', 'path': 'ubuntu-focal-ctxt' }), dirs=[ abspath( join(__file__, pardir, 'contexts', 'ubuntu-focal-ctxt')) ]) self.cluster.api.force(branch=local_repo.branch) # Check bootstrap bootstrap = self.cluster.api.get_builder('bootstrap') bootstrap_build = self.cluster.api.get_finished_build('bootstrap', timeout=180) self.assertEqual(bootstrap_build['results'], SUCCESS) bootstrap_steps = self.cluster.api.get_build_steps(bootstrap_build) step_names_and_descriptions = [(step['name'], step['state_string']) for step in bootstrap_steps] self.assertEqual( step_names_and_descriptions, [ (u'worker_preparation', u'worker ready'), (u'set the bootstrap build number', u'Set'), (u'check index.lock', u"'test ! ...'"), (u'checkout git branch', u'update'), (u'cancel builds for commits that are not branch tips', u'CancelNonTipBuild'), # noqa (u'set the master_builddir property', u'Set'), (u'get the product version', u"property 'product_version' set"), # noqa (u'check the product version', u"'echo 0.0.0 ...'"), (u'read eve/main.yml', u'uploading main.yml'), (u'get the commit short_revision', u"property 'commit_short_revision' set"), # noqa (u'get the commit timestamp', u"property 'commit_timestamp' set"), (u'set the artifacts name', u"property 'artifacts_name' set"), (u'set the artifacts public url', u"property 'artifacts_public_url' set"), # noqa (u'get the API version', u'Set'), (u'prepare 1 stage(s)', u'finished'), (u'[ubuntu-focal-ctxt_e36a] fingerprint', u"Ran"), # noqa (u'[ubuntu-focal-ctxt_e36a] look up', u"failed (1)"), # noqa (u'[ubuntu-focal-ctxt_e36a] pull', u"failed (1)"), # noqa (u'[ubuntu-focal-ctxt_e36a] build', u'Ran'), (u'[ubuntu-focal-ctxt_e36a] build retry', u'Ran (skipped)'), (u'[ubuntu-focal-ctxt_e36a] push', u'Ran'), (u'trigger', u'triggered pre-merge') ]) # Check build build = self.cluster.api.get_finished_build('pre-merge') self.assertEqual(build['results'], SUCCESS) steps = self.cluster.api.get_build_steps(build) step_names_and_descriptions = [(step['name'], step['state_string']) for step in steps] self.assertEqual(step_names_and_descriptions, [(u'worker_preparation', u'worker ready'), (u'prevent unuseful restarts', u"'[ $(expr ...'"), (u'set the artifacts private url', u"property 'artifacts_private_url' set"), (u'Check worker OS distribution', u'finished'), (u'Set the current builder id', u'finished'), (u'Set the current build url', u'finished'), (u'extract steps from yaml', u'finished'), (u'shell', u"'exit 0'")]) # Check properties bootstrap_properties = self.cluster.api.getw( '/builds/{}'.format(bootstrap_build['buildid']), get_params={'property': '*'}) pprint(bootstrap_properties) # now do the same build again, and check image is reused buildset = self.cluster.api.force(branch=local_repo.branch) buildrequestid = self.cluster.api.getw( '/buildrequests', {'buildsetid': buildset.bsid})['buildrequestid'] bootstrap_build = self.cluster.api.getw( '/builds', { 'buildrequestid': buildrequestid, 'builderid': bootstrap['builderid'], 'results': SUCCESS }) bootstrap_steps = self.cluster.api.get_build_steps(bootstrap_build) step_names_and_descriptions = [(step['name'], step['state_string']) for step in bootstrap_steps] self.assertEqual( step_names_and_descriptions, [ (u'worker_preparation', u'worker ready'), (u'set the bootstrap build number', u'Set'), (u'check index.lock', u"'test ! ...'"), (u'checkout git branch', u'update'), (u'cancel builds for commits that are not branch tips', u'CancelNonTipBuild'), # noqa (u'set the master_builddir property', u'Set'), (u'get the product version', u"property 'product_version' set"), (u'check the product version', u"'echo 0.0.0 ...'"), (u'read eve/main.yml', u'uploading main.yml'), (u'get the commit short_revision', u"property 'commit_short_revision' set"), # noqa (u'get the commit timestamp', u"property 'commit_timestamp' set"), (u'set the artifacts name', u"property 'artifacts_name' set"), (u'set the artifacts public url', u"property 'artifacts_public_url' set"), # noqa (u'get the API version', u'Set'), (u'prepare 1 stage(s)', u'finished'), (u'[ubuntu-focal-ctxt_e36a] fingerprint', u"Ran"), # noqa (u'[ubuntu-focal-ctxt_e36a] look up', u"Ran"), # noqa (u'[ubuntu-focal-ctxt_e36a] pull', u"Ran (skipped)"), # noqa (u'[ubuntu-focal-ctxt_e36a] build', u'Ran (skipped)'), (u'[ubuntu-focal-ctxt_e36a] build retry', u'Ran (skipped)'), (u'[ubuntu-focal-ctxt_e36a] push', u'Ran (skipped)'), (u'trigger', u'triggered pre-merge') ]) # do the same build one last time, but erase the local image first build = self.cluster.api.getw('/builds', { 'builderid': build['builderid'], 'results': SUCCESS }, expected_count=2)[1] image = self.cluster.api.getw( '/builds/{}'.format(build['buildid']), get_params={'property': '*'})['properties']['docker_image'][0] cmd('docker rmi -f {}'.format(image)) buildset = self.cluster.api.force(branch=local_repo.branch) buildrequestid = self.cluster.api.getw( '/buildrequests', {'buildsetid': buildset.bsid})['buildrequestid'] bootstrap_build = self.cluster.api.getw( '/builds', { 'buildrequestid': buildrequestid, 'builderid': bootstrap['builderid'], 'results': SUCCESS }) bootstrap_steps = self.cluster.api.get_build_steps(bootstrap_build) step_names_and_descriptions = [(step['name'], step['state_string']) for step in bootstrap_steps] self.assertEqual( step_names_and_descriptions, [ (u'worker_preparation', u'worker ready'), (u'set the bootstrap build number', u'Set'), (u'check index.lock', u"'test ! ...'"), (u'checkout git branch', u'update'), (u'cancel builds for commits that are not branch tips', u'CancelNonTipBuild'), # noqa (u'set the master_builddir property', u'Set'), (u'get the product version', u"property 'product_version' set"), # noqa (u'check the product version', u"'echo 0.0.0 ...'"), (u'read eve/main.yml', u'uploading main.yml'), (u'get the commit short_revision', u"property 'commit_short_revision' set"), # noqa (u'get the commit timestamp', u"property 'commit_timestamp' set"), # noqa (u'set the artifacts name', u"property 'artifacts_name' set"), (u'set the artifacts public url', u"property 'artifacts_public_url' set"), # noqa (u'get the API version', u'Set'), (u'prepare 1 stage(s)', u'finished'), (u'[ubuntu-focal-ctxt_e36a] fingerprint', u"Ran"), # noqa (u'[ubuntu-focal-ctxt_e36a] look up', u"failed (1)"), # noqa (u'[ubuntu-focal-ctxt_e36a] pull', u"Ran"), # noqa (u'[ubuntu-focal-ctxt_e36a] build', u'Ran (skipped)'), (u'[ubuntu-focal-ctxt_e36a] build retry', u'Ran (skipped)'), (u'[ubuntu-focal-ctxt_e36a] push', u'Ran (skipped)'), (u'trigger', u'triggered pre-merge') ])
def pre_start_hook(self): """Create a bare git repo before starting the git daemon.""" repo_path = os.path.join(self._base_path, 'repo_owner', 'test.git') cmd('mkdir -p {}'.format(repo_path), cwd=self._base_path) cmd('git init --bare', cwd=repo_path)
def test_start_stop(self): cmd('docker pull ubuntu:focal') name = '{}:{}/test_registry_start_stop'.format( self.registry.external_ip, self.registry.port) cmd('docker tag ubuntu:focal {}'.format(name)) cmd('docker push {}'.format(name)) assert name in cmd('docker images') cmd('docker rmi {}'.format(name)) assert name not in cmd('docker images') cmd('docker pull {}'.format(name)) assert name in cmd('docker images')
def pre_start_hook(self): cmd('docker pull registry:2') self._start_cmd = [ 'docker', 'run', '--name', self._name, '-p', '{}:5000'.format(self.port), 'registry:2' ]
def test_start_and_stop(self): """Test start and stop a githost mock daemon. Steps: - Start a GitHostMock. - Clone it to a temp directory. - Create a branch and push it. - Check that the branch has been pushed successfully. - Stop the GitHostMock. """ ghm = GitHostMock().start() clone_dir = tempfile.mkdtemp() cmd('git clone git://localhost:{}/repo_owner/test.git .'.format( ghm.port), cwd=clone_dir) cmd('git config user.email "*****@*****.**"', cwd=clone_dir) cmd('git config user.name "John Doe"', cwd=clone_dir) cmd('echo hello > readme.txt', cwd=clone_dir) cmd('git add -A', cwd=clone_dir) cmd('git commit -m "add yaml file"', cwd=clone_dir) cmd('git push -u origin HEAD:mybranch', cwd=clone_dir) output = cmd('git checkout mybranch', cwd=clone_dir) # Check output from GitHostMock. # To support development setups, it's easier to check for the various # possible formats here. # 1. This format was supported originally by git (quite an old format) git_msg_origin = 'set up to track remote branch mybranch' # 2. This format appeared in git v2.15.0 (note additional quotes) git_msg_2_15 = 'set up to track remote branch \'mybranch\'' # Check that at least one matches. assert git_msg_origin in output or git_msg_2_15 in output ghm.stop()