def pre_start_hook(self):
"""Pull a mysql docker image and prepare the command to run it."""
cmd('docker pull mysql/mysql-server:5.7')
self._start_cmd = [
'docker', 'run', '--name', self._name, '-p',
'{}:3306'.format(self.port), '-e',
'MYSQL_ROOT_PASSWORD={}'.format(MYSQL_ROOT_PASSWORD), '-e',
'MYSQL_DATABASE={}'.format(MYSQL_DATABASE), '-e',
'MYSQL_USER={}'.format(MYSQL_USER), '-e',
'MYSQL_PASSWORD={}'.format(MYSQL_PASSWORD),
'mysql/mysql-server:5.7'
]
def external_ip(self):
"""Return the external IP address of the cluster.
Returns:
str: '172.17.0.1' On Linux.
Try to find the address that connects the machine to the
Internet on Mac.
"""
if self._ext_ip:
return self._ext_ip
if platform.system() == 'Darwin':
self._ext_ip = cmd(
# The following expression warns due to only one `\` character
# used, but this is actually voluntary.
# pylint: disable=anomalous-backslash-in-string
"ifconfig | grep -E '([0-9]{1,3}\.){3}[0-9]{1,3}'" # noqa: W605, E501
" | grep -v 127.0.0.1"
" | awk '{ print $2 }' | cut -f2 -d:"
" | head -n1").strip()
assert self._ext_ip, 'Not connected to the internet ?'
return self._ext_ip
else:
self._ext_ip = '172.17.0.1' # Should work on Linux systems
return self._ext_ip
def stop(self):
"""Send a SIGTERM to the daemon and wait until it finishes."""
print('terminating {} PID: {}...'.format(self._name,
self._process.pid))
if self._stop_cmd is not None:
cmd(self._stop_cmd.format(name=self._name), cwd=self._base_path)
# self.wait_for_it()
else:
self._process.terminate()
for _ in range(30):
if self._process.poll() is not None:
time.sleep(1) # allow logs to flush to disk
return
print('process still here, retry in 1s')
time.sleep(1)
raise Exception('{} never stopped {}'.format(self._name,
self._process.pid))
def loglines(self):
"""Return a list of loglines from the git log command.
Returns:
list: The lines resulting from git log command
`git log --pretty=format:"%an %ae|%s|%H|%cd" --date=iso`.
"""
res = cmd('git log --pretty=format:"%an %ae|%s|%H|%cd" --date=iso',
cwd=self._dir)
return reversed(res.splitlines())
def test_git_repo(self):
"""Test that a fake git repo works correctly.
Steps:
- Create a fake remote git repo.
- Clone it with the LocalGitRepo class.
- Push a specified yaml file on a specified branch.
- Check that the file is on that branch.
- Check the branch has been pushed to the remote.
"""
remote = tempfile.mkdtemp(prefix='eve_remote_')
cmd('git init --bare', cwd=remote)
local = LocalGitRepo(remote=remote)
command = 'do something'
branch = 'feature/1'
local.push(SingleCommandYaml(command), branch=branch)
self.assertIn(command, cmd('cat eve/main.yml', cwd=local._dir))
self.assertIn(branch, cmd('git branch', cwd=local._dir))
self.assertIn(branch, cmd('git branch', cwd=remote))
def __init__(self, remote):
self.remote = remote
self._dir = tempfile.mkdtemp(prefix='eve_local_')
cmd('git clone {} .'.format(self.remote), cwd=self._dir)
cmd('git config user.email "*****@*****.**"', cwd=self._dir)
cmd('git config user.name "John Doe"', cwd=self._dir)
self.branch = None
def push(self, yaml=None, dirs=(), branch=None):
"""Create a new commit to trigger a test build.
Args:
yaml (YamlFactory or str): The yaml file to be pushed.
dirs (list): Additional folders to be pushed to the git repo root.
branch (str): The branch name to push to.
"""
if branch is None:
branch = 'bugfix/heal_the_world_{}'.format(uuid4())
if yaml is None:
yaml = SingleCommandYaml()
self.branch = branch
try:
cmd('git checkout %s' % branch, cwd=self._dir)
except CalledProcessError:
cmd('git checkout -b %s' % branch, cwd=self._dir)
try:
mkdir(join(self._dir, 'eve'))
except OSError as error:
if not error.errno == errno.EEXIST:
raise
if isinstance(yaml, RawYaml):
yaml.filedump(join(self._dir, 'eve', 'main.yml'))
else:
shutil.copyfile(yaml, join(self._dir, 'eve', 'main.yml'))
for src in dirs:
shutil.copytree(src, join(self._dir, basename(src)))
cmd('git add -A', cwd=self._dir)
cmd('git commit -m "add yaml file"', cwd=self._dir)
cmd('git push -u origin HEAD:%s' % branch, cwd=self._dir)
return self
def cmd(self, *args, **kwargs):
"""Run a command in the git repo directory."""
kwargs['cwd'] = self._dir
return cmd(*args, **kwargs)
def revision(self):
return cmd('git rev-parse %s' % self.branch, cwd=self._dir).strip()
def _log(self):
"""Return the logs of the Vault docker container."""
return cmd('docker logs {}'.format(self._name))
def pre_start_hook(self):
"""Build an eve docker image and prepare the environment to run it."""
cmd('docker build --build-arg INTERNAL_DOCKER=1 -t eve_master .')
self._env = self.environ
self.dump(self._env_file)
def test_simple_build(self):
"""Test a simple build with registry support.
Steps:
- Check that the tag doesn't belong to the registry.
- Build the project.
- Check all steps for correctness, including image built
and uploaded to the registry.
- Check that the tagged image was added to the registry.
- Build the project again.
- Check all steps again, make sure image was not built,
and reused straight from the local images.
- Remove all local images.
- Build the project again.
- Check the image is pulled from the registry.
"""
local_repo = self.cluster.clone()
local_repo.push(yaml=SingleCommandYaml('exit 0',
worker={
'type': 'docker',
'path': 'ubuntu-focal-ctxt'
}),
dirs=[
abspath(
join(__file__, pardir, 'contexts',
'ubuntu-focal-ctxt'))
])
self.cluster.api.force(branch=local_repo.branch)
# Check bootstrap
bootstrap = self.cluster.api.get_builder('bootstrap')
bootstrap_build = self.cluster.api.get_finished_build('bootstrap',
timeout=180)
self.assertEqual(bootstrap_build['results'], SUCCESS)
bootstrap_steps = self.cluster.api.get_build_steps(bootstrap_build)
step_names_and_descriptions = [(step['name'], step['state_string'])
for step in bootstrap_steps]
self.assertEqual(
step_names_and_descriptions,
[
(u'worker_preparation', u'worker ready'),
(u'set the bootstrap build number', u'Set'),
(u'check index.lock', u"'test ! ...'"),
(u'checkout git branch', u'update'),
(u'cancel builds for commits that are not branch tips',
u'CancelNonTipBuild'), # noqa
(u'set the master_builddir property', u'Set'),
(u'get the product version',
u"property 'product_version' set"), # noqa
(u'check the product version', u"'echo 0.0.0 ...'"),
(u'read eve/main.yml', u'uploading main.yml'),
(u'get the commit short_revision',
u"property 'commit_short_revision' set"), # noqa
(u'get the commit timestamp',
u"property 'commit_timestamp' set"),
(u'set the artifacts name', u"property 'artifacts_name' set"),
(u'set the artifacts public url',
u"property 'artifacts_public_url' set"), # noqa
(u'get the API version', u'Set'),
(u'prepare 1 stage(s)', u'finished'),
(u'[ubuntu-focal-ctxt_e36a] fingerprint', u"Ran"), # noqa
(u'[ubuntu-focal-ctxt_e36a] look up', u"failed (1)"), # noqa
(u'[ubuntu-focal-ctxt_e36a] pull', u"failed (1)"), # noqa
(u'[ubuntu-focal-ctxt_e36a] build', u'Ran'),
(u'[ubuntu-focal-ctxt_e36a] build retry', u'Ran (skipped)'),
(u'[ubuntu-focal-ctxt_e36a] push', u'Ran'),
(u'trigger', u'triggered pre-merge')
])
# Check build
build = self.cluster.api.get_finished_build('pre-merge')
self.assertEqual(build['results'], SUCCESS)
steps = self.cluster.api.get_build_steps(build)
step_names_and_descriptions = [(step['name'], step['state_string'])
for step in steps]
self.assertEqual(step_names_and_descriptions,
[(u'worker_preparation', u'worker ready'),
(u'prevent unuseful restarts', u"'[ $(expr ...'"),
(u'set the artifacts private url',
u"property 'artifacts_private_url' set"),
(u'Check worker OS distribution', u'finished'),
(u'Set the current builder id', u'finished'),
(u'Set the current build url', u'finished'),
(u'extract steps from yaml', u'finished'),
(u'shell', u"'exit 0'")])
# Check properties
bootstrap_properties = self.cluster.api.getw(
'/builds/{}'.format(bootstrap_build['buildid']),
get_params={'property': '*'})
pprint(bootstrap_properties)
# now do the same build again, and check image is reused
buildset = self.cluster.api.force(branch=local_repo.branch)
buildrequestid = self.cluster.api.getw(
'/buildrequests', {'buildsetid': buildset.bsid})['buildrequestid']
bootstrap_build = self.cluster.api.getw(
'/builds', {
'buildrequestid': buildrequestid,
'builderid': bootstrap['builderid'],
'results': SUCCESS
})
bootstrap_steps = self.cluster.api.get_build_steps(bootstrap_build)
step_names_and_descriptions = [(step['name'], step['state_string'])
for step in bootstrap_steps]
self.assertEqual(
step_names_and_descriptions,
[
(u'worker_preparation', u'worker ready'),
(u'set the bootstrap build number', u'Set'),
(u'check index.lock', u"'test ! ...'"),
(u'checkout git branch', u'update'),
(u'cancel builds for commits that are not branch tips',
u'CancelNonTipBuild'), # noqa
(u'set the master_builddir property', u'Set'),
(u'get the product version',
u"property 'product_version' set"),
(u'check the product version', u"'echo 0.0.0 ...'"),
(u'read eve/main.yml', u'uploading main.yml'),
(u'get the commit short_revision',
u"property 'commit_short_revision' set"), # noqa
(u'get the commit timestamp',
u"property 'commit_timestamp' set"),
(u'set the artifacts name', u"property 'artifacts_name' set"),
(u'set the artifacts public url',
u"property 'artifacts_public_url' set"), # noqa
(u'get the API version', u'Set'),
(u'prepare 1 stage(s)', u'finished'),
(u'[ubuntu-focal-ctxt_e36a] fingerprint', u"Ran"), # noqa
(u'[ubuntu-focal-ctxt_e36a] look up', u"Ran"), # noqa
(u'[ubuntu-focal-ctxt_e36a] pull', u"Ran (skipped)"), # noqa
(u'[ubuntu-focal-ctxt_e36a] build', u'Ran (skipped)'),
(u'[ubuntu-focal-ctxt_e36a] build retry', u'Ran (skipped)'),
(u'[ubuntu-focal-ctxt_e36a] push', u'Ran (skipped)'),
(u'trigger', u'triggered pre-merge')
])
# do the same build one last time, but erase the local image first
build = self.cluster.api.getw('/builds', {
'builderid': build['builderid'],
'results': SUCCESS
},
expected_count=2)[1]
image = self.cluster.api.getw(
'/builds/{}'.format(build['buildid']),
get_params={'property': '*'})['properties']['docker_image'][0]
cmd('docker rmi -f {}'.format(image))
buildset = self.cluster.api.force(branch=local_repo.branch)
buildrequestid = self.cluster.api.getw(
'/buildrequests', {'buildsetid': buildset.bsid})['buildrequestid']
bootstrap_build = self.cluster.api.getw(
'/builds', {
'buildrequestid': buildrequestid,
'builderid': bootstrap['builderid'],
'results': SUCCESS
})
bootstrap_steps = self.cluster.api.get_build_steps(bootstrap_build)
step_names_and_descriptions = [(step['name'], step['state_string'])
for step in bootstrap_steps]
self.assertEqual(
step_names_and_descriptions,
[
(u'worker_preparation', u'worker ready'),
(u'set the bootstrap build number', u'Set'),
(u'check index.lock', u"'test ! ...'"),
(u'checkout git branch', u'update'),
(u'cancel builds for commits that are not branch tips',
u'CancelNonTipBuild'), # noqa
(u'set the master_builddir property', u'Set'),
(u'get the product version',
u"property 'product_version' set"), # noqa
(u'check the product version', u"'echo 0.0.0 ...'"),
(u'read eve/main.yml', u'uploading main.yml'),
(u'get the commit short_revision',
u"property 'commit_short_revision' set"), # noqa
(u'get the commit timestamp',
u"property 'commit_timestamp' set"), # noqa
(u'set the artifacts name', u"property 'artifacts_name' set"),
(u'set the artifacts public url',
u"property 'artifacts_public_url' set"), # noqa
(u'get the API version', u'Set'),
(u'prepare 1 stage(s)', u'finished'),
(u'[ubuntu-focal-ctxt_e36a] fingerprint', u"Ran"), # noqa
(u'[ubuntu-focal-ctxt_e36a] look up', u"failed (1)"), # noqa
(u'[ubuntu-focal-ctxt_e36a] pull', u"Ran"), # noqa
(u'[ubuntu-focal-ctxt_e36a] build', u'Ran (skipped)'),
(u'[ubuntu-focal-ctxt_e36a] build retry', u'Ran (skipped)'),
(u'[ubuntu-focal-ctxt_e36a] push', u'Ran (skipped)'),
(u'trigger', u'triggered pre-merge')
])
def pre_start_hook(self):
"""Create a bare git repo before starting the git daemon."""
repo_path = os.path.join(self._base_path, 'repo_owner', 'test.git')
cmd('mkdir -p {}'.format(repo_path), cwd=self._base_path)
cmd('git init --bare', cwd=repo_path)
def test_start_stop(self):
cmd('docker pull ubuntu:focal')
name = '{}:{}/test_registry_start_stop'.format(
self.registry.external_ip, self.registry.port)
cmd('docker tag ubuntu:focal {}'.format(name))
cmd('docker push {}'.format(name))
assert name in cmd('docker images')
cmd('docker rmi {}'.format(name))
assert name not in cmd('docker images')
cmd('docker pull {}'.format(name))
assert name in cmd('docker images')
def pre_start_hook(self):
cmd('docker pull registry:2')
self._start_cmd = [
'docker', 'run', '--name', self._name, '-p',
'{}:5000'.format(self.port), 'registry:2'
]
def test_start_and_stop(self):
"""Test start and stop a githost mock daemon.
Steps:
- Start a GitHostMock.
- Clone it to a temp directory.
- Create a branch and push it.
- Check that the branch has been pushed successfully.
- Stop the GitHostMock.
"""
ghm = GitHostMock().start()
clone_dir = tempfile.mkdtemp()
cmd('git clone git://localhost:{}/repo_owner/test.git .'.format(
ghm.port),
cwd=clone_dir)
cmd('git config user.email "*****@*****.**"', cwd=clone_dir)
cmd('git config user.name "John Doe"', cwd=clone_dir)
cmd('echo hello > readme.txt', cwd=clone_dir)
cmd('git add -A', cwd=clone_dir)
cmd('git commit -m "add yaml file"', cwd=clone_dir)
cmd('git push -u origin HEAD:mybranch', cwd=clone_dir)
output = cmd('git checkout mybranch', cwd=clone_dir)
# Check output from GitHostMock.
# To support development setups, it's easier to check for the various
# possible formats here.
# 1. This format was supported originally by git (quite an old format)
git_msg_origin = 'set up to track remote branch mybranch'
# 2. This format appeared in git v2.15.0 (note additional quotes)
git_msg_2_15 = 'set up to track remote branch \'mybranch\''
# Check that at least one matches.
assert git_msg_origin in output or git_msg_2_15 in output
ghm.stop()
