class TestDistro(unittest.TestCase):
def setUp(self):
# Setting two backends to ensure this feature works with multiMaster
self.cluster = Cluster(backends=1).start()
self.local_repo = self.cluster.clone()
def tearDown(self):
self.cluster.stop()
del self.cluster
del self.local_repo
def test_distro_implicit_call(self):
"""Test SetWorkerDistro and ensuring the step is implicitly called."""
steps = [{'ShellCommand': {'command': 'exit 0'}}]
branch = {'default': {'stage': 'pre-merge'}}
stage = {'pre-merge': {
'worker': {'type': 'local'},
'steps': steps,
}}
self.local_repo.push(yaml=YamlFactory(branches=branch, stages=stage))
buildset = self.cluster.api.force(branch=self.local_repo.branch)
build = buildset.buildrequest.build
build.wait_for_finish()
premerge = build.children[0].buildrequest.build
# Just ensuring that the properties exists, and
# that the default value has been overwritten
assert premerge.properties['distribution_id'][0] != 'unknown'
assert premerge.properties['distribution_version_id'][0] != 'unknown'
def test_distro_non_existent_os_release_file(self):
"""Test SetWorkerDistro when os release file does not exist."""
steps = [{'SetWorkerDistro': {'osReleaseFilePath': '/does/not/exist'}}]
branch = {'default': {'stage': 'pre-merge'}}
stage = {'pre-merge': {
'worker': {'type': 'local'},
'steps': steps,
}}
self.local_repo.push(yaml=YamlFactory(branches=branch, stages=stage))
buildset = self.cluster.api.force(branch=self.local_repo.branch)
build = buildset.buildrequest.build
build.wait_for_finish()
premerge = build.children[0].buildrequest.build
# Defaults to unknown when the os release file does not exist
assert premerge.properties['distribution_id'][0] == 'unknown'
assert premerge.properties['distribution_version_id'][0] == 'unknown'
class TestCanStartBuild(unittest.TestCase):
def setUp(self):
# Setting two backends to ensure this feature works with multiMaster
self.cluster = Cluster(backends=2).start()
self.local_repo = self.cluster.clone()
def tearDown(self):
self.cluster.stop()
del self.cluster
del self.local_repo
def test_can_start_one_build(self):
"""Check behaviour with one simultaneous build."""
file_path = os.path.join(mkdtemp(), 'file')
command = """
test ! -f {file_path} &&
touch {file_path} &&
test -f {file_path} &&
sleep 5 &&
rm {file_path}
""".format(file_path=file_path)
steps = [{'ShellCommand': {'command': command}}]
branch = {'default': {'stage': 'pre-merge'}}
stage = {
'pre-merge': {
'worker': {
'type': 'local'
},
'steps': steps,
'simultaneous_builds': 1
}
}
self.local_repo.push(yaml=YamlFactory(branches=branch, stages=stage))
for _ in range(8):
self.cluster.api.force(branch=self.local_repo.branch)
builds = self.cluster.api.get_builds()
while len(builds) != 8:
builds = self.cluster.api.get_builds()
buildids = [b['buildid'] for b in builds]
for buildid in buildids:
build = self.cluster.api.get_build_for_id(buildid)
build.wait_for_finish()
self.assertEqual(build.result, 'success')
def test_can_start_few_builds(self):
"""Check can start build method behaviour with N builds."""
def get_running_builds(builds):
return [
build for build in builds
if 'complete' not in build or build['complete'] is False
]
total_builds = 8
simultaneous_builds = 4
command = "echo Ola Mundo!"
steps = [{'ShellCommand': {'command': command}}]
branch = {'default': {'stage': 'pre-merge'}}
stage = {
'pre-merge': {
'worker': {
'type': 'local'
},
'steps': steps,
'simultaneous_builds': simultaneous_builds
}
}
self.local_repo.push(yaml=YamlFactory(branches=branch, stages=stage))
for _ in range(total_builds):
self.cluster.api.force(branch=self.local_repo.branch)
bootstrap = self.cluster.api.get_builds()
# Wait for the pre-merge builder to be created
for sleep in range(10):
pre_merge = self.cluster.api.get('/builders',
{'name': 'pre-merge'})
if sleep == 10 and not pre_merge:
self.fail('pre-merge builder was not created')
elif not pre_merge:
time.sleep(sleep)
else:
pre_merge = get_running_builds(pre_merge)
break
while any(pre_merge) or any(bootstrap):
assert len(pre_merge) <= simultaneous_builds
pre_merge = get_running_builds(
self.cluster.api.get_builds(builder='pre-merge'))
bootstrap = get_running_builds(self.cluster.api.get_builds())
# Just so that we don't spam the API
time.sleep(0.5)
# Retrieving all builds to check that they all ran successfully
pre_merge = self.cluster.api.get_builds(builder='pre-merge')
self.assertEqual(len(pre_merge), total_builds)
for build in pre_merge:
self.assertEqual(build['results'], SUCCESS)
def test_can_start_build_error_handling(self):
"""Ensure string parameters are ignored."""
steps = [{'ShellCommand': {'command': 'exit 0'}}]
branch = {'default': {'stage': 'pre-merge'}}
stage = {
'pre-merge': {
'worker': {
'type': 'local'
},
'steps': steps,
'simultaneous_builds': "string"
}
}
self.local_repo.push(yaml=YamlFactory(branches=branch, stages=stage))
buildset = self.cluster.api.force(branch=self.local_repo.branch)
buildset.wait_for_finish()
self.assertEqual(buildset.result, 'success')
class TestPublishCodeCoverage(unittest.TestCase):
"""Test code coverage report publication step.
``PublishCodeCoverage`` is the generic buildbot step to use to
publish several code coverage reports to an external service.
At this moment, only ``codecov.io`` service is supported.
These tests are two operating modes:
- The first is to use an internal mock ``codecov.io`` server.
- The second use the real ``codecov.io`` service.
By default, in the automatic tests, we use the mock ``codecov.io``
server to avoid being dependent external services that could not
be available in the future.
To use the real ``codecov.io`` server, we need to define the
environment variable **CODECOV_IO_UPLOAD_TOKEN** which is the
``codecov.io`` upload token of the repository given in
*yaml/generate_coverage_report/main.yml* YAML file.
"""
def __init__(self, *args, **kwargs):
super(TestPublishCodeCoverage, self).__init__(*args, **kwargs)
self.codecov_io_server = None
def setUp(self):
"""Instantiate a ``codecov.io`` mock HTTP server if needed.
We define the **CODECOV_IO_BASE_URL** environment variable to
communicate with our mock HTTP server rather than the real
``codecov.io`` server (see
`~master/steps/publish_coverage_report`).
We define the **CODECOV_IO_UPLOAD_TOKEN** environment variable
to avoid to skip the step. It's the default behaviour if we
don't give this variable to Eve.
"""
if not os.environ.get('CODECOV_IO_BASE_URL', None):
self.codecov_io_server = CodecovIOMockServer()
self.codecov_io_server.start()
conf = {
'CODECOV_IO_BASE_URL':
os.environ.get('CODECOV_IO_BASE_URL', self.codecov_io_server.url),
'CODECOV_IO_UPLOAD_TOKEN':
os.environ.get('CODECOV_IO_UPLOAD_TOKEN', 'FAKETOKEN')
}
self.cluster = Cluster(extra_conf=conf)
self.cluster.start()
self.local_repo = self.cluster.clone()
super(TestPublishCodeCoverage, self).setUp()
def tearDown(self):
"""Stop the ``codecov.io`` mock HTTP server if needed.
And restore old environment variables.
"""
super(TestPublishCodeCoverage, self).tearDown()
if self.codecov_io_server:
self.codecov_io_server.stop()
self.codecov_io_server = None
self.cluster.stop()
def test_codecovio_success(self):
"""Test PublishCoverageReport success.
If we use the mock HTTP server, we ensure that we execute the
requests with the correct query parameters and headers, in
accordance with the ``codecov.io`` API (see
https://docs.codecov.io/v4.3.0/reference#upload).
"""
import xml.etree.cElementTree as ET
coverage = ET.Element(
'coverage', {
'branch-rate': '0',
'line-rate': '0',
'timestamp': 'XXXXXXXXXXXXX',
'version': '4.3.4',
})
sources = ET.SubElement(coverage, 'sources')
ET.SubElement(sources, 'source').text = '/srv/test_codecov_io'
packages = ET.SubElement(coverage, 'packages')
pacakge = ET.SubElement(packages, 'package', {
'branch-rate': '0',
'complexity': '0',
'line-rate': '0',
'name': '.',
})
classes = ET.SubElement(pacakge, 'classes')
class_ = ET.SubElement(
classes, 'class', {
'branch-rate': '0',
'complexity': '0',
'filename': 'test.py',
'line-rate': '0',
'name': 'test.py'
})
# Based on https://raw.githubusercontent.com/cobertura/web/
# f0366e5e2cf18f111cbd61fc34ef720a6584ba02/htdocs/xml/coverage-03.dtd
ET.SubElement(class_, 'methods')
lines = ET.SubElement(class_, 'lines')
for i in [3, 4, 5, 7, 8]:
ET.SubElement(lines, 'line', {'hits': '0', 'number': str(i)})
report_file = '/tmp/coverage.xml'
tree = ET.ElementTree(coverage)
tree.write(report_file, encoding='utf-8', xml_declaration=True)
self.local_repo.push(yaml=PreMerge(steps=[{
'SetProperty': {
'name': 'set property my_revision',
'property': 'my_revision',
'value': '98f9379054719da6e7b5fd537b5a8e0ede096968',
}
}, {
'PublishCoverageReport': {
'repository': 'scality/test_codecov_io',
'revision': '%(prop:my_revision)s',
'filepaths': [report_file],
'branch': 'master',
'uploadName': 'ucheck',
'configFile': '.codecov.yml',
}
}]))
buildset = self.cluster.api.force(branch=self.local_repo.branch)
self.assertEqual(buildset.result, 'success')
build = buildset.buildrequest.build
child_buildsets = build.children
self.assertEqual(len(child_buildsets), 1)
child_build = child_buildsets[0].buildrequest.build
self.assertEqual(child_build.result, 'success')
if self.codecov_io_server is None:
return
# The method build.getUrl() returns for some reason an url with:
# - the builder id of the main builder, and not the virtual_builder id
# - a buildid linked to the main builder as well, not with the virtual_build id
self.codecov_io_server.assert_request_received_with(
('POST', '/upload/v4', {
'commit':
'98f9379054719da6e7b5fd537b5a8e0ede096968',
'token':
'FAKETOKEN',
'build':
build.number,
'build_url':
'{0}#builders/{1}/builds/{2}'.format(self.cluster.api.url,
child_build.builderid,
child_build.number),
'service':
'buildbot',
'branch':
'master',
'name':
'ucheck',
'slug':
'scality/test_codecov_io',
'yaml':
'.codecov.yml',
}, {
'Accept': 'text/plain',
'Content-Length': '0',
}), ('PUT', '/s3/fake_report.txt', {
'AWSAccessKeyId': 'FAKEAWSACCESSKID',
'Expires': str(self.codecov_io_server.expires),
'Signature': 'FAKESIGNATURE',
}, {
'Content-Length': str(os.path.getsize(report_file)),
'Content-Type': 'text/plain',
'x-amz-acl': 'public-read',
'x-amz-storage-class': 'REDUCED_REDUNDANCY',
}))
def test_junit_step(self): # pylint: disable=too-many-statements
"""Test customized JUnitShellCommand step with OK tests.
Steps:
- Spawn worker.
- Have various commands create JUnit reports and parse them.
"""
cluster = Cluster().start()
cluster.sanity_check()
local_repo = cluster.clone()
parent = abspath(join(__file__, pardir))
yaml = join(parent, 'main.yml')
reports_dir = join(parent, 'reports')
local_repo.push(yaml=yaml, dirs=(reports_dir, ))
cluster.sanity_check()
buildset = cluster.api.force(branch=local_repo.branch)
self.assertEqual(buildset.result, 'failure')
child_build = \
buildset.buildrequest.build.children[0].buildrequest.build
results = [(step.name, step.state_string, step.result)
for step in child_build.steps]
expected = [
(
u'worker_preparation',
u'worker ready',
'success'
),
(
u'prevent unuseful restarts',
u"'[ $(expr ...'",
'success'
),
(
u'set the artifacts private url',
u"property 'artifacts_private_url' set",
'success'
),
(
u'Check worker OS distribution',
u'finished',
u'success'
),
(
u'Set the current builder id',
u'finished',
u'success'
),
(
u'Set the current build url',
u'finished',
u'success'
),
(
u'extract steps from yaml',
u'finished',
'success'
),
(
u'git pull',
u'update',
'success'),
(
u'SetProperty',
u'Set',
'success'
),
(
u'single report with one pass',
u'T:1 E:0 F:0 S:0',
'success'
),
(
u'three reports with lots of pass',
u'T:217 E:0 F:0 S:0',
# u'T:2134 E:0 F:0 S:108',
'success'
),
(
u'no files in directory',
u'no test results found',
'warnings'
),
(
u'missing report directory',
u'no test results found',
'warnings'
),
(
u'single report with invalid data',
u'no test results found',
'warnings'
),
(
u'report with invalid data along valid report',
u'T:1 E:0 F:0 S:0',
'success'
),
(
u'single report with invalid extension',
u'no test results found',
'warnings'
),
(
u'report with failures and successful command',
u'FAIL: toto.tests.sample.test_sample.test_sample',
'failure'
),
(
u'report with no failures and failed command',
u'T:1 E:0 F:0 S:0',
'failure'
),
(
u'report with failures',
u'FAIL: toto.tests.sample.test_sample.test_sample',
'failure'
),
(
u'report with errors',
u'ERROR: supervisor.test_01_deployment.TestGenericDeployment.'
u'test_supervisor_configuration[os_trusty]',
'failure'
),
(
u'report with skips',
u'T:144 E:0 F:0 S:24',
'success'),
(
u'report with both errors and failures',
u'ERROR: supervisor.test_01_deployment.TestGenericDeployment.'
u'test_supervisor_configuration[os_trusty]',
'failure'
),
(
u'report with one xfail and one xpass',
u'T:2 E:0 F:0 S:2',
'success'
),
(
u'undeclared report directory and a pass',
u'no test results found',
'warnings'
),
(
u'undeclared report directory and a fail',
u'no test results found',
'failure'
),
(
u'test report paths with success',
u'T:1 E:0 F:0 S:0',
'success'
),
(
u'test report paths with failure',
u'FAIL: toto.tests.sample.test_sample.test_sample',
'failure'
),
(
u'test report paths with success and failure',
u'FAIL: toto.tests.sample.test_sample.test_sample',
'failure'
),
(
u'test report paths with success and failure in same',
u'FAIL: toto.tests.sample.test_sample.test_sample',
'failure'
),
(
u'test report paths only uploading success',
u'T:1 E:0 F:0 S:0',
'success'
),
(
u'test report paths only uploading failure',
u'FAIL: toto.tests.sample.test_sample.test_sample',
'failure'
),
(
u'test report paths without list',
u'T:1 E:0 F:0 S:0',
'success'
),
(
u'test report paths with non existing file',
u'no test results found',
'warnings'
),
(
u'test report paths with non matching glob',
u'no test results found',
'warnings'
)
]
self.assertEqual(results, expected)
cluster.stop()