def test_copy_plain_resource_from_brew(mocker, tmpdir):
config.cfg['common']['work_dir'] = str(tmpdir)
config.cfg['common']['redhat'] = True
urlopen_class_mock = mocker.patch('cekit.descriptor.resource.urlopen')
mock_urlopen = urlopen_class_mock.return_value
mock_urlopen.getcode.return_value = 200
mock_urlopen.read.side_effect = [b"one", b"two", None]
ctx = get_ctx(mocker)
get_mock_ssl(mocker, ctx)
with open('file', 'w') as f: # noqa: F841
pass
res = Resource({'name': 'foo',
'md5': '5b9164ad6f496d9dee12ec7634ce253f'})
mocker.spy(res, '_Resource__substitute_cache_url')
mock_get_brew_url = mocker.patch(
'cekit.descriptor.resource.get_brew_url', return_value='http://cache/abc')
res.copy(str(tmpdir))
mock_get_brew_url.assert_called_once_with('5b9164ad6f496d9dee12ec7634ce253f')
res._Resource__substitute_cache_url.call_count == 0
urlopen_class_mock.assert_called_with('http://cache/abc', context=ctx)
def test_copy_plain_resource_with_cacher(mocker, tmpdir):
config.cfg['common']['cache_url'] = '#filename#,#algorithm#,#hash#'
config.cfg['common']['work_dir'] = str(tmpdir)
urlopen_class_mock = mocker.patch('cekit.descriptor.resource.urlopen')
mock_urlopen = urlopen_class_mock.return_value
mock_urlopen.getcode.return_value = 200
mock_urlopen.read.side_effect = [b"one", b"two", None]
ctx = get_ctx(mocker)
get_mock_ssl(mocker, ctx)
with open('file', 'w') as f: # noqa: F841
pass
res = Resource({'name': 'foo',
'md5': '5b9164ad6f496d9dee12ec7634ce253f'})
substitute_cache_url_mock = mocker.patch.object(
res, '_Resource__substitute_cache_url', return_value='http://cache/abc')
res.copy(str(tmpdir))
substitute_cache_url_mock.assert_called_once_with(None)
urlopen_class_mock.assert_called_with('http://cache/abc', context=ctx)
def test_url_resource_download_cleanup_after_failure(mocker, tmpdir, caplog):
caplog.set_level(logging.DEBUG, logger="cekit")
mocker.patch('os.path.exists', return_value=False)
mocker.patch('os.makedirs')
os_remove_mock = mocker.patch('os.remove')
urlopen_class_mock = mocker.patch('cekit.descriptor.resource.urlopen')
urlopen_mock = urlopen_class_mock.return_value
urlopen_mock.getcode.return_value = 200
urlopen_mock.read.side_effect = Exception
res = Resource({'url': 'http://server.org/dummy',
'sha256': 'justamocksum'})
targetfile = os.path.join(str(tmpdir), 'targetfile')
with pytest.raises(CekitError) as excinfo:
res.guarded_copy(targetfile)
assert "Error copying resource: 'dummy'. See logs for more info" in str(excinfo.value)
assert "Removing incompletely downloaded '{}' file".format(targetfile) in caplog.text
urlopen_class_mock.assert_called_with('http://server.org/dummy', context=mocker.ANY)
os_remove_mock.assert_called_with(targetfile)
def test_resource_verify(mocker):
mock = mocker.patch(
'cekit.descriptor.resource.Resource._Resource__check_sum')
res = Resource({'url': 'dummy'})
res.checksums = {'sha256': 'justamocksum'}
res._Resource__verify('dummy')
mock.assert_called_with('dummy', 'sha256', 'justamocksum')
def test_generated_url_with_cacher():
tools.cfg['common']['cache_url'] = '#filename#,#algorithm#,#hash#'
res = Resource({'url': 'dummy', 'sha256': 'justamocksum'})
res.name = 'file'
assert res._Resource__substitute_cache_url(
'file') == 'file,sha256,justamocksum'
tools.cfg = {}
def test_git_clone(mocker):
mock = mocker.patch('subprocess.check_output')
mocker.patch('os.path.isdir', ret='True')
res = Resource({'git': {'url': 'url', 'ref': 'ref'}})
res.copy('dir')
mock.assert_called_with(
['git', 'clone', '--depth', '1', 'url', 'dir/url-ref', '-b', 'ref'],
stderr=-2)
def test_path_local_non_existing_resource_with_cacher_use(mocker):
config.cfg['common']['cache_url'] = '#filename#,#algorithm#,#hash#'
mocker.patch('os.path.exists', return_value=False)
mocker.patch('os.makedirs')
res = Resource({'name': 'foo',
'path': 'bar'}, directory='/foo')
mocker.spy(res, '_download_file')
download_file_mock = mocker.patch.object(res, '_download_file')
res.guarded_copy('target')
download_file_mock.assert_called_with('/foo/bar', 'target')
def test_fetching_file_exists_but_used_as_is(mocker):
"""
It should not download the file, because we didn't
specify any hash algorithm, so integrity checking is
implicitly disabled here.
"""
with open('file', 'w') as f: # noqa: F841
pass
mock_urlopen = get_mock_urlopen(mocker)
res = Resource({'name': 'file',
'url': 'http:///dummy',
'md5': 'd41d8cd98f00b204e9800998ecf8427e'})
res.copy()
mock_urlopen.assert_not_called()
def test_fetching_with_ssl_verify(mocker):
ctx = get_ctx(mocker)
get_mock_ssl(mocker, ctx)
mock_urlopen = get_mock_urlopen(mocker)
res = Resource({'name': 'file', 'url': 'https:///dummy'})
try:
res.copy()
except:
pass
mock_urlopen.assert_called_with('https:///dummy', context=ctx)
assert ctx.check_hostname is True
assert ctx.verify_mode == 1
def test_path_local_existing_resource_no_cacher_use(mocker):
config.cfg['common']['cache_url'] = '#filename#,#algorithm#,#hash#'
mocker.patch('os.path.exists', return_value=True)
shutil_mock = mocker.patch('shutil.copy')
res = Resource({'name': 'foo',
'path': 'bar'}, directory='/foo')
mocker.spy(res, '_download_file')
res.guarded_copy('target')
shutil_mock.assert_called_with('/foo/bar', 'target')
assert res._download_file.call_count == 0
def _prepare(self):
self._descriptor['labels'] = [
Label(x) for x in self._descriptor.get('labels', [])
]
self._descriptor['envs'] = [
Env(x) for x in self._descriptor.get('envs', [])
]
self._descriptor['ports'] = [
Port(x) for x in self._descriptor.get('ports', [])
]
if 'run' in self._descriptor:
self._descriptor['run'] = Run(self._descriptor['run'])
self._descriptor['artifacts'] = [
Resource(a, directory=self._artifact_dir)
for a in self._descriptor.get('artifacts', [])
]
self._descriptor['modules'] = Modules(
self._descriptor.get('modules', {}), self.path)
self._descriptor['packages'] = Packages(
self._descriptor.get('packages', {}), self.path)
self._descriptor['osbs'] = Osbs(self._descriptor.get('osbs', {}),
self.path)
self._descriptor['volumes'] = [
Volume(x) for x in self._descriptor.get('volumes', [])
]
# make sure image declarations override any module definitions
self._image_overrides = {
'artifacts': Image._to_dict(self.artifacts),
'modules': Image._to_dict(self.modules.install)
}
self._all_artifacts = Image._to_dict(self.artifacts)
def fetch(self, target_dir):
if not self._descriptor['url']['repository']:
raise CekitError("Repository not defined for '%s'." % (self.name))
if not os.path.exists(target_dir):
os.makedirs(target_dir)
Resource({'url': self._descriptor['url']['repository']}) \
.copy(os.path.join(target_dir, self._descriptor['filename']))
def add(self, location, md5, sha1, sha256, sha512):
artifact_cache = ArtifactCache()
resource = {}
resource['url'] = location
if md5:
resource['md5'] = md5
if sha1:
resource['sha1'] = sha1
if sha256:
resource['sha256'] = sha256
if sha512:
resource['sha512'] = sha512
artifact = Resource(resource)
cached = artifact_cache.cached(artifact)
if cached:
click.echo("Artifact {} is already cached!".format(location))
sys.exit(0)
try:
artifact_id = artifact_cache.add(artifact)
click.echo("Artifact {} cached with UUID '{}'".format(
location, artifact_id))
except Exception as ex: # pylint: disable=broad-except
click.secho("Cannot cache artifact {}: {}".format(
location, str(ex)),
fg='red')
sys.exit(1)
def test_fetching_disable_ssl_verify(mocker):
config.cfg['common']['ssl_verify'] = False
mock_urlopen = get_mock_urlopen(mocker)
ctx = get_ctx(mocker)
get_mock_ssl(mocker, ctx)
res = Resource({'name': 'file', 'url': 'https:///dummy'})
try:
res.copy()
except:
pass
mock_urlopen.assert_called_with('https:///dummy', context=ctx)
assert ctx.check_hostname is False
assert ctx.verify_mode == 0
def test_fetching_file_exists_no_hash_fetched_again(mocker):
"""
It should download the file again, because available
file locally doesn't match checksum.
"""
mock_urlopen = get_mock_urlopen(mocker)
ctx = get_ctx(mocker)
get_mock_ssl(mocker, ctx)
with open('file', 'w') as f: # noqa: F841
pass
res = Resource({'name': 'file', 'url': 'http:///dummy'})
with pytest.raises(CekitError):
# url is not valid so we get error, but we are not interested
# in it. We just need to check that we attempted to downlad.
res.copy()
mock_urlopen.assert_called_with('http:///dummy', context=ctx)
def test_fetching_file_exists_fetched_again(mocker):
"""
It should download the file again, because available
file locally doesn't match checksum.
"""
mock_urlopen = get_mock_urlopen(mocker)
ctx = get_ctx(mocker)
get_mock_ssl(mocker, ctx)
with open('file', 'w') as f: # noqa: F841
pass
res = Resource({'name': 'file', 'url': 'http:///dummy', 'md5': '123456'})
with pytest.raises(CekitError):
# Checksum will fail, because the "downloaded" file
# will not have md5 equal to 123456. We need investigate
# mocking of requests get calls to do it properly
res.copy()
mock_urlopen.assert_called_with('http:///dummy', context=ctx)
def __init__(self, descriptor, path):
self.schemas = modules_schema
super(Modules, self).__init__(descriptor)
self._descriptor['repositories'] = [
Resource(r, directory=path)
for r in self._descriptor.get('repositories', [])
]
self._descriptor['install'] = [
Install(x) for x in self._descriptor.get('install', [])
]
def run(self):
tools.cfg = tools.get_cfg(self.args.config)
if self.args.verbose:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
if self.args.work_dir:
tools.cfg['common']['work_dir'] = self.args.work_dir
if self.args.cmd == 'add':
artifact_cache = ArtifactCache()
resource = {}
resource['url'] = self.args.url
for alg in SUPPORTED_HASH_ALGORITHMS:
val = getattr(self.args, alg)
if val:
resource[alg] = val
artifact = Resource(resource)
if artifact_cache.is_cached(artifact):
print('Artifact is already cached!')
sys.exit(0)
try:
artifact_id = artifact_cache.add(artifact)
if self.args.verbose:
print(artifact_id)
except Exception as ex:
if self.args.verbose:
traceback.print_exc()
else:
print(ex)
sys.exit(1)
if self.args.cmd == 'ls':
self.list()
if self.args.cmd == 'rm':
try:
artifact_cache = ArtifactCache()
artifact_cache.delete(self.args.uuid)
print("Artifact removed")
except Exception:
print("Artifact doesn't exists")
sys.exit(1)
sys.exit(0)
def test_fetching_bad_status_code():
res = Resource({'name': 'file', 'url': 'http:///dummy'})
with pytest.raises(CekitError):
res.copy()
def fetch(self, target_dir):
if not os.path.exists(target_dir):
os.makedirs(target_dir)
Resource({'url': self._descriptor['url']['repository']}) \
.copy(os.path.join(target_dir, self._descriptor['filename']))
def test_path_resource_relative():
res = Resource({'name': 'foo', 'path': 'bar'}, directory='/foo')
assert res.path == '/foo/bar'
def test_repository_dir_is_constructed_properly(mocker):
mocker.patch('subprocess.check_output')
mocker.patch('os.path.isdir', ret='True')
res = Resource({'git': {'url': 'url/repo', 'ref': 'ref'}})
assert res.copy('dir') == 'dir/repo-ref'
def test_path_resource_absolute():
res = Resource({'name': 'foo', 'path': '/bar'}, directory='/foo')
assert res.path == '/bar'
def test_generated_url_without_cacher():
res = Resource({'url': 'url'})
assert res._Resource__substitute_cache_url('url') == 'url'
def _prepare(self):
"""Updates self._descriptor with objects and prepare sane label"""
self._descriptor['labels'] = self._descriptor.get('labels', [])
# we will persist cekit version in a label here, so we know which version of cekit
# was used to build the image
self['labels'].extend([{
'name': 'org.concrt.version',
'value': cekit_version
}, {
'name': 'io.cekit.version',
'value': cekit_version
}])
# The description key available in image descriptor's
# root is added as labels to the image
key = 'description'
# If we define the label in the image descriptor
# we should *not* override it with value from
# the root's key
if key in self._descriptor and not self.label(key):
value = self._descriptor[key]
self._descriptor['labels'].append({'name': key, 'value': value})
# Last - if there is no 'summary' label added to image descriptor
# we should use the value of the 'description' key and create
# a 'summary' label with it's content. If there is even that
# key missing - we should not add anything.
description = self.label('description')
if not self.label('summary') and description:
self._descriptor['labels'].append({
'name': 'summary',
'value': description['value']
})
self._descriptor['labels'] = [
Label(x) for x in self._descriptor.get('labels', [])
]
self._descriptor['envs'] = [
Env(x) for x in self._descriptor.get('envs', [])
]
self._descriptor['ports'] = [
Port(x) for x in self._descriptor.get('ports', [])
]
if 'run' in self._descriptor:
self._descriptor['run'] = Run(self._descriptor['run'])
self._descriptor['artifacts'] = [
Resource(a, directory=self._artifact_dir)
for a in self._descriptor.get('artifacts', [])
]
if 'modules' in self._descriptor:
self._descriptor['modules'] = Modules(self._descriptor['modules'],
self.path)
if 'packages' in self._descriptor:
self._descriptor['packages'] = Packages(
self._descriptor['packages'])
if 'osbs' in self._descriptor:
self._descriptor['osbs'] = Osbs(self._descriptor['osbs'])
self._descriptor['volumes'] = [
Volume(x) for x in self._descriptor.get('volumes', [])
]
