def test_copy_plain_resource_from_brew(mocker, tmpdir): config.cfg['common']['work_dir'] = str(tmpdir) config.cfg['common']['redhat'] = True urlopen_class_mock = mocker.patch('cekit.descriptor.resource.urlopen') mock_urlopen = urlopen_class_mock.return_value mock_urlopen.getcode.return_value = 200 mock_urlopen.read.side_effect = [b"one", b"two", None] ctx = get_ctx(mocker) get_mock_ssl(mocker, ctx) with open('file', 'w') as f: # noqa: F841 pass res = Resource({'name': 'foo', 'md5': '5b9164ad6f496d9dee12ec7634ce253f'}) mocker.spy(res, '_Resource__substitute_cache_url') mock_get_brew_url = mocker.patch( 'cekit.descriptor.resource.get_brew_url', return_value='http://cache/abc') res.copy(str(tmpdir)) mock_get_brew_url.assert_called_once_with('5b9164ad6f496d9dee12ec7634ce253f') res._Resource__substitute_cache_url.call_count == 0 urlopen_class_mock.assert_called_with('http://cache/abc', context=ctx)
def test_copy_plain_resource_with_cacher(mocker, tmpdir): config.cfg['common']['cache_url'] = '#filename#,#algorithm#,#hash#' config.cfg['common']['work_dir'] = str(tmpdir) urlopen_class_mock = mocker.patch('cekit.descriptor.resource.urlopen') mock_urlopen = urlopen_class_mock.return_value mock_urlopen.getcode.return_value = 200 mock_urlopen.read.side_effect = [b"one", b"two", None] ctx = get_ctx(mocker) get_mock_ssl(mocker, ctx) with open('file', 'w') as f: # noqa: F841 pass res = Resource({'name': 'foo', 'md5': '5b9164ad6f496d9dee12ec7634ce253f'}) substitute_cache_url_mock = mocker.patch.object( res, '_Resource__substitute_cache_url', return_value='http://cache/abc') res.copy(str(tmpdir)) substitute_cache_url_mock.assert_called_once_with(None) urlopen_class_mock.assert_called_with('http://cache/abc', context=ctx)
def test_url_resource_download_cleanup_after_failure(mocker, tmpdir, caplog): caplog.set_level(logging.DEBUG, logger="cekit") mocker.patch('os.path.exists', return_value=False) mocker.patch('os.makedirs') os_remove_mock = mocker.patch('os.remove') urlopen_class_mock = mocker.patch('cekit.descriptor.resource.urlopen') urlopen_mock = urlopen_class_mock.return_value urlopen_mock.getcode.return_value = 200 urlopen_mock.read.side_effect = Exception res = Resource({'url': 'http://server.org/dummy', 'sha256': 'justamocksum'}) targetfile = os.path.join(str(tmpdir), 'targetfile') with pytest.raises(CekitError) as excinfo: res.guarded_copy(targetfile) assert "Error copying resource: 'dummy'. See logs for more info" in str(excinfo.value) assert "Removing incompletely downloaded '{}' file".format(targetfile) in caplog.text urlopen_class_mock.assert_called_with('http://server.org/dummy', context=mocker.ANY) os_remove_mock.assert_called_with(targetfile)
def test_resource_verify(mocker): mock = mocker.patch( 'cekit.descriptor.resource.Resource._Resource__check_sum') res = Resource({'url': 'dummy'}) res.checksums = {'sha256': 'justamocksum'} res._Resource__verify('dummy') mock.assert_called_with('dummy', 'sha256', 'justamocksum')
def test_generated_url_with_cacher(): tools.cfg['common']['cache_url'] = '#filename#,#algorithm#,#hash#' res = Resource({'url': 'dummy', 'sha256': 'justamocksum'}) res.name = 'file' assert res._Resource__substitute_cache_url( 'file') == 'file,sha256,justamocksum' tools.cfg = {}
def test_git_clone(mocker): mock = mocker.patch('subprocess.check_output') mocker.patch('os.path.isdir', ret='True') res = Resource({'git': {'url': 'url', 'ref': 'ref'}}) res.copy('dir') mock.assert_called_with( ['git', 'clone', '--depth', '1', 'url', 'dir/url-ref', '-b', 'ref'], stderr=-2)
def test_path_local_non_existing_resource_with_cacher_use(mocker): config.cfg['common']['cache_url'] = '#filename#,#algorithm#,#hash#' mocker.patch('os.path.exists', return_value=False) mocker.patch('os.makedirs') res = Resource({'name': 'foo', 'path': 'bar'}, directory='/foo') mocker.spy(res, '_download_file') download_file_mock = mocker.patch.object(res, '_download_file') res.guarded_copy('target') download_file_mock.assert_called_with('/foo/bar', 'target')
def test_fetching_file_exists_but_used_as_is(mocker): """ It should not download the file, because we didn't specify any hash algorithm, so integrity checking is implicitly disabled here. """ with open('file', 'w') as f: # noqa: F841 pass mock_urlopen = get_mock_urlopen(mocker) res = Resource({'name': 'file', 'url': 'http:///dummy', 'md5': 'd41d8cd98f00b204e9800998ecf8427e'}) res.copy() mock_urlopen.assert_not_called()
def test_fetching_with_ssl_verify(mocker): ctx = get_ctx(mocker) get_mock_ssl(mocker, ctx) mock_urlopen = get_mock_urlopen(mocker) res = Resource({'name': 'file', 'url': 'https:///dummy'}) try: res.copy() except: pass mock_urlopen.assert_called_with('https:///dummy', context=ctx) assert ctx.check_hostname is True assert ctx.verify_mode == 1
def test_path_local_existing_resource_no_cacher_use(mocker): config.cfg['common']['cache_url'] = '#filename#,#algorithm#,#hash#' mocker.patch('os.path.exists', return_value=True) shutil_mock = mocker.patch('shutil.copy') res = Resource({'name': 'foo', 'path': 'bar'}, directory='/foo') mocker.spy(res, '_download_file') res.guarded_copy('target') shutil_mock.assert_called_with('/foo/bar', 'target') assert res._download_file.call_count == 0
def _prepare(self): self._descriptor['labels'] = [ Label(x) for x in self._descriptor.get('labels', []) ] self._descriptor['envs'] = [ Env(x) for x in self._descriptor.get('envs', []) ] self._descriptor['ports'] = [ Port(x) for x in self._descriptor.get('ports', []) ] if 'run' in self._descriptor: self._descriptor['run'] = Run(self._descriptor['run']) self._descriptor['artifacts'] = [ Resource(a, directory=self._artifact_dir) for a in self._descriptor.get('artifacts', []) ] self._descriptor['modules'] = Modules( self._descriptor.get('modules', {}), self.path) self._descriptor['packages'] = Packages( self._descriptor.get('packages', {}), self.path) self._descriptor['osbs'] = Osbs(self._descriptor.get('osbs', {}), self.path) self._descriptor['volumes'] = [ Volume(x) for x in self._descriptor.get('volumes', []) ] # make sure image declarations override any module definitions self._image_overrides = { 'artifacts': Image._to_dict(self.artifacts), 'modules': Image._to_dict(self.modules.install) } self._all_artifacts = Image._to_dict(self.artifacts)
def fetch(self, target_dir): if not self._descriptor['url']['repository']: raise CekitError("Repository not defined for '%s'." % (self.name)) if not os.path.exists(target_dir): os.makedirs(target_dir) Resource({'url': self._descriptor['url']['repository']}) \ .copy(os.path.join(target_dir, self._descriptor['filename']))
def add(self, location, md5, sha1, sha256, sha512): artifact_cache = ArtifactCache() resource = {} resource['url'] = location if md5: resource['md5'] = md5 if sha1: resource['sha1'] = sha1 if sha256: resource['sha256'] = sha256 if sha512: resource['sha512'] = sha512 artifact = Resource(resource) cached = artifact_cache.cached(artifact) if cached: click.echo("Artifact {} is already cached!".format(location)) sys.exit(0) try: artifact_id = artifact_cache.add(artifact) click.echo("Artifact {} cached with UUID '{}'".format( location, artifact_id)) except Exception as ex: # pylint: disable=broad-except click.secho("Cannot cache artifact {}: {}".format( location, str(ex)), fg='red') sys.exit(1)
def test_fetching_disable_ssl_verify(mocker): config.cfg['common']['ssl_verify'] = False mock_urlopen = get_mock_urlopen(mocker) ctx = get_ctx(mocker) get_mock_ssl(mocker, ctx) res = Resource({'name': 'file', 'url': 'https:///dummy'}) try: res.copy() except: pass mock_urlopen.assert_called_with('https:///dummy', context=ctx) assert ctx.check_hostname is False assert ctx.verify_mode == 0
def test_fetching_file_exists_no_hash_fetched_again(mocker): """ It should download the file again, because available file locally doesn't match checksum. """ mock_urlopen = get_mock_urlopen(mocker) ctx = get_ctx(mocker) get_mock_ssl(mocker, ctx) with open('file', 'w') as f: # noqa: F841 pass res = Resource({'name': 'file', 'url': 'http:///dummy'}) with pytest.raises(CekitError): # url is not valid so we get error, but we are not interested # in it. We just need to check that we attempted to downlad. res.copy() mock_urlopen.assert_called_with('http:///dummy', context=ctx)
def test_fetching_file_exists_fetched_again(mocker): """ It should download the file again, because available file locally doesn't match checksum. """ mock_urlopen = get_mock_urlopen(mocker) ctx = get_ctx(mocker) get_mock_ssl(mocker, ctx) with open('file', 'w') as f: # noqa: F841 pass res = Resource({'name': 'file', 'url': 'http:///dummy', 'md5': '123456'}) with pytest.raises(CekitError): # Checksum will fail, because the "downloaded" file # will not have md5 equal to 123456. We need investigate # mocking of requests get calls to do it properly res.copy() mock_urlopen.assert_called_with('http:///dummy', context=ctx)
def __init__(self, descriptor, path): self.schemas = modules_schema super(Modules, self).__init__(descriptor) self._descriptor['repositories'] = [ Resource(r, directory=path) for r in self._descriptor.get('repositories', []) ] self._descriptor['install'] = [ Install(x) for x in self._descriptor.get('install', []) ]
def run(self): tools.cfg = tools.get_cfg(self.args.config) if self.args.verbose: logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.INFO) if self.args.work_dir: tools.cfg['common']['work_dir'] = self.args.work_dir if self.args.cmd == 'add': artifact_cache = ArtifactCache() resource = {} resource['url'] = self.args.url for alg in SUPPORTED_HASH_ALGORITHMS: val = getattr(self.args, alg) if val: resource[alg] = val artifact = Resource(resource) if artifact_cache.is_cached(artifact): print('Artifact is already cached!') sys.exit(0) try: artifact_id = artifact_cache.add(artifact) if self.args.verbose: print(artifact_id) except Exception as ex: if self.args.verbose: traceback.print_exc() else: print(ex) sys.exit(1) if self.args.cmd == 'ls': self.list() if self.args.cmd == 'rm': try: artifact_cache = ArtifactCache() artifact_cache.delete(self.args.uuid) print("Artifact removed") except Exception: print("Artifact doesn't exists") sys.exit(1) sys.exit(0)
def test_fetching_bad_status_code(): res = Resource({'name': 'file', 'url': 'http:///dummy'}) with pytest.raises(CekitError): res.copy()
def fetch(self, target_dir): if not os.path.exists(target_dir): os.makedirs(target_dir) Resource({'url': self._descriptor['url']['repository']}) \ .copy(os.path.join(target_dir, self._descriptor['filename']))
def test_path_resource_relative(): res = Resource({'name': 'foo', 'path': 'bar'}, directory='/foo') assert res.path == '/foo/bar'
def test_repository_dir_is_constructed_properly(mocker): mocker.patch('subprocess.check_output') mocker.patch('os.path.isdir', ret='True') res = Resource({'git': {'url': 'url/repo', 'ref': 'ref'}}) assert res.copy('dir') == 'dir/repo-ref'
def test_path_resource_absolute(): res = Resource({'name': 'foo', 'path': '/bar'}, directory='/foo') assert res.path == '/bar'
def test_generated_url_without_cacher(): res = Resource({'url': 'url'}) assert res._Resource__substitute_cache_url('url') == 'url'
def _prepare(self): """Updates self._descriptor with objects and prepare sane label""" self._descriptor['labels'] = self._descriptor.get('labels', []) # we will persist cekit version in a label here, so we know which version of cekit # was used to build the image self['labels'].extend([{ 'name': 'org.concrt.version', 'value': cekit_version }, { 'name': 'io.cekit.version', 'value': cekit_version }]) # The description key available in image descriptor's # root is added as labels to the image key = 'description' # If we define the label in the image descriptor # we should *not* override it with value from # the root's key if key in self._descriptor and not self.label(key): value = self._descriptor[key] self._descriptor['labels'].append({'name': key, 'value': value}) # Last - if there is no 'summary' label added to image descriptor # we should use the value of the 'description' key and create # a 'summary' label with it's content. If there is even that # key missing - we should not add anything. description = self.label('description') if not self.label('summary') and description: self._descriptor['labels'].append({ 'name': 'summary', 'value': description['value'] }) self._descriptor['labels'] = [ Label(x) for x in self._descriptor.get('labels', []) ] self._descriptor['envs'] = [ Env(x) for x in self._descriptor.get('envs', []) ] self._descriptor['ports'] = [ Port(x) for x in self._descriptor.get('ports', []) ] if 'run' in self._descriptor: self._descriptor['run'] = Run(self._descriptor['run']) self._descriptor['artifacts'] = [ Resource(a, directory=self._artifact_dir) for a in self._descriptor.get('artifacts', []) ] if 'modules' in self._descriptor: self._descriptor['modules'] = Modules(self._descriptor['modules'], self.path) if 'packages' in self._descriptor: self._descriptor['packages'] = Packages( self._descriptor['packages']) if 'osbs' in self._descriptor: self._descriptor['osbs'] = Osbs(self._descriptor['osbs']) self._descriptor['volumes'] = [ Volume(x) for x in self._descriptor.get('volumes', []) ]