def test_retry_generator(exc, in_init, retry_times): def simplegen(): yield "log line" my_args = ('some', 'new') if not in_init: my_kwargs = { 'one': 'first', 'two': 'second', 'retry_times': retry_times } else: my_kwargs = {'one': 'first', 'two': 'second'} if in_init: t = DockerTasker(retry_times=retry_times) else: t = DockerTasker() (flexmock(time).should_receive('sleep').and_return(None)) if not exc: cr = CommandResult() cr._error = "cmd_error" cr._error_detail = {"message": "error_detail"} if exc == APIError: error_message = 'api_error' response = flexmock(content=error_message, status_code=408) (flexmock(atomic_reactor.util).should_receive( 'wait_for_command').times(retry_times + 1).and_raise( APIError, error_message, response)) elif exc == ProtocolError: error_message = 'protocol_error' (flexmock(atomic_reactor.util).should_receive( 'wait_for_command').times(retry_times + 1).and_raise( ProtocolError, error_message)) elif exc == ReadTimeoutError: pool = 'pool' message = 'read_timeout_error' error_message = '{}: {}'.format(pool, message) (flexmock(atomic_reactor.util).should_receive( 'wait_for_command').times(retry_times + 1).and_raise( ReadTimeoutError, pool, 'url', message)) else: (flexmock(atomic_reactor.util).should_receive( 'wait_for_command').times(retry_times + 1).and_return(cr)) error_message = 'cmd_error' if retry_times >= 0: with pytest.raises(RetryGeneratorException) as ex: t.retry_generator(lambda *args, **kwargs: simplegen(), *my_args, **my_kwargs) assert repr(error_message) in repr(ex.value) else: t.retry_generator(lambda *args, **kwargs: simplegen(), *my_args, **my_kwargs)
def test_adddockerfile_plugin(tmpdir): df_content = """ FROM fedora RUN yum install -y python-django CMD blabla""" df = DockerfileParser(str(tmpdir)) df.content = df_content tasker = DockerTasker() workflow = DockerBuildWorkflow(MOCK_SOURCE, 'test-image') workflow.builder = X workflow.builder.df_path = df.dockerfile_path workflow.builder.df_dir = str(tmpdir) runner = PreBuildPluginsRunner(tasker, workflow, [{ 'name': AddDockerfilePlugin.key, 'args': { 'nvr': 'rhel-server-docker-7.1-20' } }]) runner.run() assert AddDockerfilePlugin.key is not None expected_output = """ FROM fedora RUN yum install -y python-django ADD Dockerfile-rhel-server-docker-7.1-20 /root/buildinfo/Dockerfile-rhel-server-docker-7.1-20 CMD blabla""" assert df.content == expected_output
def test_rpmqa_plugin(remove_container_error, ignore_autogenerated): if MOCK: should_raise_error = {} if remove_container_error: should_raise_error['remove_container'] = None mock_docker(should_raise_error=should_raise_error) tasker = DockerTasker() workflow = DockerBuildWorkflow(SOURCE, "test-image") setattr(workflow, 'builder', X()) setattr(workflow.builder, 'image_id', "asd123") setattr(workflow.builder, 'base_image', ImageName(repo='fedora', tag='21')) setattr(workflow.builder, "source", X()) setattr(workflow.builder.source, 'dockerfile_path', "/non/existent") setattr(workflow.builder.source, 'path', "/non/existent") flexmock(docker.Client, logs=mock_logs) runner = PostBuildPluginsRunner( tasker, workflow, [{ "name": PostBuildRPMqaPlugin.key, "args": { 'image_id': TEST_IMAGE, "ignore_autogenerated_gpg_keys": ignore_autogenerated["ignore"] } }]) results = runner.run() assert results[ PostBuildRPMqaPlugin.key] == ignore_autogenerated["package_list"]
def test_compress(self, tmpdir, method, load_exported_image, extension): if MOCK: mock_docker() tasker = DockerTasker() workflow = DockerBuildWorkflow({ 'provider': 'git', 'uri': 'asd' }, 'test-image') workflow.builder = X() exp_img = os.path.join(str(tmpdir), 'img.tar') if load_exported_image: tarfile.open(exp_img, mode='w').close() workflow.exported_image_sequence.append({'path': exp_img}) runner = PostBuildPluginsRunner( tasker, workflow, [{ 'name': CompressPlugin.key, 'args': { 'method': method, 'load_exported_image': load_exported_image, }, }]) runner.run() compressed_img = os.path.join( workflow.source.tmpdir, EXPORTED_COMPRESSED_IMAGE_NAME_TEMPLATE.format(extension)) assert os.path.exists(compressed_img) assert workflow.exported_image_sequence[-1]['path'] == compressed_img
def __init__(self): self.tasker = DockerTasker() self.base_image = ImageName(repo='Fedora', tag='22') self.image_id = 'image_id' self.image = 'image' self.df_path = 'df_path' self.df_dir = 'df_dir'
def __init__(self, source, image, **kwargs): """ """ LastLogger.__init__(self) BuilderStateMachine.__init__(self) print_version_of_tools() self.tasker = DockerTasker() info, version = self.tasker.get_info(), self.tasker.get_version() logger.debug(json.dumps(info, indent=2)) logger.info(json.dumps(version, indent=2)) # arguments for build self.source = source self.base_image = None self.image_id = None self.built_image_info = None self.image = ImageName.parse(image) # get info about base image from dockerfile build_file_path, build_file_dir = self.source.get_build_file_path() self.df_dir = build_file_dir self._df_path = None # If the Dockerfile will be entirely generated from the container.yaml # (in the Flatpak case, say), then a plugin needs to create the Dockerfile # and set the base image if build_file_path.endswith(DOCKERFILE_FILENAME): self.set_df_path(build_file_path)
def mock_environment(tmpdir, session=None, build_process_failed=False, koji_build_id=None, use_import=False): if session is None: session = MockedClientSession('') if MOCK: mock_docker() tasker = DockerTasker() workflow = DockerBuildWorkflow(SOURCE, 'test-image') setattr(workflow, 'builder', X()) setattr(workflow.builder, 'image_id', '123456imageid') setattr(workflow.builder, 'base_image', ImageName(repo='Fedora', tag='22')) setattr(workflow.builder, 'source', X()) setattr(workflow.builder.source, 'dockerfile_path', None) setattr(workflow.builder.source, 'path', None) flexmock(koji, ClientSession=lambda hub, opts: session) if build_process_failed: workflow.build_result = BuildResult(fail_reason="not built") else: workflow.build_result = BuildResult(image_id="id1234") if koji_build_id: if use_import: workflow.exit_results[KojiImportPlugin.key] = koji_build_id else: workflow.exit_results[KojiPromotePlugin.key] = koji_build_id (flexmock(time).should_receive('sleep').and_return(None)) return tasker, workflow
def test_get_version(): if MOCK: mock_docker() t = DockerTasker() response = t.get_info() assert isinstance(response, dict)
def test_add_labels_plugin(tmpdir, labels_conf_base, labels_conf, dont_overwrite, expected_output): df = DockerfileParser(str(tmpdir)) df.content = DF_CONTENT tasker = DockerTasker() workflow = DockerBuildWorkflow(MOCK_SOURCE, 'test-image') setattr(workflow, 'builder', X) flexmock(workflow, base_image_inspect=labels_conf_base) setattr(workflow.builder, 'df_path', df.dockerfile_path) runner = PreBuildPluginsRunner(tasker, workflow, [{ 'name': AddLabelsPlugin.key, 'args': { 'labels': labels_conf, "dont_overwrite": dont_overwrite } }]) if isinstance(expected_output, RuntimeError): with pytest.raises(RuntimeError): runner.run() else: runner.run() assert AddLabelsPlugin.key is not None assert df.content in expected_output
def test_retry_pull_base_image(exc, failures, should_succeed): if MOCK: mock_docker(remember_images=True) tasker = DockerTasker() workflow = DockerBuildWorkflow(MOCK_SOURCE, 'test-image') workflow.builder = MockBuilder() workflow.builder.base_image = ImageName.parse('parent-image') class MockResponse(object): content = '' expectation = flexmock(tasker).should_receive('tag_image') for _ in range(failures): expectation = expectation.and_raise(exc('', MockResponse())) expectation.and_return('foo') expectation.and_return('parent-image') runner = PreBuildPluginsRunner( tasker, workflow, [{ 'name': PullBaseImagePlugin.key, 'args': {'parent_registry': 'registry.example.com', 'parent_registry_insecure': True}, }], ) if should_succeed: runner.run() else: with pytest.raises(Exception): runner.run()
def test_login(tmpdir, dockerconfig_contents, should_raise): if MOCK: mock_docker() fake_api = flexmock(docker.APIClient, login=lambda username, registry, dockercfg_path: {'Status': 'Login Succeeded'}) tmpdir_path = str(tmpdir.realpath()) file_name = '.dockercfg' dockercfg_path = os.path.join(tmpdir_path, file_name) with open(dockercfg_path, "w+") as dockerconfig: dockerconfig.write(json.dumps(dockerconfig_contents)) dockerconfig.flush() t = DockerTasker() if should_raise: if 'auth' in dockerconfig_contents[LOCALHOST_REGISTRY]: with pytest.raises(ValueError) as exc: t.login(LOCALHOST_REGISTRY, tmpdir_path) assert "Failed to parse 'auth'" in str(exc) else: with pytest.raises(RuntimeError) as exc: t.login(LOCALHOST_REGISTRY, tmpdir_path) assert "Failed to extract a username" in str(exc) else: if MOCK: (fake_api.should_receive('login').with_args( username='******', registry=LOCALHOST_REGISTRY, dockercfg_path=dockercfg_path).once().and_return( {'Status': 'Login Succeeded'})) t.login(LOCALHOST_REGISTRY, tmpdir_path)
def prepare(): """ Boiler-plate test set-up """ tasker = DockerTasker() workflow = DockerBuildWorkflow(SOURCE, "test-image") setattr(workflow, 'builder', X()) setattr(workflow.builder, 'image_id', 'asd123') setattr(workflow.builder, 'source', X()) setattr(workflow.builder.source, 'dockerfile_path', None) setattr(workflow.builder.source, 'path', None) flexmock(OSBS) # No-op implementation until this is implemented in osbs-client setattr(OSBS, 'import_image', lambda **kwargs: None) flexmock(OSBS, import_image=lambda name: None) runner = PostBuildPluginsRunner(tasker, workflow, [{ 'name': ImportImagePlugin.key, 'args': { 'url': '', 'verify_ssl': False, 'use_auth': False } }]) return runner
def test_delete_from_registry_plugin(saved_digests, req_registries, tmpdir): if MOCK: mock_docker() tasker = DockerTasker() workflow = DockerBuildWorkflow({"provider": "git", "uri": "asd"}, TEST_IMAGE) setattr(workflow, 'builder', X) args_registries = {} for reg, use_secret in req_registries.items(): if use_secret: temp_dir = mkdtemp(dir=str(tmpdir)) with open(os.path.join(temp_dir, ".dockercfg"), "w+") as dockerconfig: dockerconfig_contents = { reg: { "username": "******", "password": reg } } dockerconfig.write(json.dumps(dockerconfig_contents)) dockerconfig.flush() args_registries[reg] = { 'secret': temp_dir } else: args_registries[reg] = {} for reg, digests in saved_digests.items(): r = DockerRegistry(reg) for tag, dig in digests.items(): r.digests[tag] = ManifestDigest(v1='not-used', v2=dig) workflow.push_conf._registries['docker'].append(r) runner = ExitPluginsRunner( tasker, workflow, [{ 'name': DeleteFromRegistryPlugin.key, 'args': { 'registries': args_registries }, }] ) deleted_digests = set() for reg, digests in saved_digests.items(): if reg not in req_registries: continue for tag, dig in digests.items(): if dig in deleted_digests: continue url = "https://" + reg + "/v2/" + tag.split(":")[0] + "/manifests/" + dig auth_type = requests.auth.HTTPBasicAuth if req_registries[reg] else None (flexmock(requests) .should_receive('delete') .with_args(url, verify=bool, auth=auth_type) .once() .and_return(flexmock(status_code=202))) deleted_digests.add(dig) result = runner.run() assert result[DeleteFromRegistryPlugin.key] == deleted_digests
def test_adddockerfile_todest(tmpdir): df_content = """ FROM fedora RUN yum install -y python-django CMD blabla""" df = DockerfileParser(str(tmpdir)) df.content = df_content tasker = DockerTasker() workflow = DockerBuildWorkflow(MOCK_SOURCE, 'test-image') workflow.builder = X workflow.builder.df_path = df.dockerfile_path workflow.builder.df_dir = str(tmpdir) runner = PreBuildPluginsRunner(tasker, workflow, [{ 'name': AddDockerfilePlugin.key, 'args': { 'nvr': 'jboss-eap-6-docker-6.4-77', 'destdir': '/usr/share/doc/' } }]) runner.run() assert AddDockerfilePlugin.key is not None expected_output = """ FROM fedora RUN yum install -y python-django ADD Dockerfile-jboss-eap-6-docker-6.4-77 /usr/share/doc/Dockerfile-jboss-eap-6-docker-6.4-77 CMD blabla""" assert df.content == expected_output
def test_get_image_info_by_id_nonexistent(): if MOCK: mock_docker() t = DockerTasker() response = t.get_image_info_by_image_id("asd") assert response is None
def __init__(self): self.tasker = DockerTasker(retry_times=0) self.dockerfile_images = DockerfileImages(['Fedora:22']) self.image_id = 'image_id' self.image = 'image' self.df_path = 'df_path' self.df_dir = 'df_dir'
def test_get_image_info_by_name_tag_in_name(): if MOCK: mock_docker() t = DockerTasker() response = t.get_image_info_by_image_name(input_image_name) assert len(response) == 1
def test_get_image_info_by_name_tag_in_name_nonexisten(temp_image_name): if MOCK: mock_docker() t = DockerTasker() response = t.get_image_info_by_image_name(temp_image_name) assert len(response) == 0
def test_image_doesnt_exist(): image = "lerknglekrnglekrnglekrnglekrng" if MOCK: mock_docker(should_raise_error={'inspect_image': [image]}) t = DockerTasker() assert t.image_exists(image) is False
def prepare(self, df_path, inherited_user='', hide_files=None, parent_images=None): if MOCK: mock_docker() tasker = DockerTasker() workflow = DockerBuildWorkflow("test-image", source=SOURCE) workflow.source = MockSource(df_path) workflow.builder = (StubInsideBuilder() .for_workflow(workflow) .set_df_path(df_path)) for parent in parent_images or []: workflow.builder.set_parent_inspection_data(parent, { INSPECT_CONFIG: { 'User': inherited_user, }, }) if hide_files is not None: reactor_config = ReactorConfig({ 'version': 1, 'hide_files': hide_files }) workflow.plugin_workspace[ReactorConfigPlugin.key] = { WORKSPACE_CONF_KEY: reactor_config } return tasker, workflow
def test_pull_base_image_plugin(df_base, parent_registry, expected_w_reg, expected_wo_reg): if MOCK: mock_docker(remember_images=True) tasker = DockerTasker() workflow = DockerBuildWorkflow(MOCK_SOURCE, 'test-image') workflow.builder = MockBuilder() workflow.builder.base_image = ImageName.parse(df_base) assert not tasker.image_exists(BASE_IMAGE) assert not tasker.image_exists(BASE_IMAGE_W_REGISTRY) runner = PreBuildPluginsRunner(tasker, workflow, [{ 'name': PullBaseImagePlugin.key, 'args': { 'parent_registry': parent_registry, 'parent_registry_insecure': True } }]) runner.run() assert tasker.image_exists(BASE_IMAGE) == expected_wo_reg assert tasker.image_exists(BASE_IMAGE_W_REGISTRY) == expected_w_reg try: tasker.remove_image(BASE_IMAGE) tasker.remove_image(BASE_IMAGE_W_REGISTRY) except: pass
def mock_environment(tmpdir, workflow, primary_images=None, floating_images=None, manifest_results=None, annotations=None): if MOCK: mock_docker() tasker = DockerTasker() base_image_id = '123456parent-id' setattr(workflow, '_base_image_inspect', {'Id': base_image_id}) setattr(workflow, 'builder', StubInsideBuilder()) setattr(workflow.builder, 'image_id', '123456imageid') setattr(workflow.builder, 'base_image', ImageName(repo='Fedora', tag='22')) setattr(workflow.builder, 'source', StubInsideBuilder()) setattr(workflow.builder, 'built_image_info', {'ParentId': base_image_id}) setattr(workflow.builder.source, 'dockerfile_path', None) setattr(workflow.builder.source, 'path', None) if primary_images: for image in primary_images: if '-' in ImageName.parse(image).tag: workflow.tag_conf.add_primary_image(image) workflow.tag_conf.add_unique_image(primary_images[0]) if floating_images: workflow.tag_conf.add_floating_images(floating_images) workflow.build_result = BuildResult(image_id='123456', annotations=annotations or {}) workflow.postbuild_results = {} if manifest_results: workflow.postbuild_results[PLUGIN_GROUP_MANIFESTS_KEY] = manifest_results return tasker, workflow
def prepare(insecure_registry=None): """ Boiler-plate test set-up """ if MOCK: mock_docker() tasker = DockerTasker() workflow = DockerBuildWorkflow(SOURCE, "test-image") setattr(workflow, 'builder', X()) setattr(workflow.builder, 'image_id', 'asd123') setattr(workflow.builder, 'source', X()) setattr(workflow.builder.source, 'dockerfile_path', None) setattr(workflow.builder.source, 'path', None) fake_conf = osbs.conf.Configuration(conf_file=None, openshift_uri='/') flexmock(osbs.conf).should_receive('Configuration').and_return(fake_conf) runner = PostBuildPluginsRunner( tasker, workflow, [{ 'name': ImportImagePlugin.key, 'args': { 'imagestream': TEST_IMAGESTREAM, 'docker_image_repo': TEST_REPO, 'url': '', 'build_json_dir': "", 'verify_ssl': False, 'use_auth': False, 'insecure_registry': insecure_registry, } }]) return runner
def __init__(self, tmpdir): self.tasker = DockerTasker() self.base_image = ImageName(repo='Fedora', tag='22') self.base_from_scratch = False self.image_id = 'image_id' self.image = 'image' self.source = MockSource(tmpdir)
def test_pull_raises_retry_error(workflow, caplog): if MOCK: mock_docker(remember_images=True) tasker = DockerTasker(retry_times=1) workflow.builder = MockBuilder() image_name = ImageName.parse(IMAGE_RAISE_RETRYGENERATOREXCEPTION) base_image_str = "{}/{}:{}".format(SOURCE_REGISTRY, image_name.repo, 'some') source_registry = image_name.registry workflow.builder.dockerfile_images = DockerfileImages([base_image_str]) workflow.plugin_workspace[ReactorConfigPlugin.key] = {} workflow.plugin_workspace[ReactorConfigPlugin.key][WORKSPACE_CONF_KEY] =\ ReactorConfig({'version': 1, 'source_registry': {'url': source_registry, 'insecure': True}}) runner = PreBuildPluginsRunner( tasker, workflow, [{ 'name': PullBaseImagePlugin.key, 'args': {}, }], ) with pytest.raises(Exception): runner.run() exp_img = ImageName.parse(base_image_str) exp_img.registry = source_registry assert 'failed to pull image: {}'.format(exp_img.to_str()) in caplog.text
def mock_environment(tmpdir, primary_images=None, worker_annotations={}): if MOCK: mock_docker() tasker = DockerTasker() workflow = DockerBuildWorkflow(SOURCE, "test-image") base_image_id = '123456parent-id' setattr(workflow, '_base_image_inspect', {'Id': base_image_id}) setattr(workflow, 'builder', X()) setattr(workflow.builder, 'image_id', '123456imageid') setattr(workflow.builder, 'base_image', ImageName(repo='Fedora', tag='22')) setattr(workflow.builder, 'source', X()) setattr(workflow.builder, 'built_image_info', {'ParentId': base_image_id}) setattr(workflow.builder.source, 'dockerfile_path', None) setattr(workflow.builder.source, 'path', None) setattr(workflow, 'tag_conf', TagConf()) if primary_images: workflow.tag_conf.add_primary_images(primary_images) workflow.tag_conf.add_unique_image(primary_images[0]) annotations = deepcopy(BUILD_ANNOTATIONS) if not worker_annotations: worker_annotations = {'ppc64le': PPC_ANNOTATIONS} for worker in worker_annotations: annotations['worker-builds'][worker] = deepcopy( worker_annotations[worker]) workflow.build_result = BuildResult(image_id='123456', annotations=annotations) return tasker, workflow
def prepare(key, value, set_labels_args=None, set_labels_kwargs=None): if MOCK: mock_docker() tasker = DockerTasker() workflow = DockerBuildWorkflow(SOURCE, "test-image") setattr(workflow, 'builder', X()) setattr(workflow.builder, 'image_id', 'asd123') setattr(workflow.builder, 'base_image', ImageName(repo='Fedora', tag='22')) setattr(workflow.builder, 'source', X()) setattr(workflow.builder.source, 'path', '/tmp') setattr(workflow.builder.source, 'dockerfile_path', None) expectation = flexmock(OSBS).should_receive('set_labels_on_build_config') if set_labels_args is not None: if set_labels_kwargs is None: set_labels_kwargs = {} expectation.with_args(*set_labels_args, **set_labels_kwargs) runner = PreBuildPluginsRunner(tasker, workflow, [{ 'name': CheckAndSetRebuildPlugin.key, 'args': { 'label_key': key, 'label_value': value, 'url': '', }, }]) return workflow, runner
def test_inspect_image(): if MOCK: mock_docker() t = DockerTasker() inspect_data = t.inspect_image(input_image_name) assert isinstance(inspect_data, dict)
def __init__(self, source, image, **kwargs): """ """ LastLogger.__init__(self) BuilderStateMachine.__init__(self) print_version_of_tools() self.tasker = DockerTasker() info, version = self.tasker.get_info(), self.tasker.get_version() logger.debug(json.dumps(info, indent=2)) logger.info(json.dumps(version, indent=2)) # arguments for build self.source = source self.base_image_id = None self.image_id = None self.built_image_info = None self.image = ImageName.parse(image) # get info about base image from dockerfile self.df_path, self.df_dir = self.source.get_dockerfile_path() self.set_base_image(df_parser(self.df_path).baseimage) logger.debug("base image specified in dockerfile = '%s'", self.base_image) if not self.base_image.tag: self.base_image.tag = 'latest'
def test_adddockerfile_nvr_from_labels2(tmpdir): df_content = """ FROM fedora RUN yum install -y python-django CMD blabla""" df = DockerfileParser(str(tmpdir)) df.content = df_content tasker = DockerTasker() workflow = DockerBuildWorkflow(MOCK_SOURCE, 'test-image') flexmock(workflow, base_image_inspect={"Config": {"Labels": {}}}) workflow.builder = X workflow.builder.df_path = df.dockerfile_path workflow.builder.df_dir = str(tmpdir) runner = PreBuildPluginsRunner(tasker, workflow, [{ 'name': AddLabelsPlugin.key, 'args': { 'labels': { 'Name': 'jboss-eap-6-docker', 'Version': '6.4', 'Release': '77' } } }, { 'name': AddDockerfilePlugin.key }]) runner.run() assert AddDockerfilePlugin.key is not None assert "ADD Dockerfile-jboss-eap-6-docker-6.4-77 /root/buildinfo/Dockerfile-jboss-eap-6-docker-6.4-77" in df.content