def run_plugin_with_args(self, workflow, plugin_args): runner = PrePublishPluginsRunner( workflow, [{ 'name': PrePublishSquashPlugin.key, 'args': plugin_args }]) result = runner.run() assert result[PrePublishSquashPlugin.key] is None if self.output_path: assert workflow.data.exported_image_sequence == [{ 'md5sum': DUMMY_TARBALL['md5sum'], 'sha256sum': DUMMY_TARBALL['sha256sum'], 'size': DUMMY_TARBALL['size'], 'type': IMAGE_TYPE_DOCKER_ARCHIVE, 'path': self.output_path, }] else: assert workflow.data.exported_image_sequence == []
def test_skip_plugin(caplog, docker_tasker, user_params): workflow = DockerBuildWorkflow(source=MOCK_SOURCE) workflow.user_params = {} setattr(workflow, 'builder', MockBuilder()) runner = PrePublishPluginsRunner(docker_tasker, workflow, [{ 'name': FlatpakCreateOciPlugin.key, 'args': {} }]) runner.run() assert 'not flatpak build, skipping plugin' in caplog.text
def run_plugin_with_args(self, plugin_args): runner = PrePublishPluginsRunner( self.tasker, self.workflow, [{'name': PrePublishSquashPlugin.key, 'args': plugin_args}] ) result = runner.run() assert result[PrePublishSquashPlugin.key] is None assert self.workflow.exported_image_sequence == [{ 'md5sum': DUMMY_TARBALL['md5sum'], 'sha256sum': DUMMY_TARBALL['sha256sum'], 'size': DUMMY_TARBALL['size'], 'path': self.output_path, }]
def build_docker_image(self): """ build docker image :return: BuildResult """ self.builder = InsideBuilder(self.source, self.image) try: self.fs_watcher.start() signal.signal(signal.SIGTERM, self.throw_canceled_build_exception) # time to run pre-build plugins, so they can access cloned repo logger.info("running pre-build plugins") prebuild_runner = PreBuildPluginsRunner(self.builder.tasker, self, self.prebuild_plugins_conf, plugin_files=self.plugin_files) try: prebuild_runner.run() except PluginFailedException as ex: logger.error("one or more prebuild plugins failed: %s", ex) raise except AutoRebuildCanceledException as ex: logger.info(str(ex)) self.autorebuild_canceled = True raise logger.info("running buildstep plugins") buildstep_runner = BuildStepPluginsRunner(self.builder.tasker, self, self.buildstep_plugins_conf, plugin_files=self.plugin_files) try: self.build_result = buildstep_runner.run() if self.build_result.is_failed(): raise PluginFailedException(self.build_result.fail_reason) except PluginFailedException as ex: self.builder.is_built = False logger.error('buildstep plugin failed: %s', ex) raise self.builder.is_built = True if self.build_result.is_image_available(): self.builder.image_id = self.build_result.image_id # run prepublish plugins prepublish_runner = PrePublishPluginsRunner(self.builder.tasker, self, self.prepublish_plugins_conf, plugin_files=self.plugin_files) try: prepublish_runner.run() except PluginFailedException as ex: logger.error("one or more prepublish plugins failed: %s", ex) raise if self.build_result.is_image_available(): self.built_image_inspect = self.builder.inspect_built_image() history = self.builder.tasker.d.history(self.builder.image_id) diff_ids = self.built_image_inspect[INSPECT_ROOTFS][INSPECT_ROOTFS_LAYERS] # diff_ids is ordered oldest first # history is ordered newest first # We want layer_sizes to be ordered oldest first self.layer_sizes = [{"diff_id": diff_id, "size": layer['Size']} for (diff_id, layer) in zip(diff_ids, reversed(history))] postbuild_runner = PostBuildPluginsRunner(self.builder.tasker, self, self.postbuild_plugins_conf, plugin_files=self.plugin_files) try: postbuild_runner.run() except PluginFailedException as ex: logger.error("one or more postbuild plugins failed: %s", ex) raise return self.build_result except Exception as ex: logger.debug("caught exception (%r) so running exit plugins", ex) raise finally: # We need to make sure all exit plugins are executed signal.signal(signal.SIGTERM, lambda *args: None) exit_runner = ExitPluginsRunner(self.builder.tasker, self, self.exit_plugins_conf, plugin_files=self.plugin_files) try: exit_runner.run(keep_going=True) except PluginFailedException as ex: logger.error("one or more exit plugins failed: %s", ex) raise finally: self.source.remove_tmpdir() self.fs_watcher.finish() signal.signal(signal.SIGTERM, signal.SIG_DFL)
def test_flatpak_create_oci(tmpdir, docker_tasker, config_name, breakage, mock_flatpak): if not mock_flatpak: # Check that we actually have flatpak available have_flatpak = False try: output = subprocess.check_output(['flatpak', '--version'], universal_newlines=True) m = re.search('(\d+)\.(\d+)\.(\d+)', output) if m and (int(m.group(1)), int(m.group(2)), int( m.group(3))) >= (0, 9, 7): have_flatpak = True except (subprocess.CalledProcessError, OSError): pass if not have_flatpak: return config = CONFIGS[config_name] if mock_flatpak: (flexmock(subprocess).should_receive("check_call").replace_with( mocked_check_call)) (flexmock(subprocess).should_receive("check_output").replace_with( mocked_check_output)) workflow = DockerBuildWorkflow({ "provider": "git", "uri": "asd" }, TEST_IMAGE) setattr(workflow, 'builder', X) setattr(workflow.builder, 'tasker', docker_tasker) filesystem_dir = os.path.join(str(tmpdir), 'filesystem') os.mkdir(filesystem_dir) filesystem_contents = config['filesystem_contents'] for path, contents in filesystem_contents.items(): parts = path.split(':', 1) path = parts[0] mode = parts[1] if len(parts) == 2 else None fullpath = os.path.join(filesystem_dir, path[1:]) parent_dir = os.path.dirname(fullpath) if not os.path.isdir(parent_dir): os.makedirs(parent_dir) if contents is None: os.mkdir(fullpath) else: with open(fullpath, 'w') as f: f.write(contents) if mode is not None: os.chmod(fullpath, int(mode, 8)) if breakage == 'stray_component': fullpath = os.path.join(filesystem_dir, 'var/tmp/flatpak-build.rpm_qf') with open(fullpath, 'a') as f: f.write( "bad-rpm;1.2.3;1.fc26;x86_64;0;42;sigmd5;0;42;1491914281;sigpgp;siggpg\n" ) expected_exception = 'bad-rpm' elif breakage == 'missing_component': fullpath = os.path.join(filesystem_dir, 'var/tmp/flatpak-build.rpm_qf') with open(fullpath, 'r') as f: with open(fullpath + '.tmp', 'w') as g: f.readline() g.write(f.read()) os.rename(fullpath + '.tmp', fullpath) expected_exception = 'Installed set of packages does not match runtime profile' elif breakage == 'no_runtime': mmd = ModuleMetadata() # Copy the parts of the config we are going to change config = dict(config) config['modules'] = dict(config['modules']) config['modules']['eog'] = dict(config['modules']['eog']) module_config = config['modules']['eog'] mmd.loads(module_config['metadata']) del mmd.buildrequires['flatpak-runtime'] module_config['metadata'] = mmd.dumps() expected_exception = 'Failed to identify runtime module' else: assert breakage is None expected_exception = None filesystem_tar = os.path.join(filesystem_dir, 'tar') with open(filesystem_tar, "wb") as f: with tarfile.TarFile(fileobj=f, mode='w') as tf: for f in os.listdir(filesystem_dir): tf.add(os.path.join(filesystem_dir, f), f) export_stream = open(filesystem_tar, "rb") (flexmock( docker_tasker.d.wrapped).should_receive('create_container').with_args( workflow.image).and_return({'Id': CONTAINER_ID})) (flexmock(docker_tasker.d.wrapped).should_receive('export').with_args( CONTAINER_ID).and_return(export_stream)) (flexmock(docker_tasker.d.wrapped).should_receive( 'remove_container').with_args(CONTAINER_ID)) modules = {} for name, module_config in config['modules'].items(): mmd = ModuleMetadata() mmd.loads(module_config['metadata']) modules[name] = ModuleInfo(name, module_config['stream'], module_config['version'], mmd, module_config['rpms']) base_module = modules[config['base_module']] repo_url = 'http://odcs.example/composes/latest-odcs-42-1/compose/Temporary/$basearch/os/' compose_info = ComposeInfo(base_module.name + '-' + base_module.stream, 42, base_module, modules, repo_url) set_compose_info(workflow, compose_info) source = FlatpakSourceInfo(FLATPAK_APP_JSON, compose_info) set_flatpak_source_info(workflow, source) runner = PrePublishPluginsRunner(docker_tasker, workflow, [{ 'name': FlatpakCreateOciPlugin.key, 'args': {} }]) if expected_exception: with pytest.raises(PluginFailedException) as ex: runner.run() assert expected_exception in str(ex) else: runner.run() dir_metadata = workflow.exported_image_sequence[-2] assert dir_metadata['type'] == IMAGE_TYPE_OCI tar_metadata = workflow.exported_image_sequence[-1] assert tar_metadata['type'] == IMAGE_TYPE_OCI_TAR # Check that the expected files ended up in the flatpak if mock_flatpak: inspector = MockInspector(tmpdir, dir_metadata) else: inspector = DefaultInspector(tmpdir, dir_metadata) files = inspector.list_files() assert sorted(files) == config['expected_contents'] components = {c['name'] for c in workflow.image_components} for n in config['expected_components']: assert n in components for n in config['unexpected_components']: assert n not in components if config_name == 'app': # Check that the desktop file was rewritten output = inspector.cat_file( '/export/share/applications/org.gnome.eog.desktop') lines = output.split('\n') assert 'Icon=org.gnome.eog' in lines else: # runtime # Check that permissions have been normalized assert inspector.get_file_perms('/files/etc/shadow') == '-00644' assert inspector.get_file_perms('/files/bin/mount') == '-00755' assert inspector.get_file_perms('/files/share/foo') == 'd00755'
def test_flatpak_create_oci(tmpdir, docker_tasker, config_name, breakage, mock_flatpak): if not mock_flatpak: # Check that we actually have flatpak available have_flatpak = False try: output = subprocess.check_output(['flatpak', '--version'], universal_newlines=True) m = re.search('(\d+)\.(\d+)\.(\d+)', output) if m and (int(m.group(1)), int(m.group(2)), int( m.group(3))) >= (0, 9, 7): have_flatpak = True except (subprocess.CalledProcessError, OSError): pass if not have_flatpak: return config = CONFIGS[config_name] if mock_flatpak: (flexmock(subprocess).should_receive("check_call").replace_with( mocked_check_call)) (flexmock(subprocess).should_receive("check_output").replace_with( mocked_check_output)) workflow = DockerBuildWorkflow({ "provider": "git", "uri": "asd" }, TEST_IMAGE) setattr(workflow, 'builder', X) setattr(workflow.builder, 'tasker', docker_tasker) filesystem_dir = os.path.join(str(tmpdir), 'filesystem') os.mkdir(filesystem_dir) filesystem_contents = config['filesystem_contents'] for path, contents in filesystem_contents.items(): parts = path.split(':', 1) path = parts[0] mode = parts[1] if len(parts) == 2 else None fullpath = os.path.join(filesystem_dir, path[1:]) parent_dir = os.path.dirname(fullpath) if not os.path.isdir(parent_dir): os.makedirs(parent_dir) if contents is None: os.mkdir(fullpath) else: with open(fullpath, 'w') as f: f.write(contents) if mode is not None: os.chmod(fullpath, int(mode, 8)) if breakage == 'no_runtime': # Copy the parts of the config we are going to change config = dict(config) config['modules'] = dict(config['modules']) config['modules']['eog'] = dict(config['modules']['eog']) module_config = config['modules']['eog'] mmd = Modulemd.Module.new_from_string(module_config['metadata']) # Clear out all dependencies. Setting via the property causes a crash # https://gitlab.gnome.org/GNOME/pygobject/issues/37 # mmd.props.dependencies = [Modulemd.Dependencies()] mmd.set_dependencies([Modulemd.Dependencies()]) module_config['metadata'] = mmd.dumps() expected_exception = 'Failed to identify runtime module' else: assert breakage is None expected_exception = None filesystem_tar = os.path.join(filesystem_dir, 'tar') with open(filesystem_tar, "wb") as f: with tarfile.TarFile(fileobj=f, mode='w') as tf: for f in os.listdir(filesystem_dir): tf.add(os.path.join(filesystem_dir, f), f) export_stream = open(filesystem_tar, "rb") def stream_to_generator(s): while True: # Yield small chunks to test the StreamAdapter code better buf = s.read(100) if len(buf) == 0: return yield buf export_generator = stream_to_generator(export_stream) (flexmock( docker_tasker.d.wrapped).should_receive('create_container').with_args( workflow.image, command=["/bin/bash"]).and_return({'Id': CONTAINER_ID})) (flexmock(docker_tasker.d.wrapped).should_receive('export').with_args( CONTAINER_ID).and_return(export_generator)) (flexmock(docker_tasker.d.wrapped).should_receive( 'remove_container').with_args(CONTAINER_ID)) setup_flatpak_source_info(workflow, config) runner = PrePublishPluginsRunner(docker_tasker, workflow, [{ 'name': FlatpakCreateOciPlugin.key, 'args': {} }]) if expected_exception: with pytest.raises(PluginFailedException) as ex: runner.run() assert expected_exception in str(ex) else: runner.run() dir_metadata = workflow.exported_image_sequence[-2] assert dir_metadata['type'] == IMAGE_TYPE_OCI tar_metadata = workflow.exported_image_sequence[-1] assert tar_metadata['type'] == IMAGE_TYPE_OCI_TAR # Check that the expected files ended up in the flatpak if mock_flatpak: inspector = MockInspector(tmpdir, dir_metadata) else: inspector = DefaultInspector(tmpdir, dir_metadata) files = inspector.list_files() assert sorted(files) == config['expected_contents'] components = {c['name'] for c in workflow.image_components} for n in config['expected_components']: assert n in components for n in config['unexpected_components']: assert n not in components metadata_lines = inspector.cat_file('/metadata').split('\n') assert any( re.match(r'runtime=org.fedoraproject.Platform/.*/f28$', l) for l in metadata_lines) assert any( re.match(r'sdk=org.fedoraproject.Sdk/.*/f28$', l) for l in metadata_lines) if config_name == 'app': # Check that the desktop file was rewritten output = inspector.cat_file( '/export/share/applications/org.gnome.eog.desktop') lines = output.split('\n') assert 'Icon=org.gnome.eog' in lines assert 'name=org.gnome.eog' in metadata_lines assert 'tags=Viewer' in metadata_lines assert 'command=eog2' in metadata_lines elif config_name == 'runtime': # runtime # Check that permissions have been normalized assert inspector.get_file_perms('/files/etc/shadow') == '-00644' assert inspector.get_file_perms('/files/bin/mount') == '-00755' assert inspector.get_file_perms('/files/share/foo') == 'd00755' assert 'name=org.fedoraproject.Platform' in metadata_lines else: # SDK assert 'name=org.fedoraproject.Sdk' in metadata_lines
def build_docker_image(self): """ build docker image :return: BuildResults """ self.builder = InsideBuilder(self.source, self.image) try: # time to run pre-build plugins, so they can access cloned repo logger.info("running pre-build plugins") prebuild_runner = PreBuildPluginsRunner( self.builder.tasker, self, self.prebuild_plugins_conf, plugin_files=self.plugin_files) try: prebuild_runner.run() except PluginFailedException as ex: logger.error("one or more prebuild plugins failed: %s", ex) raise build_result = self.builder.build() self.build_logs = build_result.logs self.build_failed = build_result.is_failed() if not build_result.is_failed(): self.built_image_inspect = self.builder.inspect_built_image() # run prepublish plugins prepublish_runner = PrePublishPluginsRunner( self.builder.tasker, self, self.prepublish_plugins_conf, plugin_files=self.plugin_files) try: prepublish_runner.run() except PluginFailedException as ex: logger.error("one or more prepublish plugins failed: %s", ex) raise if not build_result.is_failed(): for registry in self.push_conf.docker_registries: self.builder.push_built_image(registry.uri, insecure=registry.insecure) postbuild_runner = PostBuildPluginsRunner( self.builder.tasker, self, self.postbuild_plugins_conf, plugin_files=self.plugin_files) try: postbuild_runner.run() except PluginFailedException as ex: logger.error("one or more postbuild plugins failed: %s", ex) raise return build_result finally: self.source.remove_tmpdir() exit_runner = ExitPluginsRunner(self.builder.tasker, self, self.exit_plugins_conf, plugin_files=self.plugin_files) try: exit_runner.run() except PluginFailedException as ex: logger.error("one or more exit plugins failed: %s", ex)
def build_docker_image(self): """ build docker image :return: BuildResult """ self.builder = InsideBuilder(self.source, self.image) try: signal.signal(signal.SIGTERM, self.throw_canceled_build_exception) # time to run pre-build plugins, so they can access cloned repo logger.info("running pre-build plugins") prebuild_runner = PreBuildPluginsRunner( self.builder.tasker, self, self.prebuild_plugins_conf, plugin_files=self.plugin_files) try: prebuild_runner.run() except PluginFailedException as ex: logger.error("one or more prebuild plugins failed: %s", ex) raise except AutoRebuildCanceledException as ex: logger.info(str(ex)) self.autorebuild_canceled = True raise logger.info("running buildstep plugins") buildstep_runner = BuildStepPluginsRunner( self.builder.tasker, self, self.buildstep_plugins_conf, plugin_files=self.plugin_files) try: self.build_result = buildstep_runner.run() if self.build_result.is_failed(): raise PluginFailedException(self.build_result.fail_reason) except PluginFailedException as ex: self.builder.is_built = False logger.error('buildstep plugin failed: %s', ex) raise self.builder.is_built = True if self.build_result.is_image_available(): self.builder.image_id = self.build_result.image_id self.built_image_inspect = self.builder.inspect_built_image() # run prepublish plugins prepublish_runner = PrePublishPluginsRunner( self.builder.tasker, self, self.prepublish_plugins_conf, plugin_files=self.plugin_files) try: prepublish_runner.run() except PluginFailedException as ex: logger.error("one or more prepublish plugins failed: %s", ex) raise postbuild_runner = PostBuildPluginsRunner( self.builder.tasker, self, self.postbuild_plugins_conf, plugin_files=self.plugin_files) try: postbuild_runner.run() except PluginFailedException as ex: logger.error("one or more postbuild plugins failed: %s", ex) raise return self.build_result except Exception as ex: logger.debug("caught exception (%r) so running exit plugins", ex) raise finally: # We need to make sure all exit plugins are executed signal.signal(signal.SIGTERM, lambda *args: None) exit_runner = ExitPluginsRunner(self.builder.tasker, self, self.exit_plugins_conf, plugin_files=self.plugin_files) try: exit_runner.run(keep_going=True) except PluginFailedException as ex: logger.error("one or more exit plugins failed: %s", ex) raise finally: self.source.remove_tmpdir() signal.signal(signal.SIGTERM, signal.SIG_DFL)
def build_docker_image(self): """ build docker image :return: BuildResult """ exception_being_handled = False self.builder = InsideBuilder(self.source, self.image) # Make sure exit_runner is defined for finally block exit_runner = None try: self.fs_watcher.start() signal.signal(signal.SIGTERM, self.throw_canceled_build_exception) prebuild_runner = PreBuildPluginsRunner( self.builder.tasker, self, self.prebuild_plugins_conf, plugin_files=self.plugin_files) prepublish_runner = PrePublishPluginsRunner( self.builder.tasker, self, self.prepublish_plugins_conf, plugin_files=self.plugin_files) postbuild_runner = PostBuildPluginsRunner( self.builder.tasker, self, self.postbuild_plugins_conf, plugin_files=self.plugin_files) # time to run pre-build plugins, so they can access cloned repo logger.info("running pre-build plugins") try: prebuild_runner.run() except PluginFailedException as ex: logger.error("one or more prebuild plugins failed: %s", ex) raise except AutoRebuildCanceledException as ex: logger.info(str(ex)) self.autorebuild_canceled = True raise # we are delaying initialization, because prebuild plugin reactor_config # might change build method buildstep_runner = BuildStepPluginsRunner( self.builder.tasker, self, self.buildstep_plugins_conf, plugin_files=self.plugin_files) logger.info("running buildstep plugins") try: self.build_result = buildstep_runner.run() if self.build_result.is_failed(): raise PluginFailedException(self.build_result.fail_reason) except PluginFailedException as ex: self.builder.is_built = False logger.error('buildstep plugin failed: %s', ex) raise self.builder.is_built = True if self.build_result.is_image_available(): self.builder.image_id = self.build_result.image_id # run prepublish plugins try: prepublish_runner.run() except PluginFailedException as ex: logger.error("one or more prepublish plugins failed: %s", ex) raise if self.build_result.is_image_available(): self.built_image_inspect = self.builder.inspect_built_image() history = self.builder.tasker.get_image_history( self.builder.image_id) diff_ids = self.built_image_inspect[INSPECT_ROOTFS][ INSPECT_ROOTFS_LAYERS] # diff_ids is ordered oldest first # history is ordered newest first # We want layer_sizes to be ordered oldest first self.layer_sizes = [{ "diff_id": diff_id, "size": layer['Size'] } for (diff_id, layer) in zip(diff_ids, reversed(history))] try: postbuild_runner.run() except PluginFailedException as ex: logger.error("one or more postbuild plugins failed: %s", ex) raise return self.build_result except Exception as ex: logger.debug("caught exception (%s) so running exit plugins", exception_message(ex)) exception_being_handled = True raise finally: # We need to make sure all exit plugins are executed signal.signal(signal.SIGTERM, lambda *args: None) exit_runner = ExitPluginsRunner(self.builder.tasker, self, self.exit_plugins_conf, keep_going=True, plugin_files=self.plugin_files) try: exit_runner.run(keep_going=True) except PluginFailedException as ex: logger.error("one or more exit plugins failed: %s", ex) # raise exception only in case that there is no previous exception being already # handled to prevent replacing original exceptions (root cause) with exceptions # from exit plugins if not exception_being_handled: raise ex finally: self.source.remove_tmpdir() self.fs_watcher.finish() signal.signal(signal.SIGTERM, signal.SIG_DFL)
def build_docker_image(self): """ build docker image :return: BuildResults """ self.builder = InsideBuilder(self.source, self.image) try: signal.signal(signal.SIGTERM, self.throw_canceled_build_exception) # time to run pre-build plugins, so they can access cloned repo logger.info("running pre-build plugins") prebuild_runner = PreBuildPluginsRunner( self.builder.tasker, self, self.prebuild_plugins_conf, plugin_files=self.plugin_files) try: prebuild_runner.run() except PluginFailedException as ex: logger.error("one or more prebuild plugins failed: %s", ex) raise except AutoRebuildCanceledException as ex: logger.info(str(ex)) self.autorebuild_canceled = True raise start_time = datetime.datetime.now() self.plugins_timestamps['dockerbuild'] = start_time.isoformat() build_result = self.builder.build() try: finish_time = datetime.datetime.now() duration = finish_time - start_time seconds = duration.total_seconds() logger.debug("build finished in %ds", seconds) self.plugins_durations['dockerbuild'] = seconds except Exception: logger.exception("failed to save build duration") self.build_logs = build_result.logs self.build_failed = build_result.is_failed() if build_result.is_failed(): # The docker build failed. Finish here, just run the # exit plugins (from the 'finally:' block below). self.plugins_errors['dockerbuild'] = '' return build_result self.built_image_inspect = self.builder.inspect_built_image() # run prepublish plugins prepublish_runner = PrePublishPluginsRunner( self.builder.tasker, self, self.prepublish_plugins_conf, plugin_files=self.plugin_files) try: prepublish_runner.run() except PluginFailedException as ex: logger.error("one or more prepublish plugins failed: %s", ex) raise postbuild_runner = PostBuildPluginsRunner( self.builder.tasker, self, self.postbuild_plugins_conf, plugin_files=self.plugin_files) try: postbuild_runner.run() except PluginFailedException as ex: logger.error("one or more postbuild plugins failed: %s", ex) raise return build_result finally: # We need to make sure all exit plugins are executed signal.signal(signal.SIGTERM, lambda *args: None) exit_runner = ExitPluginsRunner(self.builder.tasker, self, self.exit_plugins_conf, plugin_files=self.plugin_files) try: exit_runner.run(keep_going=True) except PluginFailedException as ex: logger.error("one or more exit plugins failed: %s", ex) raise finally: self.source.remove_tmpdir() signal.signal(signal.SIGTERM, signal.SIG_DFL)
def build_docker_image(self) -> None: """ build docker image """ print_version_of_tools() exception_being_handled = False # Make sure exit_runner is defined for finally block exit_runner = None try: self.fs_watcher.start() signal.signal(signal.SIGTERM, self.throw_canceled_build_exception) prebuild_runner = PreBuildPluginsRunner( self, self.plugins.prebuild, plugin_files=self.plugin_files) prepublish_runner = PrePublishPluginsRunner( self, self.plugins.prepublish, plugin_files=self.plugin_files) postbuild_runner = PostBuildPluginsRunner( self, self.plugins.postbuild, plugin_files=self.plugin_files) # time to run pre-build plugins, so they can access cloned repo logger.info("running pre-build plugins") try: prebuild_runner.run() except PluginFailedException as ex: logger.error("one or more prebuild plugins failed: %s", ex) raise # we are delaying initialization, because prebuild plugin reactor_config # might change build method buildstep_runner = BuildStepPluginsRunner( self, self.plugins.buildstep, plugin_files=self.plugin_files) logger.info("running buildstep plugins") try: buildstep_runner.run() except PluginFailedException as ex: logger.error('buildstep plugin failed: %s', ex) raise # run prepublish plugins try: prepublish_runner.run() except PluginFailedException as ex: logger.error("one or more prepublish plugins failed: %s", ex) raise try: postbuild_runner.run() except PluginFailedException as ex: logger.error("one or more postbuild plugins failed: %s", ex) raise except Exception as ex: logger.debug("caught exception (%s) so running exit plugins", exception_message(ex)) exception_being_handled = True raise finally: # We need to make sure all exit plugins are executed signal.signal(signal.SIGTERM, lambda *args: None) exit_runner = ExitPluginsRunner(self, self.plugins.exit, keep_going=True, plugin_files=self.plugin_files) try: exit_runner.run(keep_going=True) except PluginFailedException as ex: logger.error("one or more exit plugins failed: %s", ex) # raise exception only in case that there is no previous exception being already # handled to prevent replacing original exceptions (root cause) with exceptions # from exit plugins if not exception_being_handled: raise ex finally: self.fs_watcher.finish() signal.signal(signal.SIGTERM, signal.SIG_DFL)
def test_flatpak_create_oci(tmpdir, docker_tasker, config_name, flatpak_metadata, breakage): # Check that we actually have flatpak available have_flatpak = False try: output = subprocess.check_output(['flatpak', '--version'], universal_newlines=True) m = re.search(r'(\d+)\.(\d+)\.(\d+)', output) if m and (int(m.group(1)), int(m.group(2)), int( m.group(3))) >= (0, 9, 7): have_flatpak = True except (subprocess.CalledProcessError, OSError): pytest.skip(msg='flatpak not available') if not have_flatpak: return config = CONFIGS[config_name] workflow = DockerBuildWorkflow(TEST_IMAGE, source={ "provider": "git", "uri": "asd" }) setattr(workflow, 'builder', X) X.df_path = write_docker_file(config, str(tmpdir)) setattr(workflow.builder, 'tasker', docker_tasker) make_and_store_reactor_config_map(workflow, flatpak_metadata) filesystem_dir = os.path.join(str(tmpdir), 'filesystem') os.mkdir(filesystem_dir) filesystem_contents = config['filesystem_contents'] for path, contents in filesystem_contents.items(): parts = path.split(':', 1) path = parts[0] mode = parts[1] if len(parts) == 2 else None fullpath = os.path.join(filesystem_dir, path[1:]) parent_dir = os.path.dirname(fullpath) if not os.path.isdir(parent_dir): os.makedirs(parent_dir) if contents is None: os.mkdir(fullpath) else: with open(fullpath, 'wb') as f: f.write(contents) if mode is not None: os.chmod(fullpath, int(mode, 8)) if breakage == 'no_runtime': # Copy the parts of the config we are going to change config = dict(config) config['modules'] = dict(config['modules']) config['modules']['eog'] = dict(config['modules']['eog']) module_config = config['modules']['eog'] mmd = Modulemd.ModuleStream.read_string(module_config['metadata'], strict=True) mmd.clear_dependencies() mmd.add_dependencies(Modulemd.Dependencies()) mmd_index = Modulemd.ModuleIndex.new() mmd_index.add_module_stream(mmd) module_config['metadata'] = mmd_index.dump_to_string() expected_exception = 'Failed to identify runtime module' else: assert breakage is None expected_exception = None filesystem_tar = os.path.join(filesystem_dir, 'tar') with open(filesystem_tar, "wb") as f: with tarfile.TarFile(fileobj=f, mode='w') as tf: for f in os.listdir(filesystem_dir): tf.add(os.path.join(filesystem_dir, f), f) export_stream = open(filesystem_tar, "rb") def stream_to_generator(s): while True: # Yield small chunks to test the StreamAdapter code better buf = s.read(100) if len(buf) == 0: return yield buf export_generator = stream_to_generator(export_stream) (flexmock( docker_tasker.tasker).should_receive('export_container').with_args( CONTAINER_ID).and_return(export_generator)) (flexmock(docker_tasker.tasker.d.wrapped).should_receive( 'create_container').with_args(workflow.image, command=["/bin/bash"]).and_return( {'Id': CONTAINER_ID})) (flexmock(docker_tasker.tasker.d.wrapped).should_receive( 'remove_container').with_args(CONTAINER_ID, force=False)) setup_flatpak_source_info(workflow, config) runner = PrePublishPluginsRunner(docker_tasker, workflow, [{ 'name': FlatpakCreateOciPlugin.key, 'args': {} }]) if expected_exception: with pytest.raises(PluginFailedException) as ex: runner.run() assert expected_exception in str(ex.value) else: runner.run() dir_metadata = workflow.exported_image_sequence[-2] assert dir_metadata['type'] == IMAGE_TYPE_OCI tar_metadata = workflow.exported_image_sequence[-1] assert tar_metadata['type'] == IMAGE_TYPE_OCI_TAR # Check that the correct labels and annotations were written labels, annotations = load_labels_and_annotations(dir_metadata) if config_name == 'app': assert labels['name'] == 'eog' assert labels['com.redhat.component'] == 'eog' assert labels['version'] == 'f28' assert labels['release'] == '20170629213428' elif config_name == 'runtime': # runtime assert labels['name'] == 'flatpak-runtime' assert labels[ 'com.redhat.component'] == 'flatpak-runtime-container' assert labels['version'] == 'f28' assert labels['release'] == '20170701152209' else: assert labels['name'] == 'flatpak-sdk' assert labels['com.redhat.component'] == 'flatpak-sdk-container' assert labels['version'] == 'f28' assert labels['release'] == '20170701152209' if flatpak_metadata == 'annotations': assert annotations.get( 'org.flatpak.ref') == config['expected_ref_name'] assert 'org.flatpak.ref' not in labels elif flatpak_metadata == 'labels': assert 'org.flatpak.ref' not in annotations assert labels.get('org.flatpak.ref') == config['expected_ref_name'] elif flatpak_metadata == 'both': assert annotations.get( 'org.flatpak.ref') == config['expected_ref_name'] assert labels.get('org.flatpak.ref') == config['expected_ref_name'] # Check that the expected files ended up in the flatpak # Flatpak versions before 1.6 require annotations to be present, since we don't # require such a new Flatpak, skip remaining checks in the label-only case if flatpak_metadata == 'labels': return inspector = DefaultInspector(tmpdir, dir_metadata) files = inspector.list_files() assert sorted(files) == config['expected_contents'] components = {c['name'] for c in workflow.image_components} # noqa:E501; pylint: disable=not-an-iterable for n in config['expected_components']: assert n in components for n in config['unexpected_components']: assert n not in components metadata_lines = inspector.cat_file('/metadata').split('\n') assert any( re.match(r'runtime=org.fedoraproject.Platform/.*/f28$', l) for l in metadata_lines) assert any( re.match(r'sdk=org.fedoraproject.Sdk/.*/f28$', l) for l in metadata_lines) if config_name == 'app': # Check that the desktop file was rewritten output = inspector.cat_file( '/export/share/applications/org.gnome.eog.desktop') lines = output.split('\n') assert 'Icon=org.gnome.eog' in lines assert 'name=org.gnome.eog' in metadata_lines assert 'tags=Viewer' in metadata_lines assert 'command=eog2' in metadata_lines elif config_name == 'runtime': # runtime # Check that permissions have been normalized assert inspector.get_file_perms('/files/etc/shadow') == '-00644' assert inspector.get_file_perms('/files/bin/mount') == '-00755' assert inspector.get_file_perms('/files/share/foo') == 'd00755' assert 'name=org.fedoraproject.Platform' in metadata_lines else: # SDK assert 'name=org.fedoraproject.Sdk' in metadata_lines
def build_docker_image(self): """ build docker image :return: BuildResults """ self.builder = InsideBuilder(self.source, self.image) try: # time to run pre-build plugins, so they can access cloned repo logger.info("running pre-build plugins") prebuild_runner = PreBuildPluginsRunner( self.builder.tasker, self, self.prebuild_plugins_conf, plugin_files=self.plugin_files) try: prebuild_runner.run() except PluginFailedException as ex: logger.error("one or more prebuild plugins failed: %s", ex) raise except AutoRebuildCanceledException as ex: logger.info(str(ex)) self.autorebuild_canceled = True raise build_result = self.builder.build() self.build_logs = build_result.logs self.build_failed = build_result.is_failed() if build_result.is_failed(): # The docker build failed. Finish here, just run the # exit plugins (from the 'finally:' block below). return build_result self.built_image_inspect = self.builder.inspect_built_image() # run prepublish plugins prepublish_runner = PrePublishPluginsRunner( self.builder.tasker, self, self.prepublish_plugins_conf, plugin_files=self.plugin_files) try: prepublish_runner.run() except PluginFailedException as ex: logger.error("one or more prepublish plugins failed: %s", ex) raise postbuild_runner = PostBuildPluginsRunner( self.builder.tasker, self, self.postbuild_plugins_conf, plugin_files=self.plugin_files) try: postbuild_runner.run() except PluginFailedException as ex: logger.error("one or more postbuild plugins failed: %s", ex) raise return build_result finally: exit_runner = ExitPluginsRunner(self.builder.tasker, self, self.exit_plugins_conf, plugin_files=self.plugin_files) try: exit_runner.run(keep_going=True) except PluginFailedException as ex: logger.error("one or more exit plugins failed: %s", ex) finally: self.source.remove_tmpdir()