def build_srpm(self) -> types.TaskDict: """Build the SRPM for the package.""" env = { 'SPEC': self.spec.name, 'SRPM': self.srpm.name, 'SOURCES': ' '.join(source.name for source in self.sources), 'VERSION': self.version, } buildsrpm_callable = docker_command.DockerRun( command=['/entrypoint.sh', 'buildsrpm'], builder=self.builder, environment=env, tmpfs={ '/home/build': '', '/var/tmp': '' }, mounts=self._get_buildsrpm_mounts(self.srpm.parent), read_only=True, run_config=docker_command.RPM_BASE_CONFIG) task = self.basic_task task.update({ 'name': 'pkg_srpm', 'actions': [buildsrpm_callable], 'doc': 'Build {}'.format(self.srpm.name), 'title': utils.title_with_target1('BUILD SRPM'), 'targets': [self.srpm], # Prevent Docker from polluting our output. 'verbosity': 0, }) task['file_dep'].extend([self.spec]) task['file_dep'].extend(self.sources) task['task_dep'].append('{}:{}'.format(self.basename, self.MKDIR_TASK_NAME)) return task
def build_package(self) -> types.TaskDict: """Build DEB packages from source files.""" mounts = [ utils.bind_ro_mount(source=self.sources, target=Path('/debbuild/pkg-src')), utils.bind_ro_mount(source=self.debuild_sources, target=Path('/debbuild/pkg-meta')), utils.bind_mount(source=constants.PKG_DEB_ROOT, target=Path('/debbuild/result')), ] builddeb_callable = docker_command.DockerRun( command=['/entrypoint.sh', 'builddeb'], builder=self.builder, run_config=docker_command.DEB_BASE_CONFIG, mounts=mounts, environment={ 'VERSION': '{}-{}'.format(self.version, self.build_id) }, ) task = self.basic_task task.update({ 'name': 'build_deb_pkg', 'actions': [builddeb_callable], 'doc': 'Build DEB package from sources for {}'.format(self.name), 'title': utils.title_with_target1('BUILD DEB'), 'targets': [self.deb], }) task['file_dep'].extend(coreutils.ls_files_rec(self.sources)) task['file_dep'].extend(coreutils.ls_files_rec(self.debuild_sources)) task['task_dep'].append('_package_mkdir_deb_iso_root') task['task_dep'].append('_build_deb_container') return task
def task__iso_generate_product_txt() -> types.TaskDict: """Generate the product.txt file.""" def action(targets: Sequence[str]) -> None: datefmt = "%Y-%m-%dT%H:%M:%SZ" dev_release = '1' if constants.VERSION_SUFFIX == '-dev' else '0' info = ( ('NAME', config.PROJECT_NAME), ('VERSION', constants.VERSION), ('SHORT_VERSION', constants.SHORT_VERSION), ('GIT', constants.GIT_REF or ''), ('DEVELOPMENT_RELEASE', dev_release), ('BUILD_TIMESTAMP', dt.datetime.utcnow().strftime(datefmt)), ('BUILD_HOST', socket.gethostname()), ) with open(targets[0], 'w', encoding='utf-8') as fp: data = '\n'.join('{}={}'.format(key, value) for key, value in info) fp.write(data) fp.write('\n') return { 'title': lambda task: utils.title_with_target1('GENERATE', task), 'actions': [action], 'targets': [constants.ISO_ROOT / 'product.txt'], 'file_dep': [constants.VERSION_FILE], 'task_dep': ['_iso_mkdir_root'], 'uptodate': [False], # False because we include the build timestamp. 'clean': True, }
def convert_package(self) -> types.TaskDict: """Build a DEB package from a RPM one.""" mounts = [ utils.bind_ro_mount(source=self.sources, target=Path('/rpmbuild/source.rpm')), utils.bind_mount(source=constants.PKG_DEB_ROOT, target=Path('/debbuild/result')), ] builddeb_callable = docker_command.DockerRun( command=['/entrypoint.sh', 'rpm2deb'], builder=self.builder, run_config=docker_command.DEB_BASE_CONFIG, mounts=mounts) task = self.basic_task task.update({ 'name': 'convert_rpm_pkg_to_deb', 'actions': [builddeb_callable], 'doc': 'Build DEB package from RPM for {}'.format(self.name), 'title': utils.title_with_target1('RPM2DEB'), 'targets': [self.deb], }) task['file_dep'].append(self.sources) task['task_dep'].append('_package_mkdir_deb_iso_root') task['task_dep'].append('_build_deb_container') return task
def build_repo(self) -> types.TaskDict: def clean() -> None: """Delete the repository directory and its contents.""" coreutils.rm_rf(self.rootdir) mkdir = self._mkdir_repo_root() actions = mkdir['actions'] actions.append(self._buildrepo_action()) targets = [ Path(self.rootdir, 'dists/bionic', self.fullname, 'binary-amd64/Packages') ] targets.extend(mkdir['targets']) task = self.basic_task task.update({ 'name': 'build_repo', 'actions': actions, 'doc': 'Build the {} repository.'.format(self.name), 'title': utils.title_with_target1('BUILD DEB REPO'), 'targets': targets, 'uptodate': [True], 'clean': [clean], }) for pkg in self.packages: task['file_dep'].append(pkg.deb) return task
def build_repo(self) -> types.TaskDict: """Build the repository.""" def clean() -> None: """Delete the repository metadata directory and its contents.""" coreutils.rm_rf(self.repodata) mkdir = directory.Mkdir(directory=self.repodata).task actions = mkdir['actions'] actions.append(self._buildrepo_action()) targets = [self.repodata/'repomd.xml'] targets.extend(mkdir['targets']) task = self.basic_task task.update({ 'name': self._get_task_name('build_repodata'), 'actions': actions, 'doc': 'Build the {} repository metadata.'.format(self.name), 'title': utils.title_with_target1('BUILD RPM REPO'), 'targets': targets, 'uptodate': [True], 'clean': [clean], # Prevent Docker from polluting our output. 'verbosity': 0, }) if self.packages: task['task_dep'].append(self._get_task_name(MKDIR_ROOT_TASK_NAME, with_basename=True)) task['file_dep'].extend([ self.get_rpm_path(pkg) for pkg in self.packages ]) return task
def task_doc() -> Iterator[types.TaskDict]: """Generate the documentation.""" def clean(target: DocTarget) -> Callable[[], None]: """Delete the build sub-directory for the given target.""" return lambda: coreutils.rm_rf(target.target.parent) doc_targets = (DocTarget(name='html', command='html', target=DOC_BUILD_DIR / 'html/index.html'), DocTarget(name='pdf', command='latexpdf', target=DOC_PDF_FILE)) for target in doc_targets: doc_format = target.name.upper() action = ['tox', '-e', 'docs', '--', target.command] # In CI we reuse already built PDF: just check the existence. if config.RUNNING_IN_CI: action = ['test', '-f', target.target] yield { 'name': target.name, 'title': utils.title_with_target1('DOC {}'.format(doc_format)), 'doc': 'Generate {} {} documentation'.format(config.PROJECT_NAME, doc_format), 'actions': [action], 'targets': [target.target], 'file_dep': list(utils.git_ls('docs')), 'clean': [clean(target)], }
def task(self) -> types.TaskDict: task = self.basic_task task.update({ 'title': lambda task: utils.title_with_target1('MKDIR', task), 'actions': [(self._run, [task['targets'][0]])], 'uptodate': [True], }) return task
def task(self) -> types.TaskDict: task = self.basic_task task.update({ 'title': utils.title_with_target1('RENDER'), 'doc': 'Render template {}.'.format(self._src.name), 'actions': [self._run], }) return task
def task(self) -> types.TaskDict: task = self.basic_task doc = ' '.join(self.__doc__.split()) if self.__doc__ else None task.update({ 'title': utils.title_with_target1('NGINX_CFG'), 'doc': doc, 'actions': [self._run], }) return task
def task(self) -> types.TaskDict: task = self.basic_task task.update({ 'title': utils.title_with_target1('RENDER {}'.format(self._renderer.value)), 'doc': 'Render file "{}" with "{}"'.format(self._dest, self._renderer), 'actions': [self._run], }) return task
def task__download_deb_packages() -> types.TaskDict: """Download Debian packages locally.""" witness = constants.PKG_DEB_ROOT / '.witness' def clean() -> None: """Delete downloaded Debian packages.""" for repository in DEB_REPOSITORIES: # Repository with an explicit list of packages are created by a # dedicated task that will also handle their cleaning, so we skip # them here. if repository.packages: continue coreutils.rm_rf(repository.pkgdir) utils.unlink_if_exist(witness) constants.REPO_DEB_ROOT.rmdir() def mkdirs() -> None: """Create directories for the repositories.""" for repository in DEB_REPOSITORIES: repository.pkgdir.mkdir(exist_ok=True) mounts = [ utils.bind_ro_mount( source=constants.ROOT / 'packages' / 'debian' / 'download_packages.py', target=Path('/download_packages.py'), ), utils.bind_mount(source=constants.PKG_DEB_ROOT, target=Path('/repositories')), ] dl_packages_callable = docker_command.DockerRun( command=['/download_packages.py', *DEB_TO_DOWNLOAD], builder=builder.DEB_BUILDER, mounts=mounts, environment={'SALT_VERSION': versions.SALT_VERSION}, run_config=docker_command.default_run_config( constants.DEBIAN_ENTRYPOINT)) return { 'title': utils.title_with_target1('GET DEB PKGS'), 'actions': [mkdirs, dl_packages_callable], 'targets': [constants.PKG_DEB_ROOT / '.witness'], 'task_dep': [ '_package_mkdir_deb_root', '_package_mkdir_deb_iso_root', '_build_builder:{}'.format(builder.DEB_BUILDER.name), ], 'clean': [clean], 'uptodate': [doit.tools.config_changed(_TO_DOWNLOAD_DEB_CONFIG)], # Prevent Docker from polluting our output. 'verbosity': 0, }
def task__doc_deploy() -> types.TaskDict: """Deploy the documentation on the ISO.""" source = DOC_PDF_FILE target = constants.ISO_DOCS_ROOT / 'user-guide.pdf' return { 'title': utils.title_with_target1('COPY'), 'actions': [(coreutils.cp_file, (source, target))], 'targets': [target], 'task_dep': ['_doc_mkdir_root'], 'file_dep': [source], 'clean': True, }
def get_source_files(self) -> types.TaskDict: """Download the source files to build the package.""" targets = [self.srcdir] targets.extend(self.sources) actions = directory.Mkdir(directory=self.srcdir).task['actions'] actions.append(self._get_sources) task = self.basic_task task.update({ 'name': 'pkg_get_source', 'actions': actions, 'doc': 'Download source files for {}.'.format(self.name), 'title': utils.title_with_target1('GET_SRC'), 'targets': targets, }) task['file_dep'].append(self.meta) task['task_dep'].append('{}:{}'.format(self.basename, self.MKDIR_TASK_NAME)) return task
def build_rpms(self) -> List[types.TaskDict]: """Build the RPMs from SRPMs.""" tasks = [self._mkdir_repo_root(), self._mkdir_repo_arch()] for pkg in self.packages: rpm = self._get_rpm_path(pkg) env = { 'RPMS': '{arch}/{rpm}'.format(arch=self.ARCH, rpm=rpm.name), 'SRPM': pkg.srpm.name, } buildrpm_callable = docker_command.DockerRun( command=['/entrypoint.sh', 'buildrpm'], builder=self.builder, environment=env, mounts=self._get_buildrpm_mounts(pkg.srpm, rpm.parent), tmpfs={ '/home/build': '', '/var/tmp': '' }, ) task = self.basic_task task.update({ 'name': 'build_rpm:{}'.format(pkg.name), 'actions': [buildrpm_callable], 'doc': 'Build {pkg} RPM for the {repo} repository.'.format( pkg=pkg.name, repo=self.name), 'title': lambda task: utils.title_with_target1('BUILD RPM', task), 'targets': [self._get_rpm_path(pkg)], # Prevent Docker from polluting our output. 'verbosity': 0, }) task['file_dep'].append(pkg.srpm) task['task_dep'].append('{base}:{name}'.format( base=self.basename, name=MKDIR_ARCH_TASK_NAME, )) task['task_dep'].append('_build_container') tasks.append(task) return tasks
def task__download_rpm_packages() -> types.TaskDict: """Download packages locally.""" def clean() -> None: """Delete cache and repositories on the ISO.""" coreutils.rm_rf(constants.PKG_RPM_ROOT / 'var') for repository in RPM_REPOSITORIES: # Repository with an explicit list of packages are created by a # dedicated task that will also handle their cleaning, so we skip # them here. if repository.packages: continue coreutils.rm_rf(repository.rootdir) mounts = [ utils.bind_mount(source=constants.PKG_RPM_ROOT, target=Path('/install_root')), utils.bind_mount(source=constants.REPO_RPM_ROOT, target=Path('/repositories')), ] dl_packages_callable = docker_command.DockerRun( command=['/entrypoint.sh', 'download_packages', *RPM_TO_DOWNLOAD], builder=builder.RPM_BUILDER, mounts=mounts, environment={'RELEASEVER': 7}, run_config=docker_command.default_run_config( constants.REDHAT_ENTRYPOINT)) return { 'title': utils.title_with_target1('GET RPM PKGS'), 'actions': [dl_packages_callable], 'targets': [constants.PKG_RPM_ROOT / 'var'], 'task_dep': [ '_package_mkdir_rpm_root', '_package_mkdir_rpm_iso_root', '_build_builder:{}'.format(builder.RPM_BUILDER.name), ], 'clean': [clean], 'uptodate': [doit.tools.config_changed(_TO_DOWNLOAD_RPM_CONFIG)], # Prevent Docker from polluting our output. 'verbosity': 0, }
def make_directories(self) -> types.TaskDict: """Return a task that create a directory hierarchy.""" def mkdirs(targets: Sequence[str]) -> None: for directory in reversed(targets): Path(directory).mkdir(exist_ok=True) task = self.basic_task task.update({ 'name': MAKE_TASK_NAME, 'doc': 'Create directory hierarchy for {}.'.format(self._root), 'title': lambda task: utils.title_with_target1('MKTREE', task), 'actions': [mkdirs], 'targets': self.directories, 'uptodate': [True], }) return task
def generate_meta(self) -> types.TaskDict: """Generate the .meta file for the package.""" spec_guest_file = Path('/rpmbuild/SPECS', self.spec.name) meta_guest_file = Path('/rpmbuild/META', self.meta.name) mounts = [ docker_command.DockerRun.ENTRYPOINT_MOUNT, docker_command.bind_ro_mount(source=self.spec, target=spec_guest_file), docker_command.bind_mount(source=self.meta.parent, target=meta_guest_file.parent) ] command = ['/entrypoint.sh', 'buildmeta'] rpmspec_config = { 'hostname': 'build', 'read_only': True, 'remove': True } buildmeta_callable = docker_command.DockerRun( command=command, builder=self.builder, environment={ 'SPEC': self.spec.name, 'META': self.meta.name }, run_config=rpmspec_config, mounts=mounts) task = self.basic_task task.update({ 'name': 'pkg_rpmspec', 'actions': [buildmeta_callable], 'doc': 'Generate {}.meta'.format(self.name), 'title': lambda task: utils.title_with_target1('RPMSPEC', task), 'targets': [self.meta], }) task['file_dep'].extend([self.spec]) task['task_dep'].append('{}:{}'.format(self.basename, self.MKDIR_TASK_NAME)) return task
def task__download_packages() -> types.TaskDict: """Download packages locally.""" def clean() -> None: """Delete cache and repositories on the ISO.""" coreutils.rm_rf(constants.PKG_ROOT / 'var') for repository in REPOSITORIES: # Repository with an explicit list of packages are created by a # dedicated task that will also handle their cleaning, so we skip # them here. if repository.packages: continue coreutils.rm_rf(repository.rootdir) pkg_list = constants.ROOT / 'packages/packages.list' packages = _load_package_list(pkg_list) mounts = [ docker_command.bind_mount(source=constants.PKG_ROOT, target=Path('/install_root')), docker_command.bind_mount(source=constants.REPO_ROOT, target=Path('/repositories')), ] dl_packages_callable = docker_command.DockerRun( command=['/entrypoint.sh', 'download_packages', *packages], builder=BUILDER, mounts=mounts, environment={'RELEASEVER': 7}) return { 'title': lambda task: utils.title_with_target1('GET PKGS', task), 'actions': [dl_packages_callable], 'targets': [constants.PKG_ROOT / 'var'], 'file_dep': [pkg_list], 'task_dep': ['_package_mkdir_root', '_package_mkdir_iso_root', '_build_container'], 'clean': [clean], 'uptodate': [True], # Prevent Docker from polluting our output. 'verbosity': 0, }
def task_doc() -> Iterator[types.TaskDict]: """Generate the documentation.""" def clean(target: DocTarget) -> Callable[[], None]: """Delete the build sub-directory for the given target.""" return lambda: coreutils.rm_rf(target.target.parent) doc_targets = (DocTarget(name='html', command='html', target=DOC_BUILD_DIR / 'html/index.html'), DocTarget(name='pdf', command='latexpdf', target=DOC_PDF_FILE)) for target in doc_targets: doc_format = target.name.upper() run_config = docker_command.default_run_config(constants.ROOT / 'docs/entrypoint.sh') # The builder stores the tox env in /tmp, don't shadow it! run_config.pop('tmpfs', None) build_doc = docker_command.DockerRun( command=['/entrypoint.sh', target.command], builder=builder.DOC_BUILDER, run_config=run_config, mounts=[ utils.bind_mount(target=Path('/usr/src/metalk8s/'), source=constants.ROOT) ]) yield { 'name': target.name, 'title': utils.title_with_target1('DOC {}'.format(doc_format)), 'doc': 'Generate {} {} documentation'.format(config.PROJECT_NAME, doc_format), 'actions': [build_doc], 'targets': [target.target], 'file_dep': list(utils.git_ls('docs')), 'task_dep': ['_build_builder:{}'.format(builder.DOC_BUILDER.name)], 'clean': [clean(target)], }
def task__iso_add_node_manifest() -> types.TaskDict: """Copy the node announcement manifest to examples.""" dest_generic = constants.ISO_ROOT / 'examples' / 'new-node.yaml' dest_vagrant = constants.ISO_ROOT / 'examples' / 'new-node_vagrant.yaml' new_node_generic = [ constants.ROOT / 'examples' / 'new-node.yaml', dest_generic ] new_node_vagrant = [ constants.ROOT / 'examples' / 'new-node_vagrant.yaml', dest_vagrant ] return { 'title': lambda task: utils.title_with_target1('GENERATE', task), 'actions': [(coreutils.cp_file, new_node_generic), (coreutils.cp_file, new_node_vagrant)], 'targets': [dest_generic, dest_vagrant], 'task_dep': ['_iso_mkdir_examples'], 'uptodate': [True], 'clean': True, }
def task__iso_build() -> types.TaskDict: """Create the ISO from the files in ISO_ROOT.""" def mkisofs() -> None: """Create an ISO file (delete on error).""" cmd = [ config.ExtCommand.MKISOFS.value, '-output', ISO_FILE, '-quiet', '-rock', '-joliet', '-joliet-long', '-full-iso9660-filenames', '-volid', '{} {}'.format(config.PROJECT_NAME, versions.VERSION), '--iso-level', '3', '-gid', '0', '-uid', '0', '-input-charset', 'utf-8', '-output-charset', 'utf-8', constants.ISO_ROOT ] try: subprocess.run(cmd, check=True) except: utils.unlink_if_exist(ISO_FILE) raise doc = 'Create the ISO from the files in {}.'.format( utils.build_relpath(constants.ISO_ROOT) ) # Every file used for the ISO is a dependency. depends = list(coreutils.ls_files_rec(constants.ISO_ROOT)) depends.append(versions.VERSION_FILE) return { 'title': utils.title_with_target1('MKISOFS'), 'doc': doc, 'actions': [mkisofs], 'targets': [ISO_FILE], 'file_dep': depends, 'task_dep': ['check_for:mkisofs', '_build_root', '_iso_mkdir_root'], 'clean': True, }
def generate_meta(self) -> types.TaskDict: """Generate the .meta file for the package.""" spec_guest_file = Path('/rpmbuild/SPECS', self.spec.name) meta_guest_file = Path('/rpmbuild/META', self.meta.name) mounts = [ utils.bind_ro_mount( source=self.spec, target=spec_guest_file ), utils.bind_mount( source=self.meta.parent, target=meta_guest_file.parent ) ] rpmspec_config = docker_command.default_run_config( constants.REDHAT_ENTRYPOINT ) rpmspec_config['read_only'] = True buildmeta_callable = docker_command.DockerRun( command=['/entrypoint.sh', 'buildmeta'], builder=self.builder, environment={ 'SPEC': self.spec.name, 'META': self.meta.name }, run_config=rpmspec_config, mounts=mounts ) task = self.basic_task task.update({ 'name': self._get_task_name('rpmspec'), 'actions': [buildmeta_callable], 'doc': 'Generate {}.meta'.format(self.name), 'title': utils.title_with_target1('RPMSPEC'), 'targets': [self.meta], }) task['file_dep'].extend([self.spec]) task['task_dep'].append(self._get_task_name(self.MKDIR_TASK_NAME, with_basename=True)) return task
def task__iso_build() -> types.TaskDict: """Create the ISO from the files in ISO_ROOT.""" mkisofs = [ config.MKISOFS, '-output', ISO_FILE, '-quiet', '-rock', '-joliet', '-joliet-long', '-full-iso9660-filenames', '-volid', '{} {}'.format(config.PROJECT_NAME, constants.VERSION), '--iso-level', '3', '-gid', '0', '-uid', '0', '-input-charset', 'utf-8', '-output-charset', 'utf-8', constants.ISO_ROOT ] doc = 'Create the ISO from the files in {}.'.format( utils.build_relpath(constants.ISO_ROOT)) # Every file used for the ISO is a dependency. depends = list(coreutils.ls_files_rec(constants.ISO_ROOT)) depends.append(constants.VERSION_FILE) return { 'title': lambda task: utils.title_with_target1('MKISOFS', task), 'doc': doc, 'actions': [mkisofs], 'targets': [ISO_FILE], 'file_dep': depends, 'task_dep': ['_build_root', '_iso_mkdir_root'], 'clean': True, }
def _show(task: types.Task) -> str: """Return a description of the task.""" return utils.title_with_target1('NGINX_CFG', task)