def build_srpm(self) -> types.TaskDict:
"""Build the SRPM for the package."""
env = {
'SPEC': self.spec.name,
'SRPM': self.srpm.name,
'SOURCES': ' '.join(source.name for source in self.sources),
'VERSION': self.version,
}
buildsrpm_callable = docker_command.DockerRun(
command=['/entrypoint.sh', 'buildsrpm'],
builder=self.builder,
environment=env,
tmpfs={
'/home/build': '',
'/var/tmp': ''
},
mounts=self._get_buildsrpm_mounts(self.srpm.parent),
read_only=True,
run_config=docker_command.RPM_BASE_CONFIG)
task = self.basic_task
task.update({
'name': 'pkg_srpm',
'actions': [buildsrpm_callable],
'doc': 'Build {}'.format(self.srpm.name),
'title': utils.title_with_target1('BUILD SRPM'),
'targets': [self.srpm],
# Prevent Docker from polluting our output.
'verbosity': 0,
})
task['file_dep'].extend([self.spec])
task['file_dep'].extend(self.sources)
task['task_dep'].append('{}:{}'.format(self.basename,
self.MKDIR_TASK_NAME))
return task
def build_package(self) -> types.TaskDict:
"""Build DEB packages from source files."""
mounts = [
utils.bind_ro_mount(source=self.sources,
target=Path('/debbuild/pkg-src')),
utils.bind_ro_mount(source=self.debuild_sources,
target=Path('/debbuild/pkg-meta')),
utils.bind_mount(source=constants.PKG_DEB_ROOT,
target=Path('/debbuild/result')),
]
builddeb_callable = docker_command.DockerRun(
command=['/entrypoint.sh', 'builddeb'],
builder=self.builder,
run_config=docker_command.DEB_BASE_CONFIG,
mounts=mounts,
environment={
'VERSION': '{}-{}'.format(self.version, self.build_id)
},
)
task = self.basic_task
task.update({
'name':
'build_deb_pkg',
'actions': [builddeb_callable],
'doc':
'Build DEB package from sources for {}'.format(self.name),
'title':
utils.title_with_target1('BUILD DEB'),
'targets': [self.deb],
})
task['file_dep'].extend(coreutils.ls_files_rec(self.sources))
task['file_dep'].extend(coreutils.ls_files_rec(self.debuild_sources))
task['task_dep'].append('_package_mkdir_deb_iso_root')
task['task_dep'].append('_build_deb_container')
return task
def task__iso_generate_product_txt() -> types.TaskDict:
"""Generate the product.txt file."""
def action(targets: Sequence[str]) -> None:
datefmt = "%Y-%m-%dT%H:%M:%SZ"
dev_release = '1' if constants.VERSION_SUFFIX == '-dev' else '0'
info = (
('NAME', config.PROJECT_NAME),
('VERSION', constants.VERSION),
('SHORT_VERSION', constants.SHORT_VERSION),
('GIT', constants.GIT_REF or ''),
('DEVELOPMENT_RELEASE', dev_release),
('BUILD_TIMESTAMP', dt.datetime.utcnow().strftime(datefmt)),
('BUILD_HOST', socket.gethostname()),
)
with open(targets[0], 'w', encoding='utf-8') as fp:
data = '\n'.join('{}={}'.format(key, value) for key, value in info)
fp.write(data)
fp.write('\n')
return {
'title': lambda task: utils.title_with_target1('GENERATE', task),
'actions': [action],
'targets': [constants.ISO_ROOT / 'product.txt'],
'file_dep': [constants.VERSION_FILE],
'task_dep': ['_iso_mkdir_root'],
'uptodate': [False], # False because we include the build timestamp.
'clean': True,
}
def convert_package(self) -> types.TaskDict:
"""Build a DEB package from a RPM one."""
mounts = [
utils.bind_ro_mount(source=self.sources,
target=Path('/rpmbuild/source.rpm')),
utils.bind_mount(source=constants.PKG_DEB_ROOT,
target=Path('/debbuild/result')),
]
builddeb_callable = docker_command.DockerRun(
command=['/entrypoint.sh', 'rpm2deb'],
builder=self.builder,
run_config=docker_command.DEB_BASE_CONFIG,
mounts=mounts)
task = self.basic_task
task.update({
'name':
'convert_rpm_pkg_to_deb',
'actions': [builddeb_callable],
'doc':
'Build DEB package from RPM for {}'.format(self.name),
'title':
utils.title_with_target1('RPM2DEB'),
'targets': [self.deb],
})
task['file_dep'].append(self.sources)
task['task_dep'].append('_package_mkdir_deb_iso_root')
task['task_dep'].append('_build_deb_container')
return task
def build_repo(self) -> types.TaskDict:
def clean() -> None:
"""Delete the repository directory and its contents."""
coreutils.rm_rf(self.rootdir)
mkdir = self._mkdir_repo_root()
actions = mkdir['actions']
actions.append(self._buildrepo_action())
targets = [
Path(self.rootdir, 'dists/bionic', self.fullname,
'binary-amd64/Packages')
]
targets.extend(mkdir['targets'])
task = self.basic_task
task.update({
'name': 'build_repo',
'actions': actions,
'doc': 'Build the {} repository.'.format(self.name),
'title': utils.title_with_target1('BUILD DEB REPO'),
'targets': targets,
'uptodate': [True],
'clean': [clean],
})
for pkg in self.packages:
task['file_dep'].append(pkg.deb)
return task
def build_repo(self) -> types.TaskDict:
"""Build the repository."""
def clean() -> None:
"""Delete the repository metadata directory and its contents."""
coreutils.rm_rf(self.repodata)
mkdir = directory.Mkdir(directory=self.repodata).task
actions = mkdir['actions']
actions.append(self._buildrepo_action())
targets = [self.repodata/'repomd.xml']
targets.extend(mkdir['targets'])
task = self.basic_task
task.update({
'name': self._get_task_name('build_repodata'),
'actions': actions,
'doc': 'Build the {} repository metadata.'.format(self.name),
'title': utils.title_with_target1('BUILD RPM REPO'),
'targets': targets,
'uptodate': [True],
'clean': [clean],
# Prevent Docker from polluting our output.
'verbosity': 0,
})
if self.packages:
task['task_dep'].append(self._get_task_name(MKDIR_ROOT_TASK_NAME,
with_basename=True))
task['file_dep'].extend([
self.get_rpm_path(pkg) for pkg in self.packages
])
return task
def task_doc() -> Iterator[types.TaskDict]:
"""Generate the documentation."""
def clean(target: DocTarget) -> Callable[[], None]:
"""Delete the build sub-directory for the given target."""
return lambda: coreutils.rm_rf(target.target.parent)
doc_targets = (DocTarget(name='html',
command='html',
target=DOC_BUILD_DIR / 'html/index.html'),
DocTarget(name='pdf',
command='latexpdf',
target=DOC_PDF_FILE))
for target in doc_targets:
doc_format = target.name.upper()
action = ['tox', '-e', 'docs', '--', target.command]
# In CI we reuse already built PDF: just check the existence.
if config.RUNNING_IN_CI:
action = ['test', '-f', target.target]
yield {
'name':
target.name,
'title':
utils.title_with_target1('DOC {}'.format(doc_format)),
'doc':
'Generate {} {} documentation'.format(config.PROJECT_NAME,
doc_format),
'actions': [action],
'targets': [target.target],
'file_dep':
list(utils.git_ls('docs')),
'clean': [clean(target)],
}
def task(self) -> types.TaskDict:
task = self.basic_task
task.update({
'title': lambda task: utils.title_with_target1('MKDIR', task),
'actions': [(self._run, [task['targets'][0]])],
'uptodate': [True],
})
return task
def task(self) -> types.TaskDict:
task = self.basic_task
task.update({
'title': utils.title_with_target1('RENDER'),
'doc': 'Render template {}.'.format(self._src.name),
'actions': [self._run],
})
return task
def task(self) -> types.TaskDict:
task = self.basic_task
doc = ' '.join(self.__doc__.split()) if self.__doc__ else None
task.update({
'title': utils.title_with_target1('NGINX_CFG'),
'doc': doc,
'actions': [self._run],
})
return task
def task(self) -> types.TaskDict:
task = self.basic_task
task.update({
'title':
utils.title_with_target1('RENDER {}'.format(self._renderer.value)),
'doc':
'Render file "{}" with "{}"'.format(self._dest, self._renderer),
'actions': [self._run],
})
return task
def task__download_deb_packages() -> types.TaskDict:
"""Download Debian packages locally."""
witness = constants.PKG_DEB_ROOT / '.witness'
def clean() -> None:
"""Delete downloaded Debian packages."""
for repository in DEB_REPOSITORIES:
# Repository with an explicit list of packages are created by a
# dedicated task that will also handle their cleaning, so we skip
# them here.
if repository.packages:
continue
coreutils.rm_rf(repository.pkgdir)
utils.unlink_if_exist(witness)
constants.REPO_DEB_ROOT.rmdir()
def mkdirs() -> None:
"""Create directories for the repositories."""
for repository in DEB_REPOSITORIES:
repository.pkgdir.mkdir(exist_ok=True)
mounts = [
utils.bind_ro_mount(
source=constants.ROOT / 'packages' / 'debian' /
'download_packages.py',
target=Path('/download_packages.py'),
),
utils.bind_mount(source=constants.PKG_DEB_ROOT,
target=Path('/repositories')),
]
dl_packages_callable = docker_command.DockerRun(
command=['/download_packages.py', *DEB_TO_DOWNLOAD],
builder=builder.DEB_BUILDER,
mounts=mounts,
environment={'SALT_VERSION': versions.SALT_VERSION},
run_config=docker_command.default_run_config(
constants.DEBIAN_ENTRYPOINT))
return {
'title':
utils.title_with_target1('GET DEB PKGS'),
'actions': [mkdirs, dl_packages_callable],
'targets': [constants.PKG_DEB_ROOT / '.witness'],
'task_dep': [
'_package_mkdir_deb_root',
'_package_mkdir_deb_iso_root',
'_build_builder:{}'.format(builder.DEB_BUILDER.name),
],
'clean': [clean],
'uptodate': [doit.tools.config_changed(_TO_DOWNLOAD_DEB_CONFIG)],
# Prevent Docker from polluting our output.
'verbosity':
0,
}
def task__doc_deploy() -> types.TaskDict:
"""Deploy the documentation on the ISO."""
source = DOC_PDF_FILE
target = constants.ISO_DOCS_ROOT / 'user-guide.pdf'
return {
'title': utils.title_with_target1('COPY'),
'actions': [(coreutils.cp_file, (source, target))],
'targets': [target],
'task_dep': ['_doc_mkdir_root'],
'file_dep': [source],
'clean': True,
}
def get_source_files(self) -> types.TaskDict:
"""Download the source files to build the package."""
targets = [self.srcdir]
targets.extend(self.sources)
actions = directory.Mkdir(directory=self.srcdir).task['actions']
actions.append(self._get_sources)
task = self.basic_task
task.update({
'name': 'pkg_get_source',
'actions': actions,
'doc': 'Download source files for {}.'.format(self.name),
'title': utils.title_with_target1('GET_SRC'),
'targets': targets,
})
task['file_dep'].append(self.meta)
task['task_dep'].append('{}:{}'.format(self.basename,
self.MKDIR_TASK_NAME))
return task
def build_rpms(self) -> List[types.TaskDict]:
"""Build the RPMs from SRPMs."""
tasks = [self._mkdir_repo_root(), self._mkdir_repo_arch()]
for pkg in self.packages:
rpm = self._get_rpm_path(pkg)
env = {
'RPMS': '{arch}/{rpm}'.format(arch=self.ARCH, rpm=rpm.name),
'SRPM': pkg.srpm.name,
}
buildrpm_callable = docker_command.DockerRun(
command=['/entrypoint.sh', 'buildrpm'],
builder=self.builder,
environment=env,
mounts=self._get_buildrpm_mounts(pkg.srpm, rpm.parent),
tmpfs={
'/home/build': '',
'/var/tmp': ''
},
)
task = self.basic_task
task.update({
'name':
'build_rpm:{}'.format(pkg.name),
'actions': [buildrpm_callable],
'doc':
'Build {pkg} RPM for the {repo} repository.'.format(
pkg=pkg.name, repo=self.name),
'title':
lambda task: utils.title_with_target1('BUILD RPM', task),
'targets': [self._get_rpm_path(pkg)],
# Prevent Docker from polluting our output.
'verbosity':
0,
})
task['file_dep'].append(pkg.srpm)
task['task_dep'].append('{base}:{name}'.format(
base=self.basename,
name=MKDIR_ARCH_TASK_NAME,
))
task['task_dep'].append('_build_container')
tasks.append(task)
return tasks
def task__download_rpm_packages() -> types.TaskDict:
"""Download packages locally."""
def clean() -> None:
"""Delete cache and repositories on the ISO."""
coreutils.rm_rf(constants.PKG_RPM_ROOT / 'var')
for repository in RPM_REPOSITORIES:
# Repository with an explicit list of packages are created by a
# dedicated task that will also handle their cleaning, so we skip
# them here.
if repository.packages:
continue
coreutils.rm_rf(repository.rootdir)
mounts = [
utils.bind_mount(source=constants.PKG_RPM_ROOT,
target=Path('/install_root')),
utils.bind_mount(source=constants.REPO_RPM_ROOT,
target=Path('/repositories')),
]
dl_packages_callable = docker_command.DockerRun(
command=['/entrypoint.sh', 'download_packages', *RPM_TO_DOWNLOAD],
builder=builder.RPM_BUILDER,
mounts=mounts,
environment={'RELEASEVER': 7},
run_config=docker_command.default_run_config(
constants.REDHAT_ENTRYPOINT))
return {
'title':
utils.title_with_target1('GET RPM PKGS'),
'actions': [dl_packages_callable],
'targets': [constants.PKG_RPM_ROOT / 'var'],
'task_dep': [
'_package_mkdir_rpm_root',
'_package_mkdir_rpm_iso_root',
'_build_builder:{}'.format(builder.RPM_BUILDER.name),
],
'clean': [clean],
'uptodate': [doit.tools.config_changed(_TO_DOWNLOAD_RPM_CONFIG)],
# Prevent Docker from polluting our output.
'verbosity':
0,
}
def make_directories(self) -> types.TaskDict:
"""Return a task that create a directory hierarchy."""
def mkdirs(targets: Sequence[str]) -> None:
for directory in reversed(targets):
Path(directory).mkdir(exist_ok=True)
task = self.basic_task
task.update({
'name':
MAKE_TASK_NAME,
'doc':
'Create directory hierarchy for {}.'.format(self._root),
'title':
lambda task: utils.title_with_target1('MKTREE', task),
'actions': [mkdirs],
'targets':
self.directories,
'uptodate': [True],
})
return task
def generate_meta(self) -> types.TaskDict:
"""Generate the .meta file for the package."""
spec_guest_file = Path('/rpmbuild/SPECS', self.spec.name)
meta_guest_file = Path('/rpmbuild/META', self.meta.name)
mounts = [
docker_command.DockerRun.ENTRYPOINT_MOUNT,
docker_command.bind_ro_mount(source=self.spec,
target=spec_guest_file),
docker_command.bind_mount(source=self.meta.parent,
target=meta_guest_file.parent)
]
command = ['/entrypoint.sh', 'buildmeta']
rpmspec_config = {
'hostname': 'build',
'read_only': True,
'remove': True
}
buildmeta_callable = docker_command.DockerRun(
command=command,
builder=self.builder,
environment={
'SPEC': self.spec.name,
'META': self.meta.name
},
run_config=rpmspec_config,
mounts=mounts)
task = self.basic_task
task.update({
'name':
'pkg_rpmspec',
'actions': [buildmeta_callable],
'doc':
'Generate {}.meta'.format(self.name),
'title':
lambda task: utils.title_with_target1('RPMSPEC', task),
'targets': [self.meta],
})
task['file_dep'].extend([self.spec])
task['task_dep'].append('{}:{}'.format(self.basename,
self.MKDIR_TASK_NAME))
return task
def task__download_packages() -> types.TaskDict:
"""Download packages locally."""
def clean() -> None:
"""Delete cache and repositories on the ISO."""
coreutils.rm_rf(constants.PKG_ROOT / 'var')
for repository in REPOSITORIES:
# Repository with an explicit list of packages are created by a
# dedicated task that will also handle their cleaning, so we skip
# them here.
if repository.packages:
continue
coreutils.rm_rf(repository.rootdir)
pkg_list = constants.ROOT / 'packages/packages.list'
packages = _load_package_list(pkg_list)
mounts = [
docker_command.bind_mount(source=constants.PKG_ROOT,
target=Path('/install_root')),
docker_command.bind_mount(source=constants.REPO_ROOT,
target=Path('/repositories')),
]
dl_packages_callable = docker_command.DockerRun(
command=['/entrypoint.sh', 'download_packages', *packages],
builder=BUILDER,
mounts=mounts,
environment={'RELEASEVER': 7})
return {
'title':
lambda task: utils.title_with_target1('GET PKGS', task),
'actions': [dl_packages_callable],
'targets': [constants.PKG_ROOT / 'var'],
'file_dep': [pkg_list],
'task_dep':
['_package_mkdir_root', '_package_mkdir_iso_root', '_build_container'],
'clean': [clean],
'uptodate': [True],
# Prevent Docker from polluting our output.
'verbosity':
0,
}
def task_doc() -> Iterator[types.TaskDict]:
"""Generate the documentation."""
def clean(target: DocTarget) -> Callable[[], None]:
"""Delete the build sub-directory for the given target."""
return lambda: coreutils.rm_rf(target.target.parent)
doc_targets = (DocTarget(name='html',
command='html',
target=DOC_BUILD_DIR / 'html/index.html'),
DocTarget(name='pdf',
command='latexpdf',
target=DOC_PDF_FILE))
for target in doc_targets:
doc_format = target.name.upper()
run_config = docker_command.default_run_config(constants.ROOT /
'docs/entrypoint.sh')
# The builder stores the tox env in /tmp, don't shadow it!
run_config.pop('tmpfs', None)
build_doc = docker_command.DockerRun(
command=['/entrypoint.sh', target.command],
builder=builder.DOC_BUILDER,
run_config=run_config,
mounts=[
utils.bind_mount(target=Path('/usr/src/metalk8s/'),
source=constants.ROOT)
])
yield {
'name':
target.name,
'title':
utils.title_with_target1('DOC {}'.format(doc_format)),
'doc':
'Generate {} {} documentation'.format(config.PROJECT_NAME,
doc_format),
'actions': [build_doc],
'targets': [target.target],
'file_dep':
list(utils.git_ls('docs')),
'task_dep': ['_build_builder:{}'.format(builder.DOC_BUILDER.name)],
'clean': [clean(target)],
}
def task__iso_add_node_manifest() -> types.TaskDict:
"""Copy the node announcement manifest to examples."""
dest_generic = constants.ISO_ROOT / 'examples' / 'new-node.yaml'
dest_vagrant = constants.ISO_ROOT / 'examples' / 'new-node_vagrant.yaml'
new_node_generic = [
constants.ROOT / 'examples' / 'new-node.yaml', dest_generic
]
new_node_vagrant = [
constants.ROOT / 'examples' / 'new-node_vagrant.yaml', dest_vagrant
]
return {
'title':
lambda task: utils.title_with_target1('GENERATE', task),
'actions': [(coreutils.cp_file, new_node_generic),
(coreutils.cp_file, new_node_vagrant)],
'targets': [dest_generic, dest_vagrant],
'task_dep': ['_iso_mkdir_examples'],
'uptodate': [True],
'clean':
True,
}
def task__iso_build() -> types.TaskDict:
"""Create the ISO from the files in ISO_ROOT."""
def mkisofs() -> None:
"""Create an ISO file (delete on error)."""
cmd = [
config.ExtCommand.MKISOFS.value, '-output', ISO_FILE,
'-quiet',
'-rock',
'-joliet',
'-joliet-long',
'-full-iso9660-filenames',
'-volid', '{} {}'.format(config.PROJECT_NAME, versions.VERSION),
'--iso-level', '3',
'-gid', '0',
'-uid', '0',
'-input-charset', 'utf-8',
'-output-charset', 'utf-8',
constants.ISO_ROOT
]
try:
subprocess.run(cmd, check=True)
except:
utils.unlink_if_exist(ISO_FILE)
raise
doc = 'Create the ISO from the files in {}.'.format(
utils.build_relpath(constants.ISO_ROOT)
)
# Every file used for the ISO is a dependency.
depends = list(coreutils.ls_files_rec(constants.ISO_ROOT))
depends.append(versions.VERSION_FILE)
return {
'title': utils.title_with_target1('MKISOFS'),
'doc': doc,
'actions': [mkisofs],
'targets': [ISO_FILE],
'file_dep': depends,
'task_dep': ['check_for:mkisofs', '_build_root', '_iso_mkdir_root'],
'clean': True,
}
def generate_meta(self) -> types.TaskDict:
"""Generate the .meta file for the package."""
spec_guest_file = Path('/rpmbuild/SPECS', self.spec.name)
meta_guest_file = Path('/rpmbuild/META', self.meta.name)
mounts = [
utils.bind_ro_mount(
source=self.spec, target=spec_guest_file
),
utils.bind_mount(
source=self.meta.parent, target=meta_guest_file.parent
)
]
rpmspec_config = docker_command.default_run_config(
constants.REDHAT_ENTRYPOINT
)
rpmspec_config['read_only'] = True
buildmeta_callable = docker_command.DockerRun(
command=['/entrypoint.sh', 'buildmeta'],
builder=self.builder,
environment={
'SPEC': self.spec.name,
'META': self.meta.name
},
run_config=rpmspec_config,
mounts=mounts
)
task = self.basic_task
task.update({
'name': self._get_task_name('rpmspec'),
'actions': [buildmeta_callable],
'doc': 'Generate {}.meta'.format(self.name),
'title': utils.title_with_target1('RPMSPEC'),
'targets': [self.meta],
})
task['file_dep'].extend([self.spec])
task['task_dep'].append(self._get_task_name(self.MKDIR_TASK_NAME,
with_basename=True))
return task
def task__iso_build() -> types.TaskDict:
"""Create the ISO from the files in ISO_ROOT."""
mkisofs = [
config.MKISOFS, '-output', ISO_FILE, '-quiet', '-rock', '-joliet',
'-joliet-long', '-full-iso9660-filenames', '-volid',
'{} {}'.format(config.PROJECT_NAME, constants.VERSION), '--iso-level',
'3', '-gid', '0', '-uid', '0', '-input-charset', 'utf-8',
'-output-charset', 'utf-8', constants.ISO_ROOT
]
doc = 'Create the ISO from the files in {}.'.format(
utils.build_relpath(constants.ISO_ROOT))
# Every file used for the ISO is a dependency.
depends = list(coreutils.ls_files_rec(constants.ISO_ROOT))
depends.append(constants.VERSION_FILE)
return {
'title': lambda task: utils.title_with_target1('MKISOFS', task),
'doc': doc,
'actions': [mkisofs],
'targets': [ISO_FILE],
'file_dep': depends,
'task_dep': ['_build_root', '_iso_mkdir_root'],
'clean': True,
}
def _show(task: types.Task) -> str:
"""Return a description of the task."""
return utils.title_with_target1('NGINX_CFG', task)
