def response_mock(read=None, status=200):
if read is None:
read = PageTestHelper.response_data()
mock = Mock({'read': read})
mock.status = status
return mock
def test_fetching_request_token_raises_not_authorized_exception_when_token_is_wrong(self):
""" raise exception when unauthorized """
# set mock for http request response
response_mock = Mock({'read': "Invalid OAuth Request (signature_invalid, base string: POST&https%3A%2F%2Fapi.springnote.com%2Foauth%2Frequest_token&oauth_consumer_key%3Dsome%252Bconsumer%252Btoken%252Bkey%26oauth_nonce%3D39982135%26oauth_signature_method%3DHMAC-SHA1%26oauth_timestamp%3D1234870498%26oauth_version%3D1.0)"})
response_mock.status = 401
self.set_https_connection_response_for_request_token(response_mock)
self.assertRaises(springnote.SpringnoteError.Unauthorized, self.client.fetch_request_token)
def test_item_with_errors_should_have_images_redownloaded(self):
item = Item(item_test.sample_item)
db_item = Mock()
db_item.is_read = False
db_item.had_errors = True
self.db.get.return_value = db_item
main.process_item(item)
self.assertEqual(db_item.method_calls, [("redownload_images", (), {}), ("update", (), {})])
def __init__(self, config_file, build_timestamp):
"""
Constructor
Args:
config_file (str): config file path
"""
super(MockPackageBuilder, self).__init__()
self.build_dir = None
self.build_results_dir = CONF.get('result_dir')
self.archive = None
self.timestamp = build_timestamp
self.mock = Mock(config_file, self.timestamp)
def test_item_should_be_updated_with_new_feed_name(self):
item = Item(item_test.sample_item)
db_item = Mock()
item.tag_name = "feedb"
self.db.get.return_value = db_item
db_item.is_read = False
db_item.had_errors = False
main.process_item(item)
self.assertEqual(db_item.tag_name, "feedb")
self.assertEqual(self.db.method_calls, [("get", (item.google_id, None), {})])
self.assertEqual(db_item.method_calls, [("update", (), {})])
def _init_mock(self):
"""
Initialize Mock instance with common mock arguments.
"""
distro = distro_utils.get_distro(self.config.get('distro_name'),
self.config.get('distro_version'),
self.config.get('architecture'))
mock_config_file_name = "build-iso-%s-%s-%s.cfg" % (
distro.name, distro.version, distro.architecture)
mock_config_file_path = os.path.join("config/mock", distro.name,
distro.version,
mock_config_file_name)
if not os.path.isfile(mock_config_file_path):
raise exception.BaseException("Mock config file not found at %s" %
mock_config_file_path)
self.mock = Mock(mock_config_file_path, self.timestamp)
def __init__(self, config_file, build_timestamp):
"""
Constructor
Args:
config_file (str): config file path
"""
super(MockPackageBuilder, self).__init__()
self.build_dir = None
self.build_results_dir = CONF.get('result_dir')
self.archive = None
self.timestamp = build_timestamp
self.mock = Mock(config_file, self.timestamp)
def _init_mock(self):
"""
Initialize Mock instance with common mock arguments.
"""
distro = distro_utils.get_distro(
self.config.get('distro_name'),
self.config.get('distro_version'),
self.config.get('architecture'))
mock_config_file_name = "build-images-%s-%s-%s.cfg" % (
distro.name, distro.version, distro.architecture)
mock_config_file_path = os.path.join(
"config/mock", distro.name, distro.version,
mock_config_file_name)
if not os.path.isfile(mock_config_file_path):
raise exception.BaseException(
"Mock config file not found at %s" % mock_config_file_path)
self.mock = Mock(mock_config_file_path, self.timestamp)
class MockPackageBuilder(package_builder.PackageBuilder):
def __init__(self, config_file, build_timestamp):
"""
Constructor
Args:
config_file (str): config file path
"""
super(MockPackageBuilder, self).__init__()
self.build_dir = None
self.build_results_dir = CONF.get('result_dir')
self.archive = None
self.timestamp = build_timestamp
self.mock = Mock(config_file, self.timestamp)
def initialize(self):
"""
Initializes the configured chroot by installing the essential
packages. This setup is common for all packages that are built
and needs to be done only once.
"""
self.mock.run_command("--init")
def build(self, package):
"""
Build RPM package and its subpackages
Args:
package (RPM_Package): package
"""
self.clean_cache_dir(package)
LOG.info("%s: Starting build process" % package.name)
self._build_srpm(package)
self._install_external_dependencies(package)
self._build_rpm(package)
self._copy_rpms(self.build_dir, package.build_cache_dir)
if not CONF.get('keep_build_dir'):
self._destroy_build_directory()
def _build_srpm(self, package):
"""
Build source RPM package
Args:
package (RPM_Package): package
"""
LOG.info("%s: Building SRPM" % package.name)
self.mock.run_command(
"--buildsrpm --no-clean --spec %s --sources %s "
"--resultdir=%s %s" % (
package.spec_file.path, self.archive, self.build_dir,
package.macros))
def _build_rpm(self, package):
"""
Build RPM packages from a source RPM package
Args:
package (RPM_Package): package
"""
cmd = " --rebuild %s --no-clean --resultdir=%s %s" % (
self.build_dir + "/*.rpm", self.build_dir, package.macros)
if package.rpmmacro:
cmd = cmd + " --macro-file=%s" % package.rpmmacro
LOG.info("%s: Building RPM" % package.name)
try:
# On success save rpms and destroy build directory unless
# told otherwise.
self.mock.run_command(cmd)
package.built = True
except exception.SubprocessError:
LOG.info("%s: Failed to build RPMs, build artifacts are kept at "
"%s" % (package.name, self.build_dir))
raise
msg = "%s: Success! RPMs built!" % (package.name)
LOG.info(msg)
def prepare_sources(self, package):
"""
Create build directory structure, create a tar.gz file with the source code
and copy files to chroot.
Args:
package (RPM_Package): package
"""
LOG.info("%s: Preparing source files." % package.name)
self._create_build_directory(package)
self._prepare_archive(package)
if package.build_files:
self._copy_files_to_chroot(package)
def _prepare_archive(self, package):
"""
Create an archive (tar.gz) with a package source
Args:
package (RPM_Package): package
"""
LOG.info("%s: Preparing archive." % package.name)
if package.sources:
archive_to_build_dir = partial(package_source.archive,
directory=self.build_dir)
archived_sources = map(archive_to_build_dir, package.sources)
package.sources = archived_sources
self.archive = self.build_dir
elif package.repository:
file_path = package.repository.archive(
package.expects_source, self.build_dir)
self.archive = os.path.dirname(file_path)
elif package.download_source:
file_path = package._download_source(self.build_dir)
self.archive = os.path.dirname(file_path)
else:
LOG.warning("%s: Package has no external sources.", package.name)
self.archive = self.build_dir
def _copy_files_to_chroot(self, package):
"""
Copy files required to build a package to its build environment (chroot)
Args:
package (RPM_Package): package
"""
for f in os.listdir(package.build_files):
file_path = os.path.join(package.build_files, f)
LOG.info("copying %s to %s" % (file_path, self.archive))
shutil.copy(file_path, self.archive)
def clean(self):
"""
Clean build environment
"""
self.mock.run_command("--clean")
def clean_cache_dir(self, package):
"""
Delete the package's cached results directory.
Args:
package: package whose cached results will be removed
"""
if os.path.isdir(package.build_cache_dir):
LOG.debug("%s: Cleaning previously cached build results."
% package.name)
shutil.rmtree(package.build_cache_dir)
def _install_external_dependencies(self, package):
"""
Install the build dependencies of a package
Args:
package (RPM_Package): package
"""
if package.build_dependencies:
cmd = "--install"
for dep in package.build_dependencies:
cmd = " ".join([cmd, " ".join(dep.cached_build_results)])
LOG.info("%s: Installing dependencies on chroot" % package.name)
self.mock.run_command(cmd)
def _create_build_directory(self, package):
"""
Create build directory
Args:
package (RPM_Package): package
"""
self.build_dir = os.path.join(
os.path.abspath(CONF.get('work_dir')), 'mock_build',
self.timestamp, package.name)
os.makedirs(self.build_dir)
def _destroy_build_directory(self):
"""
Destroy build directory
"""
shutil.rmtree(self.build_dir)
def _copy_rpms(self, source_dir, target_dir):
"""
Copy the RPMs created by building a package to a target directory.
Args:
source_dir(str): path to the source directory containing the
RPMs
target_dir(str): path to the target directory
"""
if not os.path.exists(target_dir):
LOG.debug("Creating directory to store RPMs at %s " % target_dir)
os.makedirs(target_dir)
LOG.info("Copying RPMs from %s to %s" % (source_dir, target_dir))
for source_file_name in os.listdir(source_dir):
if (source_file_name.endswith(".rpm")
and not source_file_name.endswith(".src.rpm")):
LOG.info("Copying RPM file: %s" % source_file_name)
source_file_path = os.path.join(source_dir, source_file_name)
target_file_path = os.path.join(target_dir, source_file_name)
shutil.copy(source_file_path, target_file_path)
def copy_results(self, package):
"""
Copy cached build results to the results directory.
Args:
package(Package): package whose result files will be copied
"""
package_build_results_dir = os.path.join(
CONF.get('result_dir'), 'packages',
self.timestamp, package.name)
self._copy_rpms(package.build_cache_dir, package_build_results_dir)
def create_repository(self):
"""
Create yum repository in build results directory.
"""
result_dir = CONF.get('result_dir')
build_results_dir = os.path.join(
result_dir, 'packages', self.timestamp)
yum_repository.create_repository(build_results_dir)
repo_short_name = "host-os-local-repo-{timestamp}".format(**vars(self))
repo_long_name = ("OpenPOWER Host OS local repository built at "
"{timestamp}".format(**vars(self)))
repo_url = "file://" + os.path.abspath(build_results_dir)
repo_config = yum_repository.create_repository_config(
repo_short_name, repo_long_name, repo_url)
repo_config_dir = os.path.join(result_dir, "repository_config")
utils.create_directory(repo_config_dir)
repo_config_path = os.path.join(
repo_config_dir, self.timestamp + ".repo")
with open(repo_config_path, "w") as repo_config_file:
repo_config_file.write(repo_config)
def create_latest_symlink_result_dir(self):
"""
Create latest symlink pointing to the current result directory.
"""
result_dir = CONF.get('result_dir')
latest_package_build_results_dir = os.path.join(
result_dir, 'packages', LATEST_SYMLINK_NAME)
utils.force_symlink(self.timestamp, latest_package_build_results_dir)
latest_repo_config_path = os.path.join(
result_dir, 'repository_config', LATEST_SYMLINK_NAME)
utils.force_symlink(self.timestamp + ".repo", latest_repo_config_path)
class MockPackageBuilder(package_builder.PackageBuilder):
def __init__(self, config_file, build_timestamp):
"""
Constructor
Args:
config_file (str): config file path
"""
super(MockPackageBuilder, self).__init__()
self.build_dir = None
self.build_results_dir = CONF.get('result_dir')
self.archive = None
self.timestamp = build_timestamp
self.mock = Mock(config_file, self.timestamp)
def initialize(self):
"""
Initializes the configured chroot by installing the essential
packages. This setup is common for all packages that are built
and needs to be done only once.
"""
self.mock.run_command("--init")
def build(self, package):
"""
Build RPM package and its subpackages
Args:
package (RPM_Package): package
"""
self.clean_cache_dir(package)
LOG.info("%s: Starting build process" % package.name)
self._build_srpm(package)
self._install_external_dependencies(package)
self._build_rpm(package)
self._copy_rpms(self.build_dir, package.build_cache_dir)
if not CONF.get('keep_build_dir'):
self._destroy_build_directory()
def _build_srpm(self, package):
"""
Build source RPM package
Args:
package (RPM_Package): package
"""
LOG.info("%s: Building SRPM" % package.name)
self.mock.run_command("--buildsrpm --no-clean --spec %s --sources %s "
"--resultdir=%s %s" %
(package.spec_file.path, self.archive,
self.build_dir, package.macros))
def _build_rpm(self, package):
"""
Build RPM packages from a source RPM package
Args:
package (RPM_Package): package
"""
cmd = " --rebuild %s --no-clean --resultdir=%s %s" % (
self.build_dir + "/*.rpm", self.build_dir, package.macros)
if package.rpmmacro:
cmd = cmd + " --macro-file=%s" % package.rpmmacro
LOG.info("%s: Building RPM" % package.name)
try:
# On success save rpms and destroy build directory unless
# told otherwise.
self.mock.run_command(cmd)
package.built = True
except exception.SubprocessError:
LOG.info("%s: Failed to build RPMs, build artifacts are kept at "
"%s" % (package.name, self.build_dir))
raise
msg = "%s: Success! RPMs built!" % (package.name)
LOG.info(msg)
def prepare_sources(self, package):
"""
Create build directory structure, create a tar.gz file with the source code
and copy files to chroot.
Args:
package (RPM_Package): package
"""
LOG.info("%s: Preparing source files." % package.name)
self._create_build_directory(package)
self._prepare_archive(package)
if package.build_files:
self._copy_files_to_chroot(package)
def _prepare_archive(self, package):
"""
Create an archive (tar.gz) with a package source
Args:
package (RPM_Package): package
"""
LOG.info("%s: Preparing archive." % package.name)
if package.sources:
archive_to_build_dir = partial(package_source.archive,
directory=self.build_dir)
archived_sources = map(archive_to_build_dir, package.sources)
package.sources = archived_sources
self.archive = self.build_dir
elif package.repository:
file_path = package.repository.archive(package.expects_source,
self.build_dir)
self.archive = os.path.dirname(file_path)
elif package.download_source:
file_path = package._download_source(self.build_dir)
self.archive = os.path.dirname(file_path)
else:
LOG.warning("%s: Package has no external sources.", package.name)
self.archive = self.build_dir
def _copy_files_to_chroot(self, package):
"""
Copy files required to build a package to its build environment (chroot)
Args:
package (RPM_Package): package
"""
for f in os.listdir(package.build_files):
file_path = os.path.join(package.build_files, f)
LOG.info("copying %s to %s" % (file_path, self.archive))
shutil.copy(file_path, self.archive)
def clean(self):
"""
Clean build environment
"""
self.mock.run_command("--clean")
def clean_cache_dir(self, package):
"""
Delete the package's cached results directory.
Args:
package: package whose cached results will be removed
"""
if os.path.isdir(package.build_cache_dir):
LOG.debug("%s: Cleaning previously cached build results." %
package.name)
shutil.rmtree(package.build_cache_dir)
def _install_external_dependencies(self, package):
"""
Install the build dependencies of a package
Args:
package (RPM_Package): package
"""
if package.build_dependencies:
cmd = "--install"
for dep in package.build_dependencies:
cmd = " ".join([cmd, " ".join(dep.cached_build_results)])
LOG.info("%s: Installing dependencies on chroot" % package.name)
self.mock.run_command(cmd)
def _create_build_directory(self, package):
"""
Create build directory
Args:
package (RPM_Package): package
"""
self.build_dir = os.path.join(os.path.abspath(CONF.get('work_dir')),
'mock_build', self.timestamp,
package.name)
os.makedirs(self.build_dir)
def _destroy_build_directory(self):
"""
Destroy build directory
"""
shutil.rmtree(self.build_dir)
def _copy_rpms(self, source_dir, target_dir):
"""
Copy the RPMs created by building a package to a target directory.
Args:
source_dir(str): path to the source directory containing the
RPMs
target_dir(str): path to the target directory
"""
if not os.path.exists(target_dir):
LOG.debug("Creating directory to store RPMs at %s " % target_dir)
os.makedirs(target_dir)
LOG.info("Copying RPMs from %s to %s" % (source_dir, target_dir))
for source_file_name in os.listdir(source_dir):
if (source_file_name.endswith(".rpm")
and not source_file_name.endswith(".src.rpm")):
LOG.info("Copying RPM file: %s" % source_file_name)
source_file_path = os.path.join(source_dir, source_file_name)
target_file_path = os.path.join(target_dir, source_file_name)
shutil.copy(source_file_path, target_file_path)
def copy_results(self, package):
"""
Copy cached build results to the results directory.
Args:
package(Package): package whose result files will be copied
"""
package_build_results_dir = os.path.join(CONF.get('result_dir'),
'packages', self.timestamp,
package.name)
self._copy_rpms(package.build_cache_dir, package_build_results_dir)
def create_repository(self):
"""
Create yum repository in build results directory.
"""
result_dir = CONF.get('result_dir')
build_results_dir = os.path.join(result_dir, 'packages',
self.timestamp)
yum_repository.create_repository(build_results_dir)
repo_short_name = "host-os-local-repo-{timestamp}".format(**vars(self))
repo_long_name = ("OpenPOWER Host OS local repository built at "
"{timestamp}".format(**vars(self)))
repo_url = "file://" + os.path.abspath(build_results_dir)
repo_config = yum_repository.create_repository_config(
repo_short_name, repo_long_name, repo_url)
repo_config_dir = os.path.join(result_dir, "repository_config")
utils.create_directory(repo_config_dir)
repo_config_path = os.path.join(repo_config_dir,
self.timestamp + ".repo")
with open(repo_config_path, "w") as repo_config_file:
repo_config_file.write(repo_config)
def create_latest_symlink_result_dir(self):
"""
Create latest symlink pointing to the current result directory.
"""
result_dir = CONF.get('result_dir')
latest_package_build_results_dir = os.path.join(
result_dir, 'packages', LATEST_SYMLINK_NAME)
utils.force_symlink(self.timestamp, latest_package_build_results_dir)
latest_repo_config_path = os.path.join(result_dir, 'repository_config',
LATEST_SYMLINK_NAME)
utils.force_symlink(self.timestamp + ".repo", latest_repo_config_path)
class MockPungiIsoBuilder(object):
def __init__(self, config):
self.config = config
self.work_dir = self.config.get('work_dir')
self.timestamp = datetime.datetime.now().isoformat()
self.result_dir = os.path.join(self.config.get('result_dir'), 'iso',
self.timestamp)
self.distro = self.config.get("iso_name")
self.version = (self.config.get("iso_version")
or datetime.date.today().strftime("%y%m%d"))
(_, _, self.arch) = distro_utils.detect_distribution()
self.pungi_binary = self.config.get('pungi_binary') or "pungi"
self.pungi_args = self.config.get('pungi_args') or ""
self._init_mock()
def _init_mock(self):
"""
Initialize Mock instance with common mock arguments.
"""
distro = distro_utils.get_distro(self.config.get('distro_name'),
self.config.get('distro_version'),
self.config.get('architecture'))
mock_config_file_name = "build-iso-%s-%s-%s.cfg" % (
distro.name, distro.version, distro.architecture)
mock_config_file_path = os.path.join("config/mock", distro.name,
distro.version,
mock_config_file_name)
if not os.path.isfile(mock_config_file_path):
raise exception.BaseException("Mock config file not found at %s" %
mock_config_file_path)
self.mock = Mock(mock_config_file_path, self.timestamp)
def build(self):
LOG.info("Starting ISO build process")
self._setup()
self._build()
self._save()
def _setup(self):
LOG.info("Initializing a chroot")
self.mock.run_command("--init")
package_list = [
"yum-plugin-priorities", "yum-utils", "createrepo", "pungi"
]
LOG.info("Installing %s inside the chroot" % " ".join(package_list))
self.mock.run_command("--install %s" % " ".join(package_list))
self._create_host_os_repo()
self._create_merged_repo()
self._create_iso_kickstart()
def _create_host_os_repo(self):
LOG.info("Creating Host OS yum repository inside chroot")
LOG.debug("Creating yum repository directory")
self.mock.run_command("--shell 'mkdir -p %s'" %
CHROOT_HOST_OS_REPO_PATH)
LOG.debug("Creating package groups metadata file (comps.xml)")
groups_file_content = packages_groups_xml_creator.create_comps_xml(
self.config.get('installable_environments'))
groups_file_path = os.path.join(self.work_dir, GROUPS_FILE_NAME)
try:
with open(groups_file_path, 'wt') as groups_file:
groups_file.write(groups_file_content)
except IOError:
LOG.error("Failed to write XML to %s file." % groups_file_path)
raise
self.mock.run_command("--copyin %s %s" %
(groups_file_path, GROUPS_FILE_CHROOT_PATH))
LOG.debug("Copying packages to chroot")
packages_dir = self.config.get('packages_dir')
rpm_files = utils.recursive_glob(packages_dir, "*.rpm")
self.mock.run_command("--copyin %s %s" %
(" ".join(rpm_files), CHROOT_HOST_OS_REPO_PATH))
LOG.debug("Creating yum repository")
create_repo_command = (
"--shell 'createrepo --verbose --groupfile {groups_file} "
"{repo_path}'".format(groups_file=GROUPS_FILE_CHROOT_PATH,
repo_path=CHROOT_HOST_OS_REPO_PATH))
self.mock.run_command(create_repo_command)
def _create_merged_repo(self):
LOG.info(
"Creating base distro and Host OS merged yum repository inside chroot"
)
LOG.debug("Creating yum repository directory")
self.mock.run_command("--shell 'mkdir -p %s'" %
CHROOT_MERGED_REPO_PATH)
LOG.debug("Creating yum repository configuration")
packages_dir_url = "file://" + os.path.abspath(
CHROOT_HOST_OS_REPO_PATH)
repo_config = yum_repository.YUM_MAIN_CONFIG
repo_config += yum_repository.create_repository_config(
"host-os-local-repo",
"OpenPOWER Host OS local repository",
packages_dir_url,
priority=1)
distro_repos = self.config.get('distro_repos')
for repo in distro_repos:
repo_config += yum_repository.create_repository_config(
repo["name"],
repo["name"],
repo["url"],
url_type=repo["url_type"],
priority=2)
repo_config_file_path = os.path.join(
self.work_dir, os.path.basename(CHROOT_REPO_CONFIG_FILE_PATH))
with open(repo_config_file_path, 'w') as repo_config_file:
repo_config_file.write(repo_config)
self.mock.run_command(
"--copyin %s %s" %
(repo_config_file_path, CHROOT_REPO_CONFIG_FILE_PATH))
LOG.debug("Downloading packages")
YUM_INSTALL_ROOT_DIR = "/yum_install_root"
iso_repo_packages_groups = (
ISO_REPO_MINIMAL_PACKAGES_GROUPS +
self.config.get('iso_repo_packages_groups'))
iso_repo_packages = (ISO_REPO_MINIMAL_PACKAGES +
self.config.get('iso_repo_packages'))
groups_to_download = [
'"@{}"'.format(group) for group in iso_repo_packages_groups
]
packages_to_download = [
'"{}"'.format(package) for package in iso_repo_packages
]
mock_yum_command = (
"--shell 'yumdownloader --config {config_file} "
"--installroot {install_root} "
"--destdir {dest_dir} "
"--releasever {distro_version} --resolve {packages}'".format(
config_file=CHROOT_REPO_CONFIG_FILE_PATH,
install_root=YUM_INSTALL_ROOT_DIR,
dest_dir=CHROOT_MERGED_REPO_PATH,
distro_version=self.config.get('distro_version'),
packages=" ".join(groups_to_download + packages_to_download)))
self.mock.run_command(mock_yum_command)
LOG.debug("Merging package groups metadata files (comps.xml)")
MERGED_GROUPS_FILE_CHROOT_PATH = "/merged-comps.xml"
chroot_path = self.mock.run_command("--print-root-path").strip()
cached_groups_files_glob = os.path.join(
chroot_path, os.path.relpath(YUM_INSTALL_ROOT_DIR, "/"),
"var/cache/yum/*/gen/comps.xml")
other_groups_files = [
"--load " + os.path.relpath(groups_file_path, chroot_path)
for groups_file_path in glob.glob(cached_groups_files_glob)
]
merge_comps_command = (
"yum-groups-manager --load {host_os_groups_file} {other_loads} "
"--save {merged_groups_file} --id empty-group".format(
host_os_groups_file=GROUPS_FILE_CHROOT_PATH,
other_loads=" ".join(other_groups_files),
merged_groups_file=MERGED_GROUPS_FILE_CHROOT_PATH))
self.mock.run_command("--shell '%s'" % merge_comps_command)
LOG.debug("Creating yum repository")
create_repo_command = (
"--shell 'createrepo --verbose --groupfile {groups_file} "
"{repo_path}'".format(groups_file=MERGED_GROUPS_FILE_CHROOT_PATH,
repo_path=CHROOT_MERGED_REPO_PATH))
self.mock.run_command(create_repo_command)
LOG.info(
"Checking if created repository has any unresolvable dependencies")
mock_iso_repo_url = "file://%s/" % CHROOT_MERGED_REPO_PATH
merged_repo_config = yum_repository.YUM_MAIN_CONFIG
merged_repo_config += yum_repository.create_repository_config(
"merged-local-repo", "OpenPOWER Host OS merged local repository",
mock_iso_repo_url)
merged_repo_config_file_path = os.path.join(
self.work_dir,
os.path.basename(CHROOT_MERGED_REPO_CONFIG_FILE_PATH))
with open(merged_repo_config_file_path,
'w') as merged_repo_config_file:
merged_repo_config_file.write(merged_repo_config)
self.mock.run_command("--copyin %s %s" %
(merged_repo_config_file_path,
CHROOT_MERGED_REPO_CONFIG_FILE_PATH))
merged_repo_closure_command = (
"--chroot 'repoclosure --config {config_file} "
"--tempcache'".format(
config_file=CHROOT_MERGED_REPO_CONFIG_FILE_PATH))
self.mock.run_command(merged_repo_closure_command)
def _create_iso_kickstart(self):
kickstart_file = self.config.get('automated_install_file')
kickstart_path = os.path.join(self.work_dir, kickstart_file)
LOG.info("Creating ISO kickstart file %s" % kickstart_path)
mock_iso_repo_name = self.config.get('mock_iso_repo_name')
iso_repo_packages_groups = (
ISO_REPO_MINIMAL_PACKAGES_GROUPS +
self.config.get('iso_repo_packages_groups'))
iso_repo_packages = (ISO_REPO_MINIMAL_PACKAGES +
self.config.get('iso_repo_packages'))
with open(kickstart_path, "wt") as kickstart_file:
mock_iso_repo_url = "file://%s/" % CHROOT_MERGED_REPO_PATH
repo = "repo --name=%s --baseurl=%s\n" % (mock_iso_repo_name,
mock_iso_repo_url)
kickstart_file.write(repo)
kickstart_file.write("%packages\n")
for group in iso_repo_packages_groups:
kickstart_file.write("@{}\n".format(group))
for package in iso_repo_packages:
kickstart_file.write("{}\n".format(package))
kickstart_file.write("%end\n")
self.mock.run_command("--copyin %s /" % kickstart_path)
def _build(self):
LOG.info("Building ISO")
build_cmd = ("%s %s -c %s --name %s --ver %s" %
(self.pungi_binary, self.pungi_args,
self.config.get('automated_install_file'), self.distro,
self.version))
self.mock.run_command("--shell '%s'" % build_cmd)
def _save(self):
utils.create_directory(self.result_dir)
latest_dir = os.path.join(os.path.dirname(self.result_dir),
LATEST_SYMLINK_NAME)
utils.force_symlink(self.timestamp, latest_dir)
iso_dir = "/%s/%s/iso" % (self.version, self.arch)
iso_files = "%s/*" % iso_dir
LOG.info("Saving ISO files %s at %s" % (iso_files, self.result_dir))
self.mock.run_command("--copyout %s %s" % (iso_files, self.result_dir))
def clean(self):
self.mock.run_command("--clean")
class MockPungiIsoBuilder(object):
def __init__(self, config):
self.config = config
self.work_dir = self.config.get('work_dir')
self.timestamp = datetime.datetime.now().isoformat()
self.result_dir = os.path.join(self.config.get('result_dir'),
'iso', self.timestamp)
self.distro = self.config.get("iso_name")
self.version = (self.config.get("iso_version")
or datetime.date.today().strftime("%y%m%d"))
(_, _, self.arch) = distro_utils.detect_distribution()
self.pungi_binary = self.config.get('pungi_binary') or "pungi"
self.pungi_args = self.config.get('pungi_args') or ""
self.build_iso = self.config.get('iso')
self.build_install_tree = self.config.get('install_tree')
self._init_mock()
utils.create_directory(self.result_dir)
def _init_mock(self):
"""
Initialize Mock instance with common mock arguments.
"""
distro = distro_utils.get_distro(
self.config.get('distro_name'),
self.config.get('distro_version'),
self.config.get('architecture'))
mock_config_file_name = "build-images-%s-%s-%s.cfg" % (
distro.name, distro.version, distro.architecture)
mock_config_file_path = os.path.join(
"config/mock", distro.name, distro.version,
mock_config_file_name)
if not os.path.isfile(mock_config_file_path):
raise exception.BaseException(
"Mock config file not found at %s" % mock_config_file_path)
self.mock = Mock(mock_config_file_path, self.timestamp)
def build(self):
LOG.info("Starting ISO build process")
self._setup()
self._build()
self._save()
def _setup(self):
LOG.info("Initializing a chroot")
self.mock.run_command("--init")
package_list = [
"yum-plugin-priorities", "yum-utils", "createrepo", "pungi"]
LOG.info("Installing %s inside the chroot" % " ".join(package_list))
self.mock.run_command("--install %s" % " ".join(package_list))
self._create_host_os_repo()
self._create_merged_repo()
self._create_iso_kickstart()
def _create_host_os_repo(self):
LOG.info("Creating Host OS yum repository inside chroot")
LOG.debug("Creating yum repository directory")
self.mock.run_command(
"--shell 'mkdir -p %s'" % CHROOT_HOST_OS_REPO_PATH)
LOG.debug("Creating package groups metadata file (comps.xml)")
groups_file_content = packages_groups_xml_creator.create_comps_xml(
self.config.get('installable_environments'))
groups_file_path = os.path.join(self.work_dir, GROUPS_FILE_NAME)
try:
with open(groups_file_path, 'wt') as groups_file:
groups_file.write(groups_file_content)
except IOError:
LOG.error("Failed to write XML to %s file." % groups_file_path)
raise
self.mock.run_command("--copyin %s %s" %
(groups_file_path, GROUPS_FILE_CHROOT_PATH))
LOG.debug("Copying packages to chroot")
packages_dir = self.config.get('packages_dir')
rpm_files = utils.recursive_glob(packages_dir, "*.rpm")
self.mock.run_command(
"--copyin %s %s" % (" ".join(rpm_files), CHROOT_HOST_OS_REPO_PATH))
LOG.debug("Creating yum repository")
create_repo_command = (
"--shell 'createrepo --verbose --groupfile {groups_file} "
"{repo_path}'".format(groups_file=GROUPS_FILE_CHROOT_PATH,
repo_path=CHROOT_HOST_OS_REPO_PATH))
self.mock.run_command(create_repo_command)
def _create_merged_repo(self):
LOG.info("Creating base distro and Host OS merged yum repository inside chroot")
LOG.debug("Creating yum repository directory")
self.mock.run_command(
"--shell 'mkdir -p %s'" % CHROOT_MERGED_REPO_PATH)
LOG.debug("Creating yum repository configuration")
packages_dir_url = "file://" + os.path.abspath(CHROOT_HOST_OS_REPO_PATH)
repo_config = yum_repository.YUM_MAIN_CONFIG
repo_config += yum_repository.create_repository_config(
"host-os-local-repo", "OpenPOWER Host OS local repository",
packages_dir_url, priority=1)
distro_repos = self.config.get('distro_repos')
for repo in distro_repos:
repo_config += yum_repository.create_repository_config(
repo["name"], repo["name"], repo["url"],
url_type=repo["url_type"], priority=2)
repo_config_file_path = os.path.join(
self.work_dir, os.path.basename(CHROOT_REPO_CONFIG_FILE_PATH))
with open(repo_config_file_path, 'w') as repo_config_file:
repo_config_file.write(repo_config)
self.mock.run_command("--copyin %s %s" % (
repo_config_file_path, CHROOT_REPO_CONFIG_FILE_PATH))
LOG.debug("Downloading packages")
YUM_INSTALL_ROOT_DIR = "/yum_install_root"
iso_repo_packages_groups = (
ISO_REPO_MINIMAL_PACKAGES_GROUPS
+ self.config.get('iso_repo_packages_groups'))
iso_repo_packages = (
ISO_REPO_MINIMAL_PACKAGES
+ self.config.get('iso_repo_packages'))
groups_to_download = [
'"@{}"'.format(group) for group in iso_repo_packages_groups]
packages_to_download = [
'"{}"'.format(package) for package in iso_repo_packages]
mock_yum_command = (
"--shell 'yumdownloader --config {config_file} "
"--installroot {install_root} "
"--destdir {dest_dir} "
"--releasever {distro_version} --resolve {packages}'".format(
config_file=CHROOT_REPO_CONFIG_FILE_PATH,
install_root=YUM_INSTALL_ROOT_DIR,
dest_dir=CHROOT_MERGED_REPO_PATH,
distro_version=self.config.get('distro_version'),
packages=" ".join(groups_to_download + packages_to_download)))
self.mock.run_command(mock_yum_command)
LOG.debug("Merging package groups metadata files (comps.xml)")
MERGED_GROUPS_FILE_CHROOT_PATH = "/merged-comps.xml"
chroot_path = self.mock.run_command("--print-root-path").strip()
cached_groups_files_glob = os.path.join(chroot_path, os.path.relpath(
YUM_INSTALL_ROOT_DIR, "/"), "var/cache/yum/*/gen/comps.xml")
other_groups_files = [
"--load " + os.path.relpath(groups_file_path, chroot_path)
for groups_file_path in glob.glob(cached_groups_files_glob)]
merge_comps_command = (
"yum-groups-manager --load {host_os_groups_file} {other_loads} "
"--save {merged_groups_file} --id empty-group".format(
host_os_groups_file=GROUPS_FILE_CHROOT_PATH,
other_loads=" ".join(other_groups_files),
merged_groups_file=MERGED_GROUPS_FILE_CHROOT_PATH))
self.mock.run_command("--shell '%s'" % merge_comps_command)
LOG.debug("Creating yum repository")
create_repo_command = (
"--shell 'createrepo --verbose --groupfile {groups_file} "
"{repo_path}'".format(groups_file=MERGED_GROUPS_FILE_CHROOT_PATH,
repo_path=CHROOT_MERGED_REPO_PATH))
self.mock.run_command(create_repo_command)
LOG.info("Checking if created repository has any unresolvable dependencies")
mock_iso_repo_url = "file://%s/" % CHROOT_MERGED_REPO_PATH
merged_repo_config = yum_repository.YUM_MAIN_CONFIG
merged_repo_config += yum_repository.create_repository_config(
"merged-local-repo", "OpenPOWER Host OS merged local repository",
mock_iso_repo_url)
merged_repo_config_file_path = os.path.join(
self.work_dir, os.path.basename(CHROOT_MERGED_REPO_CONFIG_FILE_PATH))
with open(merged_repo_config_file_path, 'w') as merged_repo_config_file:
merged_repo_config_file.write(merged_repo_config)
self.mock.run_command("--copyin %s %s" % (
merged_repo_config_file_path, CHROOT_MERGED_REPO_CONFIG_FILE_PATH))
merged_repo_closure_command = (
"--chroot 'repoclosure --config {config_file} "
"--tempcache'".format(
config_file=CHROOT_MERGED_REPO_CONFIG_FILE_PATH))
self.mock.run_command(merged_repo_closure_command)
def _create_iso_kickstart(self):
kickstart_file = self.config.get('automated_install_file')
kickstart_path = os.path.join(self.work_dir, kickstart_file)
LOG.info("Creating ISO kickstart file %s" % kickstart_path)
mock_iso_repo_name = self.config.get('mock_iso_repo_name')
iso_repo_packages_groups = (
ISO_REPO_MINIMAL_PACKAGES_GROUPS
+ self.config.get('iso_repo_packages_groups'))
iso_repo_packages = (
ISO_REPO_MINIMAL_PACKAGES
+ self.config.get('iso_repo_packages'))
with open(kickstart_path, "wt") as kickstart_file:
mock_iso_repo_url = "file://%s/" % CHROOT_MERGED_REPO_PATH
repo = "repo --name=%s --baseurl=%s\n" % (
mock_iso_repo_name, mock_iso_repo_url)
kickstart_file.write(repo)
kickstart_file.write("%packages\n")
for group in iso_repo_packages_groups:
kickstart_file.write("@{}\n".format(group))
for package in iso_repo_packages:
kickstart_file.write("{}\n".format(package))
kickstart_file.write("%end\n")
self.mock.run_command("--copyin %s /" % kickstart_path)
shutil.copy(kickstart_path, self.result_dir)
def _build(self):
build_iso_args = ""
if self.build_iso:
build_iso_args = "-I"
LOG.info("Building ISO")
build_cmd = ("%s %s -c %s --name %s --ver %s -G -C -B %s" %
(self.pungi_binary, self.pungi_args,
self.config.get('automated_install_file'),
self.distro, self.version, build_iso_args))
self.mock.run_command("--shell '%s'" % build_cmd)
def _save(self):
latest_dir = os.path.join(os.path.dirname(self.result_dir),
LATEST_SYMLINK_NAME)
utils.force_symlink(self.timestamp, latest_dir)
iso_dir = "/%s/%s/iso" % (self.version, self.arch)
iso_files = "%s/*" % iso_dir
if self.build_iso:
LOG.info("Saving ISO files %s at %s" % (iso_files, self.result_dir))
self.mock.run_command("--copyout %s %s" %
(iso_files, self.result_dir))
tree_src_dir = "/%s/%s/os" % (self.version, self.arch)
tree_dest_dir = os.path.join(self.result_dir, "os")
if self.build_install_tree:
LOG.info("Saving installable tree %s at %s" %
(tree_src_dir, tree_dest_dir))
self.mock.run_command("--copyout %s %s" %
(tree_src_dir, tree_dest_dir))
def clean(self):
self.mock.run_command("--clean")