def __build_yum_config(self, repoid):
"""
Builds the YUM config that will be used for YUM initialization.
@param repoid The ID of repository to be analyzed.
@return The path to generated YUM config file.
"""
config_path = temporaries.create_temporary_file("yum.conf")
config = ConfigParser()
# FIXME: This config was get from my Ubuntu default yum config, maybe
# we should somehow modify it.
config.add_section("main")
cache_path = temporaries.create_temporary_directory("yum.cache")
config.set("main", "cachedir", cache_path)
config.set("main", "keepcache", "1")
config.set("main", "debuglevel", "2")
log_path = temporaries.create_temporary_file("yum.log")
config.set("main", "logfile", log_path)
# FIXME: Is this a reason why ARM RPMs are ignored?
config.set("main", "exactarch", "1")
config.set("main", "obsoletes", "1")
config.add_section(repoid)
config.set(repoid, "name", "Analyzed repository")
config.set(repoid, "baseurl",
"file://{0}".format(self.repository_path))
with open(config_path, "w") as config_file:
config.write(config_file)
config_file.close()
return config_path
def create_image(arch, repository_names, repository_paths, kickstart_file_path,
mic_options, specific_packages):
"""
Creates an image using MIC tool, from given repository and given kickstart
file. It creates a copy of kickstart file and replaces "repo" to given
repository path.
@param arch The architecture of the image
@param repository_names The names of repositorues
@param repository_paths The repository paths
@param kickstart_file The kickstart file to be used
@param mic_options Additional options for MIC.
@param specific_packages Packages that must be additionally
installed.
"""
if repository_names is None or len(repository_names) == 0:
raise Exception("Repository names are not given! "
"{0}".format(repository_names))
modified_kickstart_file_path = temporaries.create_temporary_file("mod.ks")
shutil.copy(kickstart_file_path, modified_kickstart_file_path)
kickstart_file = KickstartFile(modified_kickstart_file_path)
kickstart_file.replace_repository_paths(repository_names, repository_paths)
kickstart_file.add_packages(specific_packages)
# Now create the image using the "mic" tool:
global mic_config_path
mic_command = [
"sudo", "mic", "create", "loop", modified_kickstart_file_path, "-A",
arch, "--config", mic_config_path, "--tmpfs"
]
if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
mic_options.extend(["--debug", "--verbose"])
if mic_options is not None:
mic_command.extend(mic_options)
hidden_subprocess.call("Building the image", mic_command)
def generate_mic_config(output_directory_path, temporary_directory_path):
"""
Generates mic config with changed locations of cachedir, tmpdir, rootdir.
@param output_directory_path The path to the mic output directory.
@param temporary_directory_path The path to cache directory root.
"""
mic_directory_path = os.path.join(temporary_directory_path, "mic")
mic_cache_directory_path = os.path.join(mic_directory_path, "cache")
mic_bootstrap_directory_path = os.path.join(mic_directory_path,
"bootstrap")
if not os.path.isdir(mic_directory_path):
os.makedirs(mic_directory_path)
logging.debug("Created directory for mic's cache "
"{0}".format(mic_directory_path))
parser = configparser.SafeConfigParser()
mic_config_path = temporaries.create_temporary_file(".mic.conf")
mic_config_path_default = "/etc/mic/mic.conf"
# FIXME: Maybe it will be better to always generate config from scratch?
if not os.path.isfile(mic_config_path_default):
logging.warning("Cannot find {0}".format(mic_config_path_default))
parser.add_section("common")
parser.set("common", "distro_name", "Tizen")
# FIXME: Is it corect to hardcode paths in a such way?
parser.set("common", "plugin_dir", "/usr/lib/mic/plugins")
parser.add_section("create")
parser.set("create", "runtime", "bootstrap")
# FIXME: Do we need some abstraction here?
parser.set("bootstrap", "packages", "mic-bootstrap-x86-arm")
else:
shutil.copy(mic_config_path_default, mic_config_path)
parser.read(mic_config_path)
for section in ["create", "bootstrap"]:
if not parser.has_section(section):
logging.warning("Config {0} does not has section "
"\"{1}\"!".format(mic_config_path_default,
section))
parser.add_section(section)
parser.set("create", "tmpdir", mic_directory_path)
parser.set("create", "cachedir", mic_cache_directory_path)
parser.set("create", "outdir", output_directory_path)
package_manager = None
if rpm_patcher.developer_disable_patching:
package_manager = "yum"
else:
package_manager = "zypp"
parser.set("create", "pkgmgr", package_manager)
parser.set("bootstrap", "rootdir", mic_bootstrap_directory_path)
with open(mic_config_path, "wb") as mic_config:
parser.write(mic_config)
with open(mic_config_path, "r") as mic_config:
logging.debug("Using following mic config file:")
for line in mic_config:
logging.debug(line)
return mic_config_path
def combine(parameters):
"""
Combines the repostories based on parameters structure.
@param parameters The parameters of combirepo run.
"""
initialize_cache_directories(parameters.output_directory_path,
parameters.temporary_directory_path)
global target_arhcitecture
target_arhcitecture = parameters.architecture
parameters.kickstart_file_path = prepare_repositories(parameters)
original_repositories = [
repository_pair.url for repository_pair in parameters.repository_pairs
]
logging.debug("Original repository URLs: "
"{0}".format(original_repositories))
packages = resolve_groups(original_repositories,
parameters.kickstart_file_path)
logging.debug("Packages:")
for package in packages:
logging.debug(" * {0}".format(package))
names = [
repository_pair.name for repository_pair in parameters.repository_pairs
]
initialize()
combined_repositories = construct_combined_repositories(
parameters, packages)
mic_options = ["--shrink"]
if parameters.mic_options is list:
mic_options.extend(args.mic_options)
hidden_subprocess.visible_mode = True
ks_modified_path = temporaries.create_temporary_file("mod.ks")
shutil.copy(parameters.kickstart_file_path, ks_modified_path)
kickstart_file = KickstartFile(ks_modified_path)
if parameters.sup_repo_url is not None:
kickstart_file.prepend_repository_path("supplementary",
parameters.sup_repo_url)
parameters.kickstart_file_path = ks_modified_path
create_image(parameters.architecture, names, combined_repositories,
parameters.kickstart_file_path, mic_options,
parameters.package_names["service"])
hidden_subprocess.visible_mode = False
if "libasan" in parameters.package_names["service"] and libasan_preloading:
prepend_preload_library("libasan", parameters.output_directory_path)
def __prepare_image(self, graphs):
"""
Prepares the image needed for the RPM patcher.
@param graphs The list of dependency graphs of repositories.
@return The directory with preliminary images.
"""
original_images_dir = None
global developer_original_image
global developer_outdir_original
if developer_outdir_original is None:
path = temporaries.create_temporary_directory("preliminary-image")
developer_outdir_original = path
self.images_directory = developer_outdir_original
if not os.path.isdir(developer_outdir_original):
os.makedirs(developer_outdir_original)
images = files.find_fast(self.images_directory, ".*\.img$")
if (images is not None and len(images) > 0):
return
if developer_original_image is None:
if developer_outdir_original is None:
directory = temporaries.create_temporary_directory("orig")
developer_outdir_original = directory
original_images_dir = developer_outdir_original
path = temporaries.create_temporary_file("mod.ks")
shutil.copy(self.kickstart_file_path, path)
kickstart_file = KickstartFile(path)
kickstart_file.comment_all_groups()
logging.debug("Repositories: {0}".format(self.repositories))
packages = prepare_minimal_packages_list(graphs)
repository_combiner.create_image(self.architecture, self.names,
self.repositories, path,
["--outdir", original_images_dir],
packages)
else:
if os.path.isdir(developer_original_image):
original_images_dir = developer_original_image
elif os.path.isfile(developer_original_image):
original_images_dir = os.path.dirname(developer_original_image)
else:
logging.error("Given {0} is not a file or a "
"directory.".format(developer_original_image))
sys.exit("Error.")
self.images_directory = original_images_dir
def call(comment, commandline):
"""
Calls the subprocess and hides all its output.
@param comment The comment that the user will see.
@param commandline The list of command-line words to be executed.
@return The return code of the process
"""
code = 0
global counter
counter = 1
global bar_comment
bar_comment = comment
logging.debug("Running the command: {0}".format(" ".join(commandline)))
logging.debug(" in the directory {0}".format(os.getcwd()))
global visible_mode
if visible_mode:
logging.info(comment)
code = subprocess.call(commandline)
else:
log_file_name = temporaries.create_temporary_file("process.log")
global latency
timer = RepeatingTimer(latency, progress_bar_print)
timer.daemon = True
timer.start()
with open(log_file_name, 'w') as log_file:
code = subprocess.call(commandline, stdout=log_file,
stderr=log_file)
timer.cancel()
if code != 0:
logging.error("The subprocess failed!")
logging.error("STDERR output:")
with open(log_file_name, 'r') as log_file:
logging.error("{0}".format(log_file.read()))
progress_bar_print_final()
sys.stdout.write('\n')
return code
def pipe_call(comment, commandline_from, commandline_to):
"""
Calls two commands redirecting the output of first command to the second
one.
@param comment The comment that the user will see.
@param commandline_from The first command.
@param commandline_to The second command.
"""
code = 0
global counter
counter = 1
global bar_comment
bar_comment = comment
logging.debug("Running the command: {0} | "
"{1}".format(" ".join(commandline_from),
" ".join(commandline_to)))
logging.debug(" in the directory {0}".format(os.getcwd()))
log_file_name = temporaries.create_temporary_file("process.log")
global latency
timer = RepeatingTimer(latency, progress_bar_print)
timer.daemon = True
timer.start()
global visible_mode
first = subprocess.Popen(commandline_from, stdout=subprocess.PIPE)
second = subprocess.Popen(commandline_to, stdin=first.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# Allow first to receive a SIGPIPE if second exits:
first.stdout.close()
output, errors = second.communicate()
with open(log_file_name, 'w') as log_file:
log_file.write(output)
log_file.write(errors)
timer.cancel()
progress_bar_print_final()
sys.stdout.write('\n')
def resolve_groups(repositories, kickstart_file_path):
"""
Resolves packages groups from kickstart file.
@param repositories The list of original repository URLs.
@param kickstart_file_path The path to the kickstart file.
@return The list of package names.
"""
groups_paths = []
for url in repositories:
repository = Repository(url)
repository.prepare_data()
groups_data = repository.data.groups_data
if groups_data is not None and len(groups_data) > 0:
groups_path = temporaries.create_temporary_file("group.xml")
with open(groups_path, "w") as groups_file:
groups_file.writelines(groups_data)
groups_paths.append(groups_path)
logging.debug("Following groups files prepared:")
for groups_path in groups_paths:
logging.debug(" * {0}".format(groups_path))
parser = mic.kickstart.read_kickstart(kickstart_file_path)
groups = mic.kickstart.get_groups(parser)
groups_resolved = {}
for group in groups:
groups_resolved[group.name] = []
for groups_path in groups_paths:
packages = get_pkglist_in_comps(group.name, groups_path)
groups_resolved[group.name] = packages
logging.debug("Group {0} contains {1} "
"packages.".format(group.name,
len(groups_resolved[group.name])))
packages_all = []
for group_name in groups_resolved.keys():
packages_all.extend(groups_resolved[group_name])
if len(groups_resolved) != len(groups):
logging.error("Not all groups were resolved.")
sys.exit("Error.")
return packages_all
def __workaround_repodata_open_checksum_bug(self):
"""
Workarounds some bug in repodata creation.
This is a workaround for the case when tag <open-checksum> for
group.xml is not created in repomd.xml file.
Nota Bene: This somehow reproduces the standard Tizen repodata
creation. If you see repodata on release servers, group.xml in them
is not registered in repomd.xml file, but *.group.xml.gz file is
registered in it.
Without this workaround mic will fail during the repodata parsing.
"""
initial_directory = os.getcwd()
repodata_path = os.path.join(self._path, "repodata")
os.chdir(repodata_path)
backup_group_files = []
for group_file in glob.glob("*group.xml"):
backup_group_file = temporaries.create_temporary_file("group.xml")
shutil.copy(group_file, backup_group_file)
backup_group_files.append((group_file, backup_group_file))
modifyrepo_command = [
"modifyrepo", "--remove", group_file, repodata_path
]
exit_value = hidden_subprocess.silent_call(modifyrepo_command)
if exit_value != 0:
raise Exception("modifyrepo failed with exit value = "
"{0}".format(exit_value))
# Restore backuped group files, but they will not be registered in
# repomd.xml file anymore.
for backup_group_file in backup_group_files:
shutil.copy(backup_group_file[1], backup_group_file[0])
os.chdir(initial_directory)
