def generate_to_install_spls(id_repository, mspl, feature_list):
"""
translate the output of the solver into a system to install (mapping from spl names to use flag selection)
:param id_repository: the id repository of hyportage
:param mspl: the mspl of hyportage
:param feature_list: the solution found by the solver
:return: a dictionary spl_name -> use flag selection
"""
# 1. translate the computed solution into a configuration
res_core = core_data.dictSet()
use_flags = []
for feature in feature_list:
el = id_repository.data_from_id(feature)
if el[0] == "package": # el = ("package", spl_name)
res_core.add_key(el[1])
else: # el = ("use", use, spl_name)
use_flags.append((el[1], el[2]))
for use_flag, spl_name in use_flags:
if spl_name in res_core:
res_core.add(spl_name, use_flag)
# 2. compares the computed solution to the actual spl use flag configuration, and generate the final configuration
res = core_data.dictSet()
for spl_name, use_selection_core in res_core.iteritems():
spl = mspl[spl_name]
if spl.use_selection_core == use_selection_core:
res[spl_name] = spl.use_selection_full
else:
annex_use_flags = spl.use_selection_full - spl.iuses_core
res[spl_name] = use_selection_core | annex_use_flags
return res
def load_packages_file(filename):
res = core_data.dictSet()
for line in parse_configuration_file(filename):
if line[0] == "*":
res.add("system", core_data.pattern_create_from_atom(line[1:]))
else:
res.add("profile", core_data.pattern_create_from_atom(line))
return res
def __init__( self, eapi, name, core, deprecated, iuses_default, use_manipulation_default, fm_local, fm_combined, keyword_set, license): """ The constructor of the class :param eapi: the EAPI of the spl/package. This information is necessary to compute the full list of use flags of the package :param name: the full name of the package, including its category, version and revision if present :param core: the core data of the package, as defined in core_data.py. it's the split of the package name into relevant information for pattern matching :param deprecated: boolean stating if the package is deprecated (i.e., is installed but not part of the portage tree anymore) :param iuses_default: the use flags declared in the IUSE variable (without the implicit ones from the profile) :param use_manipulation_default: the use manipulation declared in the IUSE variable :param fm_local: the AST corresponding to the USE_REQUIRED variable :param fm_combined: the AST corresponding to the conjunction of the DEPEND, RDEPEND and PDEPEND variables :param keyword_set: the keywords of the package :param license: the licence of the package """ ####################### # core data self.eapi = eapi self.name = name self.core = core self.is_deprecated = deprecated ####################### # feature model self.iuses_default = iuses_default # list of features declared in this spl by default self.use_manipulation_default = use_manipulation_default # default use selection self.fm_local = fm_local # the part of the feature model related to local features, i.e., portage's REQUIRED_USE self.fm_combined = fm_combined # the part of the feature model relatd to external dependencies, i.e., portage's DEPEND + RDEPEND + PDEPEND ## self.__dependencies = None # mapping from pattern dependencies to list of features they must have declared self.__revert_dependencies = core_data.dictSet() # which patterns refer to this SPL, with which features self.__required_iuses_local = None # list of local features mentioned in local constraints self.__required_iuses = None # self.__iuses_full = None # full use flag list self.__iuses_visible = None # visible use flag list self.__iuses_core = None # use flag that are relevant in the constraints self.__use_selection_full = None # selected product of this SPL self.__use_selection_core = None # selected product of this SPL, only considering the use flags relevant for the constraints ####################### # SMT self.__smt_constraint = None # translation of the full feature model into z3 constraints ####################### # visibility self.keyword_set = keyword_set # list of architectures valid for this SPL self.license = license # the licence of this SPL ## self.__unmasked = None # if portage states that this spl is masked or not self.__unmasked_keyword = None # if the keyword configuration of this package masks it self.__unmasked_license = None # if the licence configuration of this package masks it self.__installable = None # if this package is installable self.__is_stable = None # if this package is stable ####################### # graph traversal self.visited = False ####################### # initial setup self.generate_dependencies_and_requirements()
def __init__(self, name): """ The constructor of the class :param group_name: the name of the group :param spl: the first spl known to be part of this group """ self.name = name # name of the group self.spls = [] # the spls of this group self.slots_mapping = core_data.dictSet() # mapping listings all spls stored in one slot self.__smt_constraint = None # z3 constraint encoding this group
def load_installed_packages():
data = core_data.dictSet()
for directory in os.listdir(input_dir_portage_installed):
path = os.path.join(input_dir_portage_installed, directory)
for package in os.listdir(path):
package_name = directory + "/" + package
#print("looking at " + package_name)
complete_path = os.path.join(path, package)
uses = load_installed_package_uses(complete_path)
data.set(package_name, uses)
return data
###############################
# HYPORTAGE AND CONFIG DATA
# hyportage data
hyportage_db_loaded = False
id_repository = hyportage_ids.id_repository_create()
pattern_repository = hyportage_pattern.pattern_repository_create()
mspl = hyportage_data.mspl_create()
spl_groups = hyportage_data.spl_groups_create()
# configuration data
config_db_loaded = False
config = core_data.Config()
mspl_config = core_data.MSPLConfig()
keyword_list = []
installed_packages = core_data.dictSet()
world = set()
# annex info
simplify_mode = "individual"
###############################
# LOAD FUNCTIONS
def load_config(path=config_db_path_default,
save_modality=config_db_save_modality_default):
global config_db_loaded
global config, mspl_config, keyword_list, installed_packages, world
utils.phase_start("Loading the hyportage config.")
if not config_db_loaded:
def __init__(self): super(hyportage_constraint_ast.ASTVisitor, self).__init__() self.local = set() self.dependencies = core_data.dictSet() self.pattern = None
def get_user_configuration(environment):
# first is loaded the profile, which is then updated with local definitions
# 1. package.use
files_package_use = get_user_configuration_files_in_path(
os.path.join(input_dir_user_configuration, "package.use"))
pattern_use_selection = core_data.SetManipulationPattern()
for filename in files_package_use:
pattern_use_selection.update(load_package_use_file(filename))
# 2. package.accept_keywords / package.keywords
files_package_accept_keywords = get_user_configuration_files_in_path(
os.path.join(input_dir_user_configuration, "package.accept_keywords"))
pattern_accept_keywords = core_data.SetManipulationPattern()
for filename in files_package_accept_keywords:
pattern_accept_keywords.update(load_package_keywords_file(filename))
files_package_keywords = get_user_configuration_files_in_path(
os.path.join(input_dir_user_configuration, "package.keywords"))
pattern_keywords = core_data.SetManipulationPattern()
for filename in files_package_keywords:
pattern_keywords.update(load_package_keywords_file(filename))
# 3. package.mask / package.unmask
files_package_mask = get_user_configuration_files_in_path(
os.path.join(input_dir_user_configuration, "package.mask"))
pattern_mask = core_data.PatternListManipulation()
for filename in files_package_mask:
pattern_mask.update(load_package_mask_file(filename))
files_package_unmask = get_user_configuration_files_in_path(
os.path.join(input_dir_user_configuration, "package.unmask"))
pattern_unmask = core_data.PatternListManipulation()
for filename in files_package_unmask:
pattern_unmask.update(load_package_mask_file(filename))
# 6. sets
location_sets = os.path.join(input_dir_user_configuration, "sets")
pattern_required = core_data.dictSet()
if os.path.isdir(location_sets):
for filename in os.listdir(location_sets):
for line in parse_configuration_file(
os.path.join(location_sets, filename)):
pattern_required.add(filename,
core_data.pattern_create_from_atom(line))
# 7. local profile
location_local_profile = os.path.join(input_dir_user_configuration,
"profile")
if os.path.isdir(location_local_profile):
environment, config = load_profile_layer(location_local_profile,
environment)
config.update_pattern_use(pattern_use_selection)
config.update_pattern_required(pattern_required)
config.update_pattern_mask(pattern_mask)
config.update_pattern_unmask(pattern_unmask)
config.update_pattern_accept_keywords(pattern_accept_keywords)
config.update_pattern_keywords(pattern_keywords)
else:
use_selection_config = core_data.UseSelectionConfig(
pattern_use=pattern_use_selection)
config = core_data.MSPLConfig(
use_selection_config=use_selection_config,
pattern_required=pattern_required,
pattern_mask=pattern_mask,
pattern_unmask=pattern_unmask,
pattern_accept_keywords=pattern_accept_keywords,
pattern_keywords=pattern_keywords)
return environment, config
def load_profile_layer(path, environment):
#######################
# use* files
# use.force
path_use_force = os.path.join(path, "use.force")
use_force = load_use_file(path_use_force)
# use.mask
path_use_mask = os.path.join(path, "use.mask")
use_mask = load_use_file(path_use_mask)
# use.stable.force
path_use_stable_force = os.path.join(path, "use.stable.force")
use_stable_force = load_use_file(path_use_stable_force)
# use.stable.mask
path_use_stable_mask = os.path.join(path, "use.stable.mask")
use_stable_mask = load_use_file(path_use_stable_mask)
#######################
# package.use* files
# package.use
path_package_use = os.path.join(path, "package.use")
package_use = load_package_use_file(path_package_use)
# package.use.force
path_package_use_force = os.path.join(path, "package.use.force")
package_use_force = load_package_use_file(path_package_use_force)
# package.use.mask
path_package_use_mask = os.path.join(path, "package.use.mask")
package_use_mask = load_package_use_file(path_package_use_mask)
# package.use.stable.force
path_package_use_stable_force = os.path.join(path,
"package.use.stable.force")
package_use_stable_force = load_package_use_file(
path_package_use_stable_force)
# package.use.stable.mask
path_package_use_stable_mask = os.path.join(path,
"package.use.stable.mask")
package_use_stable_mask = load_package_use_file(
path_package_use_stable_mask)
#######################
# make.defaults file
path_make_defaults = os.path.join(path, "make.defaults")
if os.path.exists(path_make_defaults):
new_environment, info = load_make_defaults(path_make_defaults,
environment)
make_defaults_use_selection = info[0]
make_defaults_use_declaration_eapi4, make_defaults_use_declaration_eapi5 = info[
1], info[2]
make_defaults_use_declaration_hidden_from_user = info[3]
make_defaults_arch, make_defaults_accept_keywords = info[4], info[5]
make_defaults_use_declaration_eapi4.update(use_force.get_elements())
make_defaults_use_declaration_eapi4.update(use_mask.get_elements())
make_defaults_use_declaration_eapi4.update(
core_data.SetManipulation().add_all(
new_environment.get('BOOTSTRAP_USE',
"").split()).get_elements())
else:
new_environment = environment
make_defaults_use_selection = core_data.SetManipulation()
make_defaults_use_declaration_eapi4, make_defaults_use_declaration_eapi5 = set(
), set()
make_defaults_use_declaration_hidden_from_user = set()
make_defaults_arch, make_defaults_accept_keywords = None, core_data.SetManipulation(
)
#######################
# packages (required patterns)
path_package_required = os.path.join(path, "packages")
package_required = core_data.dictSet()
if os.path.exists(path_package_required):
for line in parse_configuration_file(path_package_required):
if line[0] == "*":
package_required.add(
"system", core_data.pattern_create_from_atom(line[1:]))
else:
package_required.add("profile",
core_data.pattern_create_from_atom(line))
#######################
# package.provided (packages implicitly provided)
path_package_provided = os.path.join(path, "package.provided")
if os.path.exists(path_package_provided):
package_provided = set(parse_configuration_file(path_package_provided))
else:
package_provided = set()
#######################
# package.mask and package.unmask (packages masking)
path_package_mask = os.path.join(path, "package.mask")
package_mask = load_package_mask_file(path_package_mask)
path_package_unmask = os.path.join(path, "package.unmask")
package_unmask = load_package_mask_file(path_package_unmask)
#######################
# package.keywords or package.accept_keywords
path_package_keywords = os.path.join(path, "package.keywords")
package_keywords = load_package_keywords_file(path_package_keywords)
path_package_accept_keywords = os.path.join(path,
"package.accept_keywords")
package_accept_keywords = load_package_keywords_file(
path_package_accept_keywords)
#######################
# return the result
res_use_selection_config = core_data.UseSelectionConfig(
make_defaults_use_selection, use_force, use_mask, use_stable_force,
use_stable_mask, package_use, package_use_force, package_use_mask,
package_use_stable_force, package_use_stable_mask)
res = core_data.MSPLConfig(make_defaults_arch,
make_defaults_use_declaration_eapi4,
make_defaults_use_declaration_eapi5,
make_defaults_use_declaration_hidden_from_user,
res_use_selection_config, package_required,
package_provided, package_mask, package_unmask,
make_defaults_accept_keywords, package_keywords,
package_accept_keywords)
return new_environment, res
def generate_installation_files(mspl, path_emerge_script,
path_use_flag_configuration,
path_mask_configuration,
path_keywords_configuration, old_installation,
new_installation):
"""
This function generates two files:
1. the script file to execute to install and uninstall the spls found by the solver
2. spl configuration file (usually package.use) from the solution found by the solver
:param mspl: the mspl of hyportage
:param path_emerge_script: path to the script file to generate
:param path_use_flag_configuration: path to the configuration file to generate
:param path_mask_configuration: the path to the unmask file to generate
:param path_keywords_configuration: the path to the accept_keywords file to generate
:param old_installation: the currently installed spls
:param new_installation: the spls to install, found by the solver
:return: None (but the script file has been generated)
"""
# the spls to emerge are the ones that are not present in the old installation, or that have a new configuration
added_spl_names = []
for spl_name, product in new_installation.iteritems():
if spl_name in old_installation:
if old_installation[spl_name] != product:
added_spl_names.append(spl_name)
else:
added_spl_names.append(spl_name)
# the spls to remove are the ones that are not in the new configuration and that are not replaced by a new version
removed_spl_names = []
new_spl_goups_info = core_data.dictSet()
for spl_name in new_installation.iterkeys():
spl = mspl[spl_name]
new_spl_goups_info.add(core_data.spl_core_get_spl_group_name(spl.core),
spl)
for spl_name in old_installation.iterkeys():
spl = mspl[spl_name]
new_versions = new_spl_goups_info.get(
core_data.spl_core_get_spl_group_name(spl.core))
if new_versions is None:
removed_spl_names.append(spl_name)
else:
replaced = False
for new_spl in new_versions:
if (spl.slot == new_spl.slot) and (spl.name != new_spl.name):
replaced = True
break
if replaced: removed_spl_names.append(spl_name)
# write the files
added_spl_names.sort()
with open(path_emerge_script, 'w') as f:
f.write("#!/bin/bash\n")
f.write("\n")
f.write("# File auto-generated by the hyportage tool\n")
f.write(
"# Do not update, any modification on this file will will overwritten by the tool\n"
)
f.write("\n")
if added_spl_names:
f.write("emerge -p --newuse " +
" ".join(["=" + spl_name
for spl_name in added_spl_names]) + "\n")
if removed_spl_names:
f.write(
"emerge -p --unmerge " +
" ".join(["=" + spl_name
for spl_name in removed_spl_names]) + "\n")
f.write("\n")
with open(path_use_flag_configuration, 'w') as f:
f.write("# File auto-generated by the hyportage tool\n")
f.write(
"# Do not update, any modification on this file will will overwritten by the tool\n"
)
f.write("\n")
for spl_name in added_spl_names:
use_selection = new_installation[spl_name]
string = "=" + spl_name + " "
string = string + " ".join(use_selection)
use_unselection = mspl[spl_name].iuses_full - use_selection
if use_unselection:
string = string + " -" + " -".join(use_unselection) + "\n"
f.write(string)
f.write("\n")
with open(path_mask_configuration, 'w') as f:
f.write("# File auto-generated by the hyportage tool\n")
f.write(
"# Do not update, any modification on this file will will overwritten by the tool\n"
)
f.write("\n")
for spl_name in added_spl_names:
f.write("=" + spl_name + "\n")
#if not mspl[spl_name].unmasked:
# f.write("=" + spl_name)
f.write("\n")
with open(path_keywords_configuration, 'w') as f:
f.write("# File auto-generated by the hyportage tool\n")
f.write(
"# Do not update, any modification on this file will will overwritten by the tool\n"
)
f.write("\n")
for spl_name in added_spl_names:
#f.write("=" + spl_name + " ~" + hyportage_db.mspl_config.arch + "\n")
f.write("=" + spl_name + " **\n")
#if not mspl[spl_name].unmasked_keyword:
# f.write("=" + spl_name + " ~" + hyportage_db.mspl_config.arch)
f.write("\n")
def solve_spls(id_repository,
config,
mspl,
spl_groups,
spls,
annex_constraint,
exploration_use,
exploration_mask,
exploration_keywords,
explain_modality=False):
"""
Solves the spls in input locally assuming that there is a command hyvar-rec
:param id_repository: the id repository of hyportage
:param config: the config of hyportage
:param mspl: the mspl of hyportage
:param spl_groups: the spl groups of hyportage
:param spls: the spls to solve
:param annex_constraint: the additional constraint to add in the solver input
:param exploration_use: boolean saying if the solver can change the use flag default selection
:param exploration_mask: boolean saying if the solver can change the use mask status of the packages
:param exploration_keywords: boolean saying if the solver can change the keywords of the packages
:param explain_modality: boolean saying if a problem should be explained (by default: False)
:return: the solution found by the solver, if it exists
"""
# 1. construct the input data for the solver
# 1.1. construct the constraint
constraint = annex_constraint[:]
spl_group_names = core_data.dictSet()
#tmp = 0
for spl in spls:
spl_group_names.add(core_data.spl_core_get_spl_group_name(spl.core),
spl)
included = (spl.unmasked
or exploration_mask) and (spl.unmasked_keyword
or exploration_keywords)
if included:
#tmp = tmp + 1
constraint.extend(spl.smt)
if exploration_use:
constraint.extend(spl.smt_use_exploration)
else:
constraint.extend(spl.smt_use_selection)
else:
#logging.info("spl \"" + spl.name + "\" is not scheduled for possible installation")
constraint.extend(spl.smt_false)
for spl_group_name, spls_tmp in spl_group_names.iteritems():
spl_group = spl_groups[spl_group_name]
constraint.extend(spl_group.smt)
for spl in spl_group:
if spl not in spls_tmp:
constraint.append(
smt_encoding.smt_to_string(
smt_encoding.get_spl_smt_not(id_repository, spl.name)))
#logging.info("included spl: " + str(tmp))
logging.debug("number of constraints to solve: " + str(len(constraint)))
# 1.2. construct the preferences
spl_names = {spl.name for spl in spls}
preferences = get_preferences_core(id_repository, mspl,
config.installed_packages, spl_names)
# 1.3. construct the current system
current_system = [
] #installed_spls_to_solver(id_repository, installed_spls, spl_names)
data_configuration = {
"selectedFeatures": current_system,
"attribute_values": [],
"context_values": []
} # current configuration
data_smt_constraints = {
"formulas": constraint,
"features": [],
"other_int_symbols": []
}
data = {
"attributes": [], # attributes of the features (empty in our case)
"contexts": [], # contexts to consider (empty in our case)
"configuration":
data_configuration,
"constraints":
[], # constraints to fill in hyvarrec format (empty in our case for efficiency)
"preferences":
preferences, # preferences in hyvarrec format
"smt_constraints":
data_smt_constraints,
"hyvar_options": [
"--features-as-boolean", "--constraints-minimization",
"--no-default-preferences"
]
}
# 2. run hyvar-rec
res = run_hyvar(data)
if res is None: return None
logging.debug("HyVarRec output: " + json.dumps(res))
# 4. managing the solver output
if res["result"] != "sat":
if explain_modality:
# todo handle explain modality when the answer is unsat
# try to print a better explanation of the constraints
constraints = get_better_constraint_visualization(
id_repository, mspl, res["constraints"])
logging.error("Conflict detected. Explanation:\n" +
"\n".join(constraints) + '\n')
return None
return generate_to_install_spls(id_repository, mspl, res['features'])