def update_smt_constraints(pattern_repository, mspl, spl_groups,
pattern_added_updated_content, spl_added_list,
implicit_use_flag_changed):
utils.phase_start("Updating the core SMT Constraints")
# the spls for which we need to recompute the constraint are:
# - all spls if the use flag changed
# (it is a good enough approximation, and computing the actual set would be far to costly)
# - otherwise the added spls and the ones with a reference to the modified patterns
# additionally, we need to update the SMT of the spl_group of these spls
#
# Note about the computation of the set of spl in case of use flag modification:
# - we need to update the constraint of all these spls
# - plus the ones that depend on these spls (because e.g. some use flags might not exist anymore)
# too costly
if implicit_use_flag_changed or (len(pattern_repository) / 2 <
len(pattern_added_updated_content)):
iterator_spl = mspl.itervalues()
iterator_spl_group = spl_groups.itervalues()
else:
spl_set = set(spl_added_list)
for pattern in pattern_added_updated_content:
spl_set.update(pattern_repository[pattern].containing_spl.keys())
spl_group_set = {spl_groups[spl.group_name] for spl in spl_set}
iterator_spl = iter(spl_set)
iterator_spl_group = iter(spl_group_set)
for spl in iterator_spl:
spl.reset_smt()
_ = spl.smt
for spl_group in iterator_spl_group:
spl_group.reset_smt()
_ = spl_group.smt
utils.phase_end("Updating completed")
def missing_externally():
utils.phase_start("Computing the USE Flags ``Missing Externally'' Statistics.")
data = {}
for pattern, el in hyportage_db.flat_pattern_repository.iteritems():
installable = False
missing = {}
for spl in hyportage_pattern.pattern_repository_element_get_spls(el, hyportage_db.mspl, hyportage_db.spl_groups):
diff = set(hyportage_pattern.pattern_repository_element_get_required_use_required(el)).difference(spl.iuses_default)
diff.difference_update(hyportage_data.keywords_get_core_set(spl.keywords_list))
if diff:
missing[spl.name] = sorted(diff)
else:
installable = True
if missing:
data[hyportage_pattern.pattern_to_atom(pattern)] = {
'installable': installable,
'missing': missing
}
res = {
'pattern_number': len(filter(lambda el: not el['installable'], data.values())),
'spl_number': len({spl_name for el in data.values() for spl_name in el['missing'].keys()}),
'use_flag_number': len({use_flag for el in data.values() for use_flags in el['missing'].values() for use_flag in use_flags}),
'data': data
}
global data
statistics['missing_externally'] = res
utils.phase_end("Computation Completed")
def update_revert_dependencies(pattern_repository, pattern_added_updated,
pattern_removed):
"""
This function resets the cached data in the spls related to the core use flags declared in the pattern.
Indeed, because the pattern repository changed, the set of core use flags of an spl may change as well.
:param pattern_repository: the hyportage pattern repository
:param pattern_added: the patterns that were added to the repository
:param pattern_updated: the patterns that were changed in the repository
:param pattern_removed: the patterns that were removed from the repository
:return: the set of spls whose cache has been reset
"""
utils.phase_start("Updating the set of externally required features.")
updated_spl_list = []
for pattern in pattern_added_updated:
pel = pattern_repository[pattern]
required_uses = pel.required_uses
for spl in pel.matched_spls:
if spl.update_revert_dependencies(pattern, required_uses):
updated_spl_list.append(spl)
for pattern in pattern_removed:
pel = pattern_repository[pattern]
for spl in pel.matched_spls:
spl.reset_revert_dependencies(pattern)
utils.phase_end("Updating completed")
return updated_spl_list
def core():
utils.phase_start("Computing the USE Flags Core Statistics.")
use_flags_max = 0
use_flags_sum = 0
use_flags_min = 100
spl_min = []
spl_max = []
nb_spl = len(hyportage_db.mspl)
for spl in hyportage_db.mspl.values():
use_flag_size = len(hyportage_data.spl_get_iuses_default(spl))
if use_flag_size < use_flags_min:
use_flags_min = use_flag_size
spl_min = [spl.name]
elif use_flag_size == use_flags_min: spl_min.append(spl.name)
if use_flag_size > use_flags_max:
use_flags_max = use_flag_size
spl_max = [spl.name]
elif use_flag_size == use_flags_max: spl_max.append(spl.name)
use_flags_sum = use_flags_sum + use_flag_size
res = {
'min': use_flags_min,
'min_spl_list': sorted(spl_min),
'max': use_flags_max,
'max_spl_list': sorted(spl_max),
'number': use_flags_sum,
'spl_number': nb_spl,
'average': use_flags_sum / nb_spl,
'keywords': len(hyportage_db.keywords)
}
global data
statistics['core'] = res
utils.phase_end("Computation Completed")
def run_local_hyvar(json_data, explain_modality, cmd, par_cores):
"""
Run hyvar locally assuming that there is a command hyvar-rec
"""
file_name = utils.get_new_temp_file(".json")
with open(file_name, "w") as f:
json.dump(json_data, f)
cmd.extend([
"--constraints-minimization", "--features-as-boolean",
"--no-default-preferences"
])
if explain_modality: cmd.append("--explain")
if par_cores > 1: cmd.extend(["-p", unicode(par_cores)])
cmd.append(file_name)
# executing the solver
utils.phase_start("Running: " + unicode(cmd))
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = process.communicate()
if process.returncode != 0:
logging.error("command ended with an error code: " +
str(process.returncode))
return None
logging.debug("Stderr of the command")
logging.debug(err)
utils.phase_end("Execution ended")
res = json.loads(out)
return res
def save_hyportage(path=config_db_path_default,
save_modality=config_db_save_modality_default):
utils.phase_start("Saving the hyportage database.")
global pattern_repository, id_repository, mspl, spl_groups
data = pattern_repository, id_repository, mspl, spl_groups
utils.store_data_file(path, data, save_modality)
utils.phase_end("Saving Completed")
def update_id_repository(id_repository, changed_ids_spl_set, spl_removed):
utils.phase_start("Updating the Id Repository")
for spl in spl_removed:
id_repository.remove_spl(spl)
for spl in changed_ids_spl_set:
id_repository.add_spl(spl)
utils.phase_end("Generation completed")
def dependencies(filter_function=host.scripts.utils.filter_function_simple):
utils.phase_start("Computing the Dependencies Statistics.")
visitor = GETGuardedDependenciesVisitor()
local_map = {
spl.name: visitor.visitSPL(spl)
for spl in hyportage_db.mspl.itervalues()
}
def extraction_function_all(data):
return data.keys()
def extraction_function_guarded(data):
return {
pattern
for pattern in data.iterkeys() if data[pattern]['guarded']
}
def extraction_function_selects(data):
return {
pattern
for pattern in data.iterkeys() if data[pattern]['selects']
}
global data
data['dependencies_all'] = generic_map(local_map, extraction_function_all,
filter_function)
data['dependencies_guarded'] = generic_map(local_map,
extraction_function_guarded,
filter_function)
data['dependencies_selects'] = generic_map(local_map,
extraction_function_selects,
filter_function)
utils.phase_end("Computation Completed")
def update_use_flag_selection(mspl, spl_added, new_keywords,
new_use_flag_config):
utils.phase_start("Updating the SPL use flag Selection")
if new_keywords or new_use_flag_config:
for spl in mspl.itervalues():
spl.reset_use_selection()
_ = spl.use_selection_core
else:
for spl in spl_added:
_ = spl.use_selection_core
utils.phase_end("Generation completed")
def pattern_refinement(
filter_function=host.scripts.utils.filter_function_simple):
utils.phase_start("Computing the Pattern (refinement) Statistics.")
def extraction_function(element):
return element.matched_spls(hyportage_db.mspl, hyportage_db.spl_groups)
global data
data['pattern_refinement'] = generic_map(
hyportage_db.flat_pattern_repository, extraction_function,
filter_function)
utils.phase_end("Computation Completed")
def graph(filter_function=host.scripts.utils.filter_function_simple):
utils.phase_start("Computing the Graph Core Statistics.")
graph_mspl, spl_nodes = graphs.mspl(filter_function, keep_self_loop=True)
nodes_spl = {node: spl for spl, node in spl_nodes.iteritems()}
visited = graph_mspl.getBooleanProperty("visited")
for n in graph_mspl.getNodes():
visited.setNodeValue(n, False)
shairplay_len = sys.maxint
cycles = []
for n in graph_mspl.getNodes():
if not visited.getNodeValue(n):
visited.setNodeValue(n, True)
path = [n]
branches = [graph_mspl.getOutNodes(n)]
if "shairplay" in nodes_spl[n].name: shairplay_len = 1
while path:
if len(path) >= shairplay_len:
print(str([nodes_spl[node].name for node in path]))
if branches[-1].hasNext():
succ = branches[-1].next()
if len(path) >= shairplay_len:
print(" found: " + nodes_spl[succ].name)
if succ in path:
if len(path) >= shairplay_len:
print(" loop found: " + str([
nodes_spl[node].name
for node in path[path.index(succ):]
]))
cycles.append([
nodes_spl[node].name
for node in path[path.index(succ):]
])
elif not visited.getNodeValue(succ):
visited.setNodeValue(succ, True)
path.append(succ)
branches.append(graph_mspl.getOutNodes(succ))
if "shairplay" in nodes_spl[succ].name:
shairplay_len = len(path)
else:
path.pop()
branches.pop()
if len(path) < shairplay_len: shairplay_len = sys.maxint
res = generic_map({tuple(v): v
for v in cycles}, core_data.identity,
host.scripts.utils.filter_function_simple)
res['cycles'] = cycles
global data
data['graph'] = res
utils.phase_end("Computation Completed")
def save_configuration(path=config_db_path_default,
save_modality=config_db_save_modality_default):
utils.phase_start("Saving the hyportage config.")
global config, mspl_config
mspl_config.new_masks = False
mspl_config.new_use_declaration_eapi4 = False
mspl_config.new_use_declaration_eapi5 = False
mspl_config.new_keywords_config = False
mspl_config.new_licenses_config = False
mspl_config.new_use_flag_config = False
utils.store_data_file(path, config, save_modality)
utils.phase_end("Saving Completed")
def statistics_pattern(
filter_function=host.scripts.utils.filter_function_simple):
utils.phase_start("Computing the Pattern Core Statistics.")
pattern_number = 0
pattern_usage = {}
pattern_usage_max = 0
for pattern_element in hyportage_db.flat_pattern_repository.itervalues():
if filter_function(pattern_element):
pattern_number = pattern_number + 1
size = len(pattern_element.containing_spl)
if pattern_usage_max < size:
pattern_usage_max = size
if size in pattern_usage:
pattern_usage[size].extend(pattern_element)
else:
pattern_usage[size] = [pattern_element]
pattern_abstraction_number = 0
pattern_abstraction_max = [0, []]
pattern_abstraction_min = [100, []]
for pattern_element in hyportage_db.flat_pattern_repository.itervalues():
if filter_function(pattern_element):
pattern_abstraction_size = len(
pattern_element.matched_spls(hyportage_db.mspl,
hyportage_db.spl_groups))
if pattern_abstraction_size < pattern_abstraction_min[0]:
pattern_abstraction_min[0] = pattern_abstraction_size
pattern_abstraction_min[1] = [pattern_element]
elif pattern_abstraction_size == pattern_abstraction_min[0]:
pattern_abstraction_min[1].append(pattern_element)
if pattern_abstraction_size > pattern_abstraction_max[0]:
pattern_abstraction_max[0] = pattern_abstraction_size
pattern_abstraction_max[1] = [pattern_element]
elif pattern_abstraction_size == pattern_abstraction_max[0]:
pattern_abstraction_max[1].append(pattern_element)
pattern_abstraction_number = pattern_abstraction_number + pattern_abstraction_size
res = {
'number': pattern_number,
'usage': pattern_usage,
'usage_max': pattern_usage_max,
'usage_average': pattern_usage_max / pattern_number,
'total_abstraction_number': pattern_abstraction_number,
'abstraction_min': pattern_abstraction_min[0],
'abstraction_min_list': pattern_abstraction_min[1],
'abstraction_max': pattern_abstraction_max[0],
'abstraction_max_list': pattern_abstraction_max[1]
}
global data
data['patterns'] = res
utils.phase_end("Computation Completed")
def features_usage(filter_function=host.scripts.utils.filter_function_simple):
utils.phase_start("Computing the USE Flags Core Statistics.")
map_features = {}
for key, value in hyportage_db.mspl.iteritems():
if filter_function(value):
for feature in hyportage_data.spl_get_required_iuses(value):
if feature in map_features: map_features[feature].add(key)
else: map_features[feature] = {key}
global data
data['feature_usage'] = generic_map(map_features, core_data.identity,
filter_function)
data['feature_usage']['map_data'] = map_features
utils.phase_end("Computation Completed")
def update_visibility(mspl, spl_added, new_masks, new_keywords, new_licenses):
utils.phase_start("Updating the SPL Visibility")
if new_masks:
for spl in mspl.itervalues():
spl.reset_unmasked()
spl.generate_visibility_data()
elif new_keywords or new_licenses:
for spl in mspl.itervalues():
spl.reset_unmasked_other()
spl.generate_visibility_data()
else:
for spl in spl_added:
spl.generate_visibility_data()
utils.phase_end("Generation completed")
def update_mspl_and_groups(mspl, spl_groups, spl_name_set, loaded_spls):
"""
This function updates the mspl and the spl_groups structures with the newly loaded spls,
while removing the spl that are not in the portage repository anymore
:param mspl: the hyportage mspl
:param spl_groups: the hyportage spl_groups
:param spl_name_set: the full set of spl names in the portage repository
:param loaded_spls: the newly loaded spls
:return: a tuple describing what changed (added spls, removed spls, added spl_groups, updated spl_groups and removed spl_groups)
Additionally, the mspl and spl_groups in parameter have been updated
"""
utils.phase_start("Updating the core hyportage data (mspl, spl_groups).")
spl_to_add, spl_to_update = [], []
for spl in loaded_spls:
if spl.name in mspl: spl_to_update.append((mspl[spl.name], spl))
else: spl_to_add.append(spl)
spl_to_remove = [
spl for spl_name, spl in mspl.iteritems()
if spl_name not in spl_name_set
]
spl_groups_added = set()
spl_groups_updated = set()
spl_groups_removed = set()
# add the added spls
for new_spl in spl_to_add:
hyportage_data.mspl_add_spl(mspl, new_spl)
added_group = hyportage_data.spl_groups_add_spl(spl_groups, new_spl)
if added_group is not None: spl_groups_added.add(added_group)
# update the updated spls
for old_spl, new_spl in spl_to_update:
hyportage_data.mspl_update_spl(mspl, old_spl, new_spl)
spl_groups_updated.add(new_spl.group_name)
# remove the removed spls
for old_spl in spl_to_remove:
hyportage_data.mspl_remove_spl(mspl, old_spl)
removed_group = hyportage_data.spl_groups_remove_spl(
spl_groups, old_spl)
if removed_group is not None: spl_groups_removed.add(removed_group)
utils.phase_end("Updating completed")
spl_added_full = loaded_spls
spl_removed_full = spl_to_remove
spl_removed_full.extend([el[0] for el in spl_to_update])
return spl_added_full, spl_removed_full, spl_groups_added, spl_groups_updated, spl_groups_removed
def load_hyportage(path=hyportage_db_path_default,
save_modality=hyportage_db_save_modality_default):
utils.phase_start("Loading the hyportage database.")
global hyportage_db_loaded
global pattern_repository, id_repository, mspl, spl_groups
if not hyportage_db_loaded:
if os.path.exists(path):
pattern_repository, id_repository, mspl, spl_groups = utils.load_data_file(
path, save_modality)
else:
logging.info("No hyportage database found: creating an empty one")
pattern_repository = hyportage_pattern.PatternRepository()
id_repository = hyportage_ids.IDRepository()
mspl = hyportage_data.mspl_create()
spl_groups = hyportage_data.spl_groups_create()
hyportage_db_loaded = True
utils.phase_end("Loading Completed")
def load_config(path=config_db_path_default,
save_modality=config_db_save_modality_default):
global config_db_loaded
global config, mspl_config, keyword_list, installed_packages, world
utils.phase_start("Loading the hyportage config.")
if not config_db_loaded:
if os.path.exists(path):
config = utils.load_data_file(path, save_modality)
else:
logging.info("No config found: creating an empty one")
config = core_data.Config()
mspl_config = config.mspl_config
keyword_list = config.keyword_list
installed_packages = config.installed_packages
world = config.world
config_db_loaded = True
utils.phase_end("Loading Completed")
def load_spl_to_load(concurrent_map, egencache_files_to_load):
"""
This function loads the spl in the list in parameter
:param concurrent_map: the map used to parallel the load
:param egencache_files_to_load: the list of files to load
:return: the list of loaded spls
"""
nb_egencache_files_to_load = len(egencache_files_to_load)
if nb_egencache_files_to_load > 0: # load new hyportage spls from egencache files
utils.phase_start("Loading the " + str(nb_egencache_files_to_load) +
" egencache files.")
loaded_spls = concurrent_map(
utils_egencache.create_spl_from_egencache_file,
egencache_files_to_load)
utils.phase_end("Loading completed")
else:
loaded_spls = []
return loaded_spls
def update_pattern_repository(pattern_repository, spl_added, spl_removed):
"""
This function updates the pattern repository in input with the newly added spls and the ones removed. In particular,
this function updates the set of patterns in the repository to only contains those referenced in the mspl,
and reset the cache of the patterns that are modified by the addition and removal of the spls in parameter
:param pattern_repository: the pattern repository to update
:param spl_added: the spls that are added to the mspl
:param spl_removed: the spls that are removed from the mspl
:return: the set of patterns that are added, updated and removed during the update
"""
utils.phase_start(
"Updating the pattern hyportage data (pattern_repository).")
pattern_added, pattern_updated_containing, pattern_removed = \
__update_pattern_repository_spl_dependencies(pattern_repository, spl_added, spl_removed)
pattern_updated_content = __update_pattern_repository_reset_pel(
pattern_repository, spl_added, spl_removed)
utils.phase_end("Updating completed")
return pattern_added, pattern_updated_containing, pattern_updated_content, pattern_removed
def missing_locally():
utils.phase_start("Computing the USE Flags ``Missing Locally'' Statistics.")
data = {}
for spl in hyportage_db.mspl.values():
diff = spl.required_iuses_local.difference(spl.iuses_default)
diff.difference_update(hyportage_data.keywords_get_core_set(spl.keywords_list))
if diff:
data[spl.name] = sorted(diff)
missing_use_flags = {use_flag for use_flags in data.values() for use_flag in use_flags}
res = {
'use_flags': sorted(missing_use_flags),
'use_flag_number': len(missing_use_flags),
'spl_number': len(data),
'data': data
}
global data
statistics['missing_locally'] = res
utils.phase_end("Computation Completed")
def reset_implicit_features(mspl, is_eapi4_updated, is_eapi5_updated):
"""
This function resets the cached data of the spls if the implicit use flags changed
:param mspl: the hyportage mspl
:param is_eapi4_updated: if the implicit use fags for eapi4 or less changed
:param is_eapi5_updated: if the implicit use fags for eapi5 or more changed
:return: the list of updated spls
"""
utils.phase_start("Adding the implicit Features to the spls.")
updated_spl_list = []
if is_eapi4_updated or is_eapi5_updated:
for spl in mspl.itervalues():
if (spl.eapi < 5) and is_eapi4_updated:
spl.reset_iuses_full()
updated_spl_list.append(spl)
elif (spl.eapi > 4) and is_eapi5_updated:
spl.reset_iuses_full()
updated_spl_list.append(spl)
utils.phase_end("Addition completed")
return updated_spl_list
def run_remote_hyvar(json_data, explain_modality, url):
"""
Run hyvar
"""
if explain_modality: url += "/explain"
else: url += "/process"
utils.phase_start("Invoking url: " + url)
response = requests.post(url,
data=json.dumps(json_data),
headers={'content-type': 'application/json'})
utils.phase_end("Execution ended")
if response.status_code != requests.codes.ok:
logging.error("server answered with an error code: " +
str(response.status_code))
return None
res = response.json()
if 'error' in res:
logging.error("server answered with an error message: " +
json.dumps(res))
return None
return res
def features(filter_function=host.scripts.utils.filter_function_simple):
utils.phase_start("Computing the USE Flags Core Statistics.")
required = sum([
len(spl.required_iuses) for spl in hyportage_db.mspl.itervalues()
if filter_function(spl)
])
local = sum([
len(spl.iuses_default) for spl in hyportage_db.mspl.itervalues()
if filter_function(spl)
])
global data
data['features'] = generic_map(hyportage_db.mspl,
hyportage_data.spl_get_iuses_full,
filter_function)
data['features']['average_required'] = required / float(
data['features']['number'])
data['features']['average_local'] = local / float(
data['features']['number'])
utils.phase_end("Computation Completed")
def compute_to_load(last_update, force, path_egencache_packages):
"""
This function collects the names of the packages in the portage repository, and lists the ones that need to be loaded
:param last_update: the date of the last update of the hyportage repository
:param force: the list of patterns to force to load
:param path_egencache_packages: the path to the portage repository
:return: a pair consisting of the list of files to load, plus the set of packages in the portage repository
(so we know which spl must be removed from hyportage)
"""
utils.phase_start("Computing what to do.")
egencache_files = utils_egencache.get_egencache_files(
path_egencache_packages)
if force:
atom_list = force.split()
patterns = [
hyportage_pattern.pattern_create_from_atom(atom)
for atom in atom_list
]
def filter_function(path_file):
return filter_egencache_file_full(path_file, last_update, patterns)
else:
def filter_function(path_file):
return filter_egencache_file(path_file, last_update)
egencache_files_to_load = filter(filter_function, egencache_files)
logging.info("number of egencache files found: " +
str(len(egencache_files)))
logging.info("number of egencache files to load: " +
str(len(egencache_files_to_load)))
utils.phase_end("Computation completed")
return egencache_files_to_load, {
utils_egencache.get_package_name_from_path(f)[0]
for f in egencache_files
}
