def load_profile_from_args(profile_name, minor_version):
"""
:param Profile profile_name: profile that will be stored for the minor version
:param str minor_version: SHA-1 representation of the minor version
:returns dict: loaded profile represented as dictionary
"""
# If the profile is in raw form
if not store.is_sha1(profile_name):
_, minor_index_file = store.split_object_name(
pcs.get_object_directory(), minor_version)
# If there is nothing at all in the index, since it is not even created ;)
# we returning nothing otherwise we lookup entries in index
if not os.path.exists(minor_index_file):
return None
with open(minor_index_file, 'rb') as minor_handle:
lookup_pred = lambda entry: entry.path == profile_name
profiles = store.lookup_all_entries_within_index(
minor_handle, lookup_pred)
else:
profiles = [profile_name]
# If there are more profiles we should chose
if not profiles:
return None
chosen_profile = profiles[0]
# Peek the type if the profile is correct and load the json
_, profile_name = store.split_object_name(pcs.get_object_directory(),
chosen_profile.checksum)
profile_type = store.peek_profile_type(profile_name)
if profile_type == PROFILE_MALFORMED:
perun_log.error("malformed profile {}".format(profile_name))
loaded_profile = profile.load_profile_from_file(profile_name, False)
return loaded_profile
def load_list_for_minor_version(minor_version):
"""Returns profiles assigned to the given minor version.
:param str minor_version: identification of the commit (preferably sha1)
:returns list: list of ProfileInfo parsed from index of the given minor_version
"""
# Compute the
profiles = store.get_profile_list_for_minor(pcs.get_object_directory(), minor_version)
profile_info_list = []
for index_entry in profiles:
_, profile_name = store.split_object_name(pcs.get_object_directory(), index_entry.checksum)
profile_info \
= ProfileInfo(index_entry.path, profile_name, index_entry.time)
profile_info_list.append(profile_info)
return profile_info_list
def degradation_between_files(baseline_file, target_file, minor_version):
"""Checks between pair of files (baseline, target) whether there are any changes in performance.
:param dict baseline_file: baseline profile we are checking against
:param dict target_file: target profile we are testing
:param str minor_version: target minor_version
"""
# First check if the configurations are compatible
baseline_config = profiles.to_config_tuple(baseline_file)
target_config = profiles.to_config_tuple(target_file)
target_minor_version = target_file.get('origin', minor_version)
if baseline_config != target_config:
log.error(
"incompatible configurations '{}' and '{}'".format(
baseline_config, target_config) +
"\n\nPerformance check does not make sense for profiles collected in different ways!"
)
detected_changes = [
(deg, profiles.config_tuple_to_cmdstr(baseline_config),
target_minor_version)
for deg in degradation_between_profiles(baseline_file, target_file)
if deg.result != PerformanceChange.NoChange
]
# Store the detected changes for given minor version
store.save_degradation_list_for(pcs.get_object_directory(),
target_minor_version, detected_changes)
print("")
log.print_list_of_degradations(detected_changes)
log.print_short_summary_of_degradations(detected_changes)
def minor_stat_retriever(minor_v):
"""Helper function for picking stats of the given minor version
:param MinorVersion minor_v: minor version for which we are retrieving the stats
:return: dictionary with stats for minor version
"""
return store.get_profile_number_for_minor(
pcs.get_object_directory(), minor_v.checksum)
def deg_count_retriever(minor_v):
"""Helper function for picking stats of the degradation strings of form ++--
:param MinorVersion minor_v: minor version for which we are retrieving the stats
:return: dictionary with stats for minor version
"""
counts = perun_log.count_degradations_per_group(
store.load_degradation_list_for(pcs.get_object_directory(),
minor_v.checksum))
return {
'changes':
counts.get('Optimization', 0) * '+' +
counts.get('Degradation', 0) * '-'
}
def remove(profile_generator, minor_version, **kwargs):
"""Removes @p profile from the @p minor_version inside the @p pcs
:param generator profile_generator: profile that will be stored for the minor version
:param str minor_version: SHA-1 representation of the minor version
:param dict kwargs: dictionary with additional options
:raisesEntryNotFoundException: when the given profile_generator points to non-tracked profile
"""
perun_log.msg_to_stdout("Running inner wrapper of the 'perun rm'", 2)
object_directory = pcs.get_object_directory()
store.remove_from_index(object_directory, minor_version, profile_generator,
**kwargs)
perun_log.info("successfully removed {} from index".format(
len(profile_generator)))
def degradation_in_minor(minor_version, quiet=False):
"""Checks for degradation according to the profiles stored for the given minor version.
:param str minor_version: representation of head point of degradation checking
:param bool quiet: if set to true then nothing will be printed
:returns: list of found changes
"""
minor_version_info = vcs.get_minor_version_info(minor_version)
baseline_version_queue = minor_version_info.parents
pre_collect_profiles(minor_version_info)
target_profile_queue = profiles_to_queue(minor_version)
detected_changes = []
while target_profile_queue and baseline_version_queue:
# Pop the nearest baseline
baseline = baseline_version_queue.pop(0)
# Enqueue the parents in BFS manner
baseline_info = vcs.get_minor_version_info(baseline)
baseline_version_queue.extend(baseline_info.parents)
# Precollect profiles if this is set
pre_collect_profiles(baseline_info)
# Print header if there is at least some profile to check against
baseline_profiles = profiles_to_queue(baseline)
# Iterate through the profiles and check degradation between those of same configuration
for baseline_config, baseline_profile in baseline_profiles.items():
target_profile = target_profile_queue.get(baseline_config)
cmdstr = profiles.config_tuple_to_cmdstr(baseline_config)
if target_profile:
# Print information about configuration
# and extend the list of the detected changes including the configuration
# and source minor version.
detected_changes.extend([
(deg, cmdstr, baseline_info.checksum)
for deg in degradation_between_profiles(
baseline_profile, target_profile)
if deg.result != PerformanceChange.NoChange
])
del target_profile_queue[target_profile.config_tuple]
# Store the detected degradation
store.save_degradation_list_for(pcs.get_object_directory(),
minor_version, detected_changes)
if not quiet:
log.print_list_of_degradations(detected_changes)
return detected_changes
def status(short=False, **_):
"""Prints the status of performance control system
:param bool short: true if the output should be short (i.e. without some information)
"""
# Obtain both of the heads
major_head = vcs.get_head_major_version()
minor_head = vcs.get_minor_head()
# Print the status of major head.
print("On major version {} ".format(
termcolor.colored(major_head, TEXT_EMPH_COLOUR, attrs=TEXT_ATTRS)),
end='')
# Print the index of the current head
print("(minor version: {})".format(
termcolor.colored(minor_head, TEXT_EMPH_COLOUR, attrs=TEXT_ATTRS)))
# Print in long format, the additional information about head commit, by default print
if not short:
print("")
minor_version = vcs.get_minor_version_info(minor_head)
print_minor_version_info(minor_version)
# Print profiles
minor_version_profiles = profile.load_list_for_minor_version(minor_head)
untracked_profiles = get_untracked_profiles()
maxs = calculate_maximal_lengths_for_object_list(
minor_version_profiles + untracked_profiles,
profile.ProfileInfo.valid_attributes)
print_profile_info_list(minor_version_profiles, maxs, short)
if not short:
print("")
print_profile_info_list(untracked_profiles, maxs, short, 'untracked')
# Print degradation info
degradation_list = store.load_degradation_list_for(
pcs.get_object_directory(), minor_head)
if not short:
print("")
perun_log.print_short_summary_of_degradations(degradation_list)
if not short:
print("")
perun_log.print_list_of_degradations(degradation_list)
def print_short_minor_version_info_list(minor_version_list, max_lengths):
"""Prints list of profiles and counts per type of tracked/untracked profiles.
Prints the list of profiles, trims the sizes of each information according to the
computed maximal lengths If the output is short, the list itself is not printed,
just the information about counts. Tracked and untracked differs in colours.
:param list minor_version_list: list of profiles of MinorVersionInfo objects
:param dict max_lengths: dictionary with maximal sizes for the output of profiles
"""
# Load formating string for profile
stat_length = sum([
max_lengths['all'], max_lengths['time'], max_lengths['mixed'],
max_lengths['memory']
]) + 3 + len(" profiles")
minor_version_output_colour = 'white'
minor_version_info_fmt = perun_config.lookup_key_recursively(
'format.shortlog')
fmt_tokens, _ = FMT_SCANNER.scan(minor_version_info_fmt)
slash = termcolor.colored(PROFILE_DELIMITER,
HEADER_SLASH_COLOUR,
attrs=HEADER_ATTRS)
# Print header (2 is padding for id)
for (token_type, token) in fmt_tokens:
if token_type == 'fmt_string':
attr_type, limit, _ = FMT_REGEX.match(token).groups()
if attr_type == 'stats':
end_msg = termcolor.colored(' profiles',
HEADER_SLASH_COLOUR,
attrs=HEADER_ATTRS)
print(termcolor.colored("{0}{4}{1}{4}{2}{4}{3}{5}".format(
termcolor.colored('a'.rjust(max_lengths['all']),
HEADER_COMMIT_COLOUR,
attrs=HEADER_ATTRS),
termcolor.colored('m'.rjust(max_lengths['memory']),
PROFILE_TYPE_COLOURS['memory'],
attrs=HEADER_ATTRS),
termcolor.colored('x'.rjust(max_lengths['mixed']),
PROFILE_TYPE_COLOURS['mixed'],
attrs=HEADER_ATTRS),
termcolor.colored('t'.rjust(max_lengths['time']),
PROFILE_TYPE_COLOURS['time'],
attrs=HEADER_ATTRS), slash, end_msg),
HEADER_SLASH_COLOUR,
attrs=HEADER_ATTRS),
end='')
else:
limit = adjust_limit(limit, attr_type, max_lengths)
token_string = attr_type.center(limit, ' ')
cprint(token_string,
minor_version_output_colour,
attrs=HEADER_ATTRS)
else:
# Print the rest (non token stuff)
cprint(token, minor_version_output_colour, attrs=HEADER_ATTRS)
print("")
# Print profiles
for minor_version in minor_version_list:
for (token_type, token) in fmt_tokens:
if token_type == 'fmt_string':
attr_type, limit, fill = FMT_REGEX.match(token).groups()
limit = max(
int(limit[1:]),
len(attr_type)) if limit else max_lengths[attr_type]
if attr_type == 'stats':
tracked_profiles = store.get_profile_number_for_minor(
pcs.get_object_directory(), minor_version.checksum)
if tracked_profiles['all']:
print(termcolor.colored("{:{}}".format(
tracked_profiles['all'], max_lengths['all']),
TEXT_EMPH_COLOUR,
attrs=TEXT_ATTRS),
end='')
# Print the coloured numbers
for profile_type in SUPPORTED_PROFILE_TYPES:
print("{}{}".format(
termcolor.colored(PROFILE_DELIMITER,
HEADER_SLASH_COLOUR),
termcolor.colored(
"{:{}}".format(
tracked_profiles[profile_type],
max_lengths[profile_type]),
PROFILE_TYPE_COLOURS[profile_type])),
end='')
print(termcolor.colored(" profiles",
HEADER_INFO_COLOUR,
attrs=TEXT_ATTRS),
end='')
else:
print(termcolor.colored(
'--no--profiles--'.center(stat_length),
TEXT_WARN_COLOUR,
attrs=TEXT_ATTRS),
end='')
elif attr_type == 'changes':
degradations = store.load_degradation_list_for(
pcs.get_object_directory(), minor_version.checksum)
change_string = perun_log.change_counts_to_string(
perun_log.count_degradations_per_group(degradations),
width=max_lengths['changes'])
print(change_string, end='')
else:
print_formating_token(
minor_version_info_fmt,
minor_version,
attr_type,
limit,
default_color=minor_version_output_colour,
value_fill=fill or ' ')
else:
cprint(token, minor_version_output_colour)
print("")
def log(minor_version, short=False, **_):
"""Prints the log of the performance control system
Either prints the short or longer version. In short version, only header and short
list according to the formatting string from stored in the configuration. Prints
the number of profiles associated with each of the minor version and some basic
information about minor versions, like e.g. description, hash, etc.
:param str minor_version: representation of the head version
:param bool short: true if the log should be in short format
"""
perun_log.msg_to_stdout("Running inner wrapper of the 'perun log '", 2)
# Print header for --short-minors
if short:
minor_versions = list(vcs.walk_minor_versions(minor_version))
# Reduce the descriptions of minor version to one liners
for mv_no, minor_version in enumerate(minor_versions):
minor_versions[mv_no] = minor_version._replace(
desc=minor_version.desc.split("\n")[0])
minor_version_maxima = calculate_maximal_lengths_for_object_list(
minor_versions, MinorVersion._fields)
# Update manually the maxima for the printed supported profile types, each requires two
# characters and 9 stands for " profiles" string
def minor_stat_retriever(minor_v):
"""Helper function for picking stats of the given minor version
:param MinorVersion minor_v: minor version for which we are retrieving the stats
:return: dictionary with stats for minor version
"""
return store.get_profile_number_for_minor(
pcs.get_object_directory(), minor_v.checksum)
def deg_count_retriever(minor_v):
"""Helper function for picking stats of the degradation strings of form ++--
:param MinorVersion minor_v: minor version for which we are retrieving the stats
:return: dictionary with stats for minor version
"""
counts = perun_log.count_degradations_per_group(
store.load_degradation_list_for(pcs.get_object_directory(),
minor_v.checksum))
return {
'changes':
counts.get('Optimization', 0) * '+' +
counts.get('Degradation', 0) * '-'
}
minor_version_maxima.update(
calculate_maximal_lengths_for_stats(minor_versions,
minor_stat_retriever))
minor_version_maxima.update(
calculate_maximal_lengths_for_stats(minor_versions,
deg_count_retriever,
" changes "))
print_short_minor_version_info_list(minor_versions,
minor_version_maxima)
else:
# Walk the minor versions and print them
for minor in vcs.walk_minor_versions(minor_version):
cprintln("Minor Version {}".format(minor.checksum),
TEXT_EMPH_COLOUR,
attrs=TEXT_ATTRS)
base_dir = pcs.get_object_directory()
tracked_profiles = store.get_profile_number_for_minor(
base_dir, minor.checksum)
print_profile_numbers(tracked_profiles, 'tracked')
print_minor_version_info(minor, indent=1)
def add(profile_names, minor_version, keep_profile=False):
"""Appends @p profile to the @p minor_version inside the @p pcs
:param generator profile_names: generator of profiles that will be stored for the minor version
:param str minor_version: SHA-1 representation of the minor version
:param bool keep_profile: if true, then the profile that is about to be added will be not
deleted, and will be kept as it is. By default false, i.e. profile is deleted.
"""
added_profile_count = 0
for profile_name in profile_names:
# Test if the given profile exists (This should hold always, or not?)
if not os.path.exists(profile_name):
perun_log.error("{} does not exists".format(profile_name),
recoverable=True)
continue
# Load profile content
# Unpack to JSON representation
unpacked_profile = profile.load_profile_from_file(profile_name, True)
if unpacked_profile['origin'] != minor_version:
error_msg = "cannot add profile '{}' to minor index of '{}':".format(
profile_name, minor_version)
error_msg += "profile originates from minor version '{}'".format(
unpacked_profile['origin'])
perun_log.error(error_msg, recoverable=True)
continue
# Remove origin from file
unpacked_profile.pop('origin')
profile_content = profile.to_string(unpacked_profile)
# Append header to the content of the file
header = "profile {} {}\0".format(unpacked_profile['header']['type'],
len(profile_content))
profile_content = (header + profile_content).encode('utf-8')
# Transform to internal representation - file as sha1 checksum and content packed with zlib
profile_sum = store.compute_checksum(profile_content)
compressed_content = store.pack_content(profile_content)
# Add to control
object_dir = pcs.get_object_directory()
store.add_loose_object_to_dir(object_dir, profile_sum,
compressed_content)
# Register in the minor_version index
store.register_in_index(object_dir, minor_version, profile_name,
profile_sum)
# Remove the file
if not keep_profile:
os.remove(profile_name)
added_profile_count += 1
profile_names_len = len(profile_names)
if added_profile_count != profile_names_len:
perun_log.error(
"only {}/{} profiles were successfully registered in index".format(
added_profile_count, profile_names_len))
perun_log.info("successfully registered {} profiles in index".format(
added_profile_count))