def degradation_between_files(baseline_file, target_file, minor_version):
"""Checks between pair of files (baseline, target) whether there are any changes in performance.
:param dict baseline_file: baseline profile we are checking against
:param dict target_file: target profile we are testing
:param str minor_version: target minor_version
"""
# First check if the configurations are compatible
baseline_config = profiles.to_config_tuple(baseline_file)
target_config = profiles.to_config_tuple(target_file)
target_minor_version = target_file.get('origin', minor_version)
if baseline_config != target_config:
log.error(
"incompatible configurations '{}' and '{}'".format(
baseline_config, target_config) +
"\n\nPerformance check does not make sense for profiles collected in different ways!"
)
detected_changes = [
(deg, profiles.config_tuple_to_cmdstr(baseline_config),
target_minor_version)
for deg in degradation_between_profiles(baseline_file, target_file)
if deg.result != PerformanceChange.NoChange
]
# Store the detected changes for given minor version
store.save_degradation_list_for(pcs.get_object_directory(),
target_minor_version, detected_changes)
print("")
log.print_list_of_degradations(detected_changes)
log.print_short_summary_of_degradations(detected_changes)
def test_degradation_between_profiles(pcs_with_degradations, capsys):
"""Set of basic tests for testing degradation between profiles
Expects correct behaviour
"""
pool_path = os.path.join(
os.path.split(__file__)[0], 'degradation_profiles')
profiles = [
factory.load_profile_from_file(
os.path.join(pool_path, 'linear_base.perf'), True),
factory.load_profile_from_file(
os.path.join(pool_path, 'linear_base_degradated.perf'), True),
factory.load_profile_from_file(
os.path.join(pool_path, 'quad_base.perf'), True),
factory.load_profile_from_file(os.path.join(pool_path, 'zero.perf'),
True)
]
# Cannot detect degradation using BMOE strategy betwen these pairs of profiles,
# since the best models are same with good confidence
result = list(bmoe.best_model_order_equality(profiles[0], profiles[1]))
assert check.PerformanceChange.NoChange in [r.result for r in result]
# Can detect degradation using BMOE strategy betwen these pairs of profiles
result = list(bmoe.best_model_order_equality(profiles[1], profiles[2]))
assert check.PerformanceChange.Degradation in [r.result for r in result]
result = list(bmoe.best_model_order_equality(profiles[0], profiles[2]))
assert check.PerformanceChange.Degradation in [r.result for r in result]
result = list(aat.average_amount_threshold(profiles[1], profiles[2]))
assert check.PerformanceChange.Degradation in [r.result for r in result]
# Can detect optimizations both using BMOE and AAT
result = list(aat.average_amount_threshold(profiles[2], profiles[1]))
assert check.PerformanceChange.Optimization in [r.result for r in result]
result = list(bmoe.best_model_order_equality(profiles[2], profiles[1]))
assert check.PerformanceChange.Optimization in [r.result for r in result]
# Try that we printed confidence
deg_list = [(res, "", "") for res in result]
log.print_list_of_degradations(deg_list)
out, _ = capsys.readouterr()
assert 'with confidence' in out
# Try that nothing is wrong when the average is 0.0
result = list(aat.average_amount_threshold(profiles[3], profiles[3]))
# Assert that DegradationInfo was yield
assert result
# Assert there was no change
assert check.PerformanceChange.NoChange in [r.result for r in result]
def degradation_in_minor(minor_version, quiet=False):
"""Checks for degradation according to the profiles stored for the given minor version.
:param str minor_version: representation of head point of degradation checking
:param bool quiet: if set to true then nothing will be printed
:returns: list of found changes
"""
minor_version_info = vcs.get_minor_version_info(minor_version)
baseline_version_queue = minor_version_info.parents
pre_collect_profiles(minor_version_info)
target_profile_queue = profiles_to_queue(minor_version)
detected_changes = []
while target_profile_queue and baseline_version_queue:
# Pop the nearest baseline
baseline = baseline_version_queue.pop(0)
# Enqueue the parents in BFS manner
baseline_info = vcs.get_minor_version_info(baseline)
baseline_version_queue.extend(baseline_info.parents)
# Precollect profiles if this is set
pre_collect_profiles(baseline_info)
# Print header if there is at least some profile to check against
baseline_profiles = profiles_to_queue(baseline)
# Iterate through the profiles and check degradation between those of same configuration
for baseline_config, baseline_profile in baseline_profiles.items():
target_profile = target_profile_queue.get(baseline_config)
cmdstr = profiles.config_tuple_to_cmdstr(baseline_config)
if target_profile:
# Print information about configuration
# and extend the list of the detected changes including the configuration
# and source minor version.
detected_changes.extend([
(deg, cmdstr, baseline_info.checksum)
for deg in degradation_between_profiles(
baseline_profile, target_profile)
if deg.result != PerformanceChange.NoChange
])
del target_profile_queue[target_profile.config_tuple]
# Store the detected degradation
store.save_degradation_list_for(pcs.get_object_directory(),
minor_version, detected_changes)
if not quiet:
log.print_list_of_degradations(detected_changes)
return detected_changes
def status(short=False, **_):
"""Prints the status of performance control system
:param bool short: true if the output should be short (i.e. without some information)
"""
# Obtain both of the heads
major_head = vcs.get_head_major_version()
minor_head = vcs.get_minor_head()
# Print the status of major head.
print("On major version {} ".format(
termcolor.colored(major_head, TEXT_EMPH_COLOUR, attrs=TEXT_ATTRS)),
end='')
# Print the index of the current head
print("(minor version: {})".format(
termcolor.colored(minor_head, TEXT_EMPH_COLOUR, attrs=TEXT_ATTRS)))
# Print in long format, the additional information about head commit, by default print
if not short:
print("")
minor_version = vcs.get_minor_version_info(minor_head)
print_minor_version_info(minor_version)
# Print profiles
minor_version_profiles = profile.load_list_for_minor_version(minor_head)
untracked_profiles = get_untracked_profiles()
maxs = calculate_maximal_lengths_for_object_list(
minor_version_profiles + untracked_profiles,
profile.ProfileInfo.valid_attributes)
print_profile_info_list(minor_version_profiles, maxs, short)
if not short:
print("")
print_profile_info_list(untracked_profiles, maxs, short, 'untracked')
# Print degradation info
degradation_list = store.load_degradation_list_for(
pcs.get_object_directory(), minor_head)
if not short:
print("")
perun_log.print_short_summary_of_degradations(degradation_list)
if not short:
print("")
perun_log.print_list_of_degradations(degradation_list)
def degradation_in_history(head):
"""Walks through the minor version starting from the given head, checking for degradation.
:param str head: starting point of the checked history for degradation.
:returns: tuple (degradation result, degradation location, degradation rate)
"""
detected_changes = []
with log.History(head) as history:
for minor_version in vcs.walk_minor_versions(head):
history.progress_to_next_minor_version(minor_version)
newly_detected_changes = degradation_in_minor(
minor_version.checksum, True)
log.print_short_change_string(
log.count_degradations_per_group(newly_detected_changes))
history.finish_minor_version(minor_version, newly_detected_changes)
log.print_list_of_degradations(newly_detected_changes)
detected_changes.extend(newly_detected_changes)
history.flush(with_border=True)
print("")
log.print_short_summary_of_degradations(detected_changes)
return detected_changes