def pre_collect_profiles(minor_version):
"""For given minor version, collects performance profiles according to the job matrix
This is applied if the profiles were not already collected by this function for the given minor,
and if the key :ckey:`degradation.collect_before_check` is set to true value.
TODO: What if error happens during run matrix? This should be caught and solved
:param MinorVersion minor_version: minor version for which we are collecting the data
"""
should_precollect = dutils.strtobool(
str(
config.lookup_key_recursively('degradation.collect_before_check',
'false')))
if should_precollect and minor_version.checksum not in pre_collect_profiles.minor_version_cache:
# Set the registering after run to true for this run
config.runtime().set('profiles.register_after_run', 'true')
# Actually collect the resources
collect_to_log = dutils.strtobool(
str(
config.lookup_key_recursively('degradation.log_collect',
'false')))
log_file = os.path.join(
pcs.get_log_directory(),
"{}-precollect.log".format(minor_version.checksum))
out = log_file if collect_to_log else os.devnull
with open(out, 'w') as black_hole:
with contextlib.redirect_stdout(black_hole):
try:
runner.run_matrix_job([minor_version])
except SystemExit as system_exit:
log.warn(
"Could not precollect data for {} minor version: {}".
format(minor_version.checksum[:6], str(system_exit)))
pre_collect_profiles.minor_version_cache.add(minor_version.checksum)
def sort_profiles(profile_list, reverse_profiles=True):
"""Sorts the profiles according to the key set in either configuration.
The key can either be specified in temporary configuration, or in any of the local or global
configs as the key :ckey:`format.sort_profiles_by` attributes. Be default, profiles are sorted
by time. In case of any errors (invalid sort key or missing key) the profiles will be sorted by
default key as well.
:param list profile_list: list of ProfileInfo object
:param true reverse_profiles: true if the order of the sorting should be reversed
"""
sort_order = DEFAULT_SORT_KEY
try:
sort_order = config.lookup_key_recursively('format.sort_profiles_by')
# If the stored key is invalid, we use the default time as well
if sort_order not in ProfileInfo.valid_attributes:
perun_log.warn("invalid sort key '{}'".format(sort_order) +
" Profiles will be sorted by '{}'\n\n".format(sort_order) +
"Please set sort key in config or cli to one"
" of ({}".format(", ".join(ProfileInfo.valid_attributes)) + ")")
sort_order = DEFAULT_SORT_KEY
except MissingConfigSectionException:
perun_log.warn("missing set option 'format.sort_profiles_by'!"
" Profiles will be sorted by '{}'\n\n".format(sort_order) +
"Please run 'perun config edit' and set 'format.sort_profiles_by' to one"
" of ({}".format(", ".join(ProfileInfo.valid_attributes)) + ")")
profile_list.sort(key=operator.attrgetter(sort_order), reverse=reverse_profiles)
def get_strategies_for(profile):
"""Retrieves the best strategy for the given profile configuration
:param ProfileInfo profile: Profile information with configuration tuple
:return: method to be used for checking degradation between profiles of
the same configuration type
"""
# Retrieve the application strategy
try:
application_strategy = config.lookup_key_recursively(
'degradation.apply')
except exceptions.MissingConfigSectionException:
log.error(
"'degradation.apply' could not be found in any configuration\n"
"Run either 'perun config --local edit' or 'perun config --shared edit' and set "
" the 'degradation.apply' to suitable value (either 'first' or 'all')."
)
# Retrieve all of the strategies from configuration
strategies = config.gather_key_recursively('degradation.strategies')
already_applied_strategies = []
first_applied = False
for strategy in strategies:
if (application_strategy == 'all' or not first_applied) \
and is_rule_applicable_for(strategy, profile)\
and 'method' in strategy.keys()\
and strategy['method'] not in already_applied_strategies:
first_applied = True
method = parse_strategy(strategy['method'])
already_applied_strategies.append(method)
yield method
def config(pcs, store_type, operation, key=None, value=None, **_):
"""Updates the configuration file @p config of the @p pcs perun file
Arguments:
pcs(PCS): object with performance control system wrapper
store_type(str): type of the store (local, shared, or recursive)
operation(str): type of the operation over the (key, value) pair (get, set, or edit)
key(str): key that is looked up or stored in config
value(str): value we are setting to config
_(dict): dictionary of keyword arguments
Raises:
ExternalEditorErrorException: raised if there are any problems during invoking of external
editor during the 'edit' operation
"""
config_store = pcs.global_config() if store_type in (
'shared', 'global') else pcs.local_config()
if operation == 'get' and key:
if store_type == 'recursive':
value = perun_config.lookup_key_recursively(
pcs.get_config_dir('local'), key)
else:
value = perun_config.get_key_from_config(config_store, key)
print("{}: {}".format(key, value))
elif operation == 'set' and key and value:
perun_config.set_key_at_config(config_store, key, value)
print("Value '{1}' set for key '{0}'".format(key, value))
# Edit operation opens the configuration in the external editor
elif operation == 'edit':
# Lookup the editor in the config and run it as external command
editor = perun_config.lookup_key_recursively(pcs.path, 'global.editor')
config_file = pcs.get_config_file(store_type)
try:
utils.run_external_command([editor, config_file])
except Exception as inner_exception:
raise ExternalEditorErrorException(editor, str(inner_exception))
else:
raise InvalidConfigOperationException(store_type, operation, key,
value)
def config_get(store_type, key):
"""Gets from the store_type configuration the value of the given key.
:param str store_type: type of the store lookup (local, shared of recursive)
:param str key: list of section delimited by dot (.)
"""
config_store = pcs.global_config() if store_type in (
'shared', 'global') else pcs.local_config()
if store_type == 'recursive':
value = perun_config.lookup_key_recursively(key)
else:
value = config_store.get(key)
print("{}: {}".format(key, value))
def configure_local_perun(perun_path):
"""Configures the local perun repository with the interactive help of the user
Arguments:
perun_path(str): destination path of the perun repository
"""
pcs = PCS(perun_path)
editor = perun_config.lookup_key_recursively(pcs.path, 'global.editor')
local_config_file = pcs.get_config_file('local')
try:
utils.run_external_command([editor, local_config_file])
except ValueError as v_exception:
perun_log.error("could not invoke '{}' editor: {}".format(
editor, str(v_exception)))
def config_edit(store_type):
"""Runs the external editor stored in general.editor key in order to edit the config file.
:param str store_type: type of the store (local, shared, or recursive)
:raises MissingConfigSectionException: when the general.editor is not found in any config
:raises ExternalEditorErrorException: raised if there are any problems during invoking of
external editor during the 'edit' operation
"""
# Lookup the editor in the config and run it as external command
editor = perun_config.lookup_key_recursively('general.editor')
config_file = pcs.get_config_file(store_type)
try:
utils.run_external_command([editor, config_file])
except Exception as inner_exception:
raise ExternalEditorErrorException(editor, str(inner_exception))
def store_generated_profile(prof, job):
"""Stores the generated profile in the pending jobs directory.
:param dict prof: profile that we are storing in the repository
:param Job job: job with additional information about generated profiles
"""
full_profile = profile.finalize_profile_for_job(prof, job)
full_profile_name = profile.generate_profile_name(full_profile)
profile_directory = pcs.get_job_directory()
full_profile_path = os.path.join(profile_directory, full_profile_name)
profile.store_profile_at(full_profile, full_profile_path)
log.info("stored profile at: {}".format(
os.path.relpath(full_profile_path)))
if dutils.strtobool(
str(
config.lookup_key_recursively("profiles.register_after_run",
"false"))):
# We either store the profile according to the origin, or we use the current head
dst = prof.get('origin', vcs.get_minor_head())
commands.add([full_profile_path], dst, keep_profile=False)
def generate_profile_name(profile):
"""Constructs the profile name with the extension .perf from the job.
The profile is identified by its binary, collector, workload and the time
it was run.
Valid tags:
`%collector%`:
Name of the collector
`%postprocessors%`:
Joined list of postprocessing phases
`%<unit>.<param>%`:
Parameter of the collector given by concrete name
`%cmd%`:
Command of the job
`%args%`:
Arguments of the job
`%workload%`:
Workload of the job
`%type%`:
Type of the generated profile
`%date%`:
Current date
`%origin%`:
Origin of the profile
`%counter%`:
Increasing argument
:param dict profile: generate the corresponding profile for given name
:returns str: string for the given profile that will be stored
"""
global PROFILE_COUNTER
fmt_parser = re.Scanner([
(r"%collector%", lambda scanner, token:
lookup_value(profile['collector_info'], 'name', "_")
),
(r"%postprocessors%", lambda scanner, token:
("after-" + "-and-".join(map(lambda p: p['name'], profile['postprocessors'])))
if len(profile['postprocessors']) else '_'
),
(r"%[^.]+\.[^%]+%", lambda scanner, token:
lookup_param(profile, *token[1:-1].split('.', maxsplit=1))
),
(r"%cmd%", lambda scanner, token:
os.path.split(lookup_value(profile['header'], 'cmd', '_'))[-1]
),
(r"%args%", lambda scanner, token:
"[" + sanitize_filepart(lookup_value(profile['header'], 'args', '_')) + "]"
),
(r"%workload%", lambda scanner, token:
"[" + sanitize_filepart(
os.path.split(lookup_value(profile['header'], 'workload', '_'))[-1]
) + "]"
),
(r"%type%", lambda scanner, token: lookup_value(profile['header'], 'type', '_')),
(r"%date%", lambda scanner, token: time.strftime("%Y-%m-%d-%H-%M-%S", time.gmtime())),
(r"%origin%", lambda scanner, token: lookup_value(profile, 'origin', '_')),
(r"%counter%", lambda scanner, token: str(PROFILE_COUNTER)),
(r"%%", lambda scanner, token: token),
('[^%]+', lambda scanner, token: token)
])
PROFILE_COUNTER += 1
# Obtain the formatting template from the configuration
template = config.lookup_key_recursively('format.output_profile_template')
tokens, rest = fmt_parser.scan(template)
if rest:
perun_log.error("formatting string '{}' could not be parsed\n\n".format(template) +
"Run perun config to modify the formatting pattern. "
"Refer to documentation for more information about formatting patterns")
return "".join(tokens) + ".perf"
def print_profile_info_list(profile_list,
max_lengths,
short,
list_type='tracked'):
"""Prints list of profiles and counts per type of tracked/untracked profiles.
Prints the list of profiles, trims the sizes of each information according to the
computed maximal lengths If the output is short, the list itself is not printed,
just the information about counts. Tracked and untracked differs in colours.
:param list profile_list: list of profiles of ProfileInfo objects
:param dict max_lengths: dictionary with maximal sizes for the output of profiles
:param bool short: true if the output should be short
:param str list_type: type of the profile list (either untracked or tracked)
"""
# Sort the profiles w.r.t time of creation
profile.sort_profiles(profile_list)
# Print with padding
profile_output_colour = 'white' if list_type == 'tracked' else 'red'
index_id_char = 'i' if list_type == 'tracked' else 'p'
ending = ':\n\n' if not short else "\n"
profile_numbers = calculate_profile_numbers_per_type(profile_list)
print_profile_numbers(profile_numbers, list_type, ending)
# Skip empty profile list
profile_list_len = len(profile_list)
profile_list_width = len(str(profile_list_len))
if not profile_list_len or short:
return
# Load formating string for profile
profile_info_fmt = perun_config.lookup_key_recursively('format.status')
fmt_tokens, _ = FMT_SCANNER.scan(profile_info_fmt)
# Compute header length
header_len = profile_list_width + 3
for (token_type, token) in fmt_tokens:
if token_type == 'fmt_string':
attr_type, limit, _ = FMT_REGEX.match(token).groups()
limit = adjust_limit(limit, attr_type, max_lengths,
(2 if attr_type == 'type' else 0))
header_len += limit
else:
header_len += len(token)
cprintln("\u2550" * header_len + "\u25A3", profile_output_colour)
# Print header (2 is padding for id)
print(" ", end='')
cprint("id".center(profile_list_width + 2, ' '), profile_output_colour)
print(" ", end='')
for (token_type, token) in fmt_tokens:
if token_type == 'fmt_string':
attr_type, limit, _ = FMT_REGEX.match(token).groups()
limit = adjust_limit(limit, attr_type, max_lengths,
(2 if attr_type == 'type' else 0))
token_string = attr_type.center(limit, ' ')
cprint(token_string, profile_output_colour, [])
else:
# Print the rest (non token stuff)
cprint(token, profile_output_colour)
print("")
cprintln("\u2550" * header_len + "\u25A3", profile_output_colour)
# Print profiles
for profile_no, profile_info in enumerate(profile_list):
print(" ", end='')
cprint(
"{}@{}".format(profile_no,
index_id_char).rjust(profile_list_width + 2, ' '),
profile_output_colour)
print(" ", end='')
for (token_type, token) in fmt_tokens:
if token_type == 'fmt_string':
attr_type, limit, fill = FMT_REGEX.match(token).groups()
limit = adjust_limit(limit, attr_type, max_lengths)
print_formating_token(profile_info_fmt,
profile_info,
attr_type,
limit,
default_color=profile_output_colour,
value_fill=fill or ' ')
else:
cprint(token, profile_output_colour)
print("")
if profile_no % 5 == 0 or profile_no == profile_list_len - 1:
cprintln("\u2550" * header_len + "\u25A3", profile_output_colour)
def print_short_minor_version_info_list(minor_version_list, max_lengths):
"""Prints list of profiles and counts per type of tracked/untracked profiles.
Prints the list of profiles, trims the sizes of each information according to the
computed maximal lengths If the output is short, the list itself is not printed,
just the information about counts. Tracked and untracked differs in colours.
:param list minor_version_list: list of profiles of MinorVersionInfo objects
:param dict max_lengths: dictionary with maximal sizes for the output of profiles
"""
# Load formating string for profile
stat_length = sum([
max_lengths['all'], max_lengths['time'], max_lengths['mixed'],
max_lengths['memory']
]) + 3 + len(" profiles")
minor_version_output_colour = 'white'
minor_version_info_fmt = perun_config.lookup_key_recursively(
'format.shortlog')
fmt_tokens, _ = FMT_SCANNER.scan(minor_version_info_fmt)
slash = termcolor.colored(PROFILE_DELIMITER,
HEADER_SLASH_COLOUR,
attrs=HEADER_ATTRS)
# Print header (2 is padding for id)
for (token_type, token) in fmt_tokens:
if token_type == 'fmt_string':
attr_type, limit, _ = FMT_REGEX.match(token).groups()
if attr_type == 'stats':
end_msg = termcolor.colored(' profiles',
HEADER_SLASH_COLOUR,
attrs=HEADER_ATTRS)
print(termcolor.colored("{0}{4}{1}{4}{2}{4}{3}{5}".format(
termcolor.colored('a'.rjust(max_lengths['all']),
HEADER_COMMIT_COLOUR,
attrs=HEADER_ATTRS),
termcolor.colored('m'.rjust(max_lengths['memory']),
PROFILE_TYPE_COLOURS['memory'],
attrs=HEADER_ATTRS),
termcolor.colored('x'.rjust(max_lengths['mixed']),
PROFILE_TYPE_COLOURS['mixed'],
attrs=HEADER_ATTRS),
termcolor.colored('t'.rjust(max_lengths['time']),
PROFILE_TYPE_COLOURS['time'],
attrs=HEADER_ATTRS), slash, end_msg),
HEADER_SLASH_COLOUR,
attrs=HEADER_ATTRS),
end='')
else:
limit = adjust_limit(limit, attr_type, max_lengths)
token_string = attr_type.center(limit, ' ')
cprint(token_string,
minor_version_output_colour,
attrs=HEADER_ATTRS)
else:
# Print the rest (non token stuff)
cprint(token, minor_version_output_colour, attrs=HEADER_ATTRS)
print("")
# Print profiles
for minor_version in minor_version_list:
for (token_type, token) in fmt_tokens:
if token_type == 'fmt_string':
attr_type, limit, fill = FMT_REGEX.match(token).groups()
limit = max(
int(limit[1:]),
len(attr_type)) if limit else max_lengths[attr_type]
if attr_type == 'stats':
tracked_profiles = store.get_profile_number_for_minor(
pcs.get_object_directory(), minor_version.checksum)
if tracked_profiles['all']:
print(termcolor.colored("{:{}}".format(
tracked_profiles['all'], max_lengths['all']),
TEXT_EMPH_COLOUR,
attrs=TEXT_ATTRS),
end='')
# Print the coloured numbers
for profile_type in SUPPORTED_PROFILE_TYPES:
print("{}{}".format(
termcolor.colored(PROFILE_DELIMITER,
HEADER_SLASH_COLOUR),
termcolor.colored(
"{:{}}".format(
tracked_profiles[profile_type],
max_lengths[profile_type]),
PROFILE_TYPE_COLOURS[profile_type])),
end='')
print(termcolor.colored(" profiles",
HEADER_INFO_COLOUR,
attrs=TEXT_ATTRS),
end='')
else:
print(termcolor.colored(
'--no--profiles--'.center(stat_length),
TEXT_WARN_COLOUR,
attrs=TEXT_ATTRS),
end='')
elif attr_type == 'changes':
degradations = store.load_degradation_list_for(
pcs.get_object_directory(), minor_version.checksum)
change_string = perun_log.change_counts_to_string(
perun_log.count_degradations_per_group(degradations),
width=max_lengths['changes'])
print(change_string, end='')
else:
print_formating_token(
minor_version_info_fmt,
minor_version,
attr_type,
limit,
default_color=minor_version_output_colour,
value_fill=fill or ' ')
else:
cprint(token, minor_version_output_colour)
print("")
def create_unit_from_template(template_type, no_edit, **kwargs):
"""Function for creating a module in the perun developer directory from template
This function serves as a generator of modules and packages of units and algorithms for perun.
According to the template_type this loads a concrete set of templates, (if needed) creates a
the target directory and then initializes all of the needed modules.
If no_edit is set to true, all of the created modules, and the registration point (i.e. file,
where new modules has to be registered) are opened in the editor set in the general.config key
(which is looked up recursively).
:param str template_type: name of the template set, that will be created
:param bool no_edit: if set to true, then external editor will not be called to edit the files
:param dict kwargs: additional parameters to the concrete templates
:raises ExternalEditorErrorException: When anything bad happens when processing newly created
files with editor
"""
def template_name_filter(template_name):
"""Helper function for filtering functions which starts with the template_type name
:param str template_name: name of the template set
:return: true if the function starts with template_type
"""
return template_name.startswith(template_type)
# Lookup the perun working dir according to the current file
perun_dev_dir = os.path.abspath(
os.path.join(os.path.split(__file__)[0], ".."))
if not os.path.isdir(perun_dev_dir) \
or not os.access(perun_dev_dir, os.W_OK)\
or template_type not in os.listdir(perun_dev_dir):
log.error("cannot use {} as target developer directory".format(
perun_dev_dir) + " (either not writeable or does not exist)\n\n" +
"Perhaps you are not working from perun dev folder?")
# Initialize the jinja2 environment and load all of the templates for template_type set
env = jinja2.Environment(loader=jinja2.PackageLoader('perun', 'templates'),
autoescape=True)
list_of_templates = env.list_templates(filter_func=template_name_filter)
# Specify the target dir (for packages we create a new directory)
if "__init__" in "".join(list_of_templates):
# We will initialize it in the isolate package
target_dir = os.path.join(perun_dev_dir, template_type,
kwargs['unit_name'])
store.touch_dir(target_dir)
else:
target_dir = os.path.join(perun_dev_dir, template_type)
print("Initializing new {} module in {}".format(template_type, target_dir))
# Iterate through all of the templates and create the new files with rendered templates
successfully_created_files = []
for template_file in list_of_templates:
# Specify the target filename of the template file
template_filename, _ = os.path.splitext(template_file)
template_filename = kwargs['unit_name'] if '.' not in template_filename else \
template_filename.split('.')[1]
template_filename += ".py"
successfully_created_files.append(template_filename)
print("> creating module '{}' from template".format(template_filename),
end='')
# Render and write the template into the resulting file
with open(os.path.join(target_dir, template_filename),
'w') as template_handle:
template_handle.write(
env.get_template(template_file).render(**kwargs))
print(' [', end='')
log.cprint('DONE', 'green', attrs=['bold'])
print(']')
# Add the registration point to the set of file
successfully_created_files.append(
os.path.join(
perun_dev_dir, {
'check': os.path.join('check', '__init__.py'),
'postprocess': os.path.join('utils', '__init__.py'),
'collect': os.path.join('utils', '__init__.py'),
'view': os.path.join('utils', '__init__.py')
}.get(template_type, 'nowhere')))
print("Please register your new module in {}".format(
successfully_created_files[-1]))
# Unless specified in other way, open all of the files in the w.r.t the general.editor key
if not no_edit:
editor = config.lookup_key_recursively('general.editor')
print("Opening created files and registration point in {}".format(
editor))
try:
utils.run_external_command([editor] +
successfully_created_files[::-1])
except Exception as inner_exception:
raise ExternalEditorErrorException(editor, str(inner_exception))
def get_local_config(path_to_repo):
"""Function for loading shared (global) Perun settings
:param str path_to_repo: path to repository
:return: dictionary containing local settings
"""
try:
local_settings = config.local(path_to_repo + '/.perun')
except Exception as e:
eprint(e)
return create_response(e, 404)
output = {
'general': [],
'format': [],
'vcs': [],
'jobMatrix': {
'profiledCommands': {
'commands': [],
'arguments': [],
'workload': [],
},
'collectionSpecification': {
'collectors': [],
'postprocessors': [],
},
},
}
items = {
'general': ['paging','editor'],
'format': ['status','log','output_profile_template'],
'vcs': ['type']
}
for section, item in items.items():
for subsection in item:
try:
value = local_settings.get(section + '.' + subsection)
nearest = ''
except:
try:
nearest = config.lookup_key_recursively(section + '.' + subsection)
value = ''
except:
value = ''
nearest = '-----'
finally:
if (subsection == "type"):
output[section].append(formatter.format_configuration(subsection, value, nearest, path_to_repo, 'edit'))
else:
output[section].append(formatter.format_configuration(subsection, value, nearest, '', 'edit'))
hooks = {
'name': 'perun hooks',
'type': 'checkbox',
'actions': {
"automatically generate profiles after" : [
{ "each commit": True },
{ "each push": False },
{ "each profile registration": False },
],
"automatically detect degradation after" : [
{ "each commit": False },
{ "each push": True },
{ "each profile registration": False },
]
}
}
output['vcs'].append(hooks)
jm_items = {'cmds':[], 'args':[], 'workloads':[], 'collectors':[], 'postprocessors':[]}
for section in jm_items:
try:
jm_items[section] = local_settings.get(section)
except:
jm_items[section] = []
output['jobMatrix']['profiledCommands']['commands'] = jm_items['cmds']
output['jobMatrix']['profiledCommands']['arguments'] = jm_items['args']
output['jobMatrix']['profiledCommands']['workload'] = jm_items['workloads']
output['jobMatrix']['collectionSpecification']['collectors'] = formatter.format_job_matrix_unit(jm_items['collectors'])
output['jobMatrix']['collectionSpecification']['postprocessors'] = formatter.format_job_matrix_unit(jm_items['postprocessors'])
return jsonify({'localSettings': output})
