def init_perun_at(perun_path, is_reinit, vcs_config, config_template='master'):
"""Initialize the .perun directory at given path
Initializes or reinitializes the .perun directory at the given path.
:param path perun_path: path where new perun performance control system will be stored
:param bool is_reinit: true if this is existing perun, that will be reinitialized
:param dict vcs_config: dictionary of form {'vcs': {'type', 'url'}} for local config init
:param str config_template: name of the configuration template
"""
# Initialize the basic structure of the .perun directory
perun_full_path = os.path.join(perun_path, '.perun')
store.touch_dir(perun_full_path)
store.touch_dir(os.path.join(perun_full_path, 'objects'))
store.touch_dir(os.path.join(perun_full_path, 'jobs'))
store.touch_dir(os.path.join(perun_full_path, 'logs'))
store.touch_dir(os.path.join(perun_full_path, 'cache'))
# If the config does not exist, we initialize the new version
if not os.path.exists(os.path.join(perun_full_path, 'local.yml')):
perun_config.init_local_config_at(perun_full_path, vcs_config,
config_template)
else:
perun_log.info(
'configuration file already exists. Run ``perun config reset`` to reset'
' configuration to default state.')
# Perun successfully created
msg_prefix = "Reinitialized existing" if is_reinit else "Initialized empty"
perun_log.msg_to_stdout(
msg_prefix + " Perun repository in {}".format(perun_path), 0)
def lookup_profile_filename(profile_name):
"""Callback function for looking up the profile, if it does not exist
Arguments:
profile_name(str): value that is being read from the commandline
Returns:
str: full path to profile
"""
# 1) if it exists return the value
if os.path.exists(profile_name):
return profile_name
perun_log.info("file '{}' does not exist. Checking pending jobs...".format(
profile_name))
# 2) if it does not exists check pending
job_dir = PCS(store.locate_perun_dir_on(os.getcwd())).get_job_directory()
job_path = os.path.join(job_dir, profile_name)
if os.path.exists(job_path):
return job_path
perun_log.info(
"file '{}' not found in pending jobs...".format(profile_name))
# 3) if still not found, check recursively all candidates for match and ask for confirmation
searched_regex = re.compile(profile_name)
for root, _, files in os.walk(os.getcwd()):
for file in files:
full_path = os.path.join(root, file)
if file.endswith('.perf') and searched_regex.search(full_path):
rel_path = os.path.relpath(full_path, os.getcwd())
if click.confirm(
"did you perhaps mean '{}'?".format(rel_path)):
return full_path
return profile_name
def config_reset(store_type, config_template):
"""Resets the given store_type to a default type (or to a selected configuration template)
For more information about configuration templates see :ref:`config-templates`.
:param str store_type: name of the store (local or global) which we are resetting
:param str config_template: name of the template that we are resetting to
:raises NotPerunRepositoryException: raised when we are outside of any perun scope
"""
if store_type in ('shared', 'global'):
shared_location = perun_config.lookup_shared_config_dir()
perun_config.init_shared_config_at(shared_location)
else:
vcs_config = {
'vcs': {
'url': pcs.get_vcs_path(),
'type': pcs.get_vcs_type()
}
}
perun_config.init_local_config_at(pcs.get_path(), vcs_config,
config_template)
perun_log.info("{} configuration reset{}".format(
'global' if store_type in ('shared', 'global') else 'local',
" to {}".format(config_template) if store not in ("shared",
"global") else ""))
def register_in_index(base_dir, minor_version, registered_file,
registered_file_checksum):
"""Registers file in the index corresponding to the minor_version
If the index for the minor_version does not exist, then it is touched and initialized
with empty prefix. Then the entry is added to the file.
Arguments:
base_dir(str): base directory of the minor version
minor_version(str): sha-1 representation of the minor version of vcs (like e.g. commit)
registered_file(path): filename that is registered
registered_file_checksum(str): sha-1 representation fo the registered file
"""
# Create the directory and index (if it does not exist)
minor_dir, minor_index_file = split_object_name(base_dir, minor_version)
touch_dir(minor_dir)
touch_index(minor_index_file)
modification_stamp = timestamps.timestamp_to_str(
os.stat(registered_file).st_mtime)
entry_name = os.path.split(registered_file)[-1]
entry = IndexEntry(modification_stamp, registered_file_checksum,
entry_name, -1)
write_entry_to_index(minor_index_file, entry)
reg_rel_path = os.path.relpath(registered_file)
perun_log.info(
"'{}' successfully registered in minor version index".format(
reg_rel_path))
def lookup_profile_in_filesystem(profile_name):
"""Helper function for looking up the profile in the filesystem
First we check if the file is an absolute path, otherwise we lookup within the pending profile,
i.e. in the .perun/jobs directory. If we still have not find the profile, we then iteratively
explore the subfolders starting from current directory and look for a potential match.
:param str profile_name: value that is being read from the commandline
:returns str: full path to profile
"""
# 1) if it exists return the value
if os.path.exists(profile_name):
return profile_name
log.info("file '{}' does not exist. Checking pending jobs...".format(
profile_name))
# 2) if it does not exists check pending
job_dir = pcs.get_job_directory()
job_path = os.path.join(job_dir, profile_name)
if os.path.exists(job_path):
return job_path
log.info("file '{}' not found in pending jobs...".format(profile_name))
# 3) if still not found, check recursively all candidates for match and ask for confirmation
searched_regex = re.compile(profile_name)
for root, _, files in os.walk(os.getcwd()):
for file in files:
full_path = os.path.join(root, file)
if file.endswith('.perf') and searched_regex.search(full_path):
rel_path = os.path.relpath(full_path, os.getcwd())
if click.confirm(
"did you perhaps mean '{}'?".format(rel_path)):
return full_path
return profile_name
def remove_from_index(base_dir, minor_version, removed_file_generator, remove_all=False):
"""Removes stream of removed files from the index.
Iterates through all of the removed files, and removes their partial/full occurence from the
index. The index is walked just once.
:param str base_dir: base directory of the minor version
:param str minor_version: sha-1 representation of the minor version of vcs (like e..g commit)
:param generator removed_file_generator: generator of filenames, that will be removed from the
tracking
:param bool remove_all: true if all of the entries should be removed
"""
# Get directory and index
_, minor_version_index = split_object_name(base_dir, minor_version)
if not os.path.exists(minor_version_index):
raise EntryNotFoundException(minor_version_index)
# Lookup all entries for the given function
with open(minor_version_index, 'rb+') as index_handle:
# Gather all of the entries from the index
all_entries = [entry for entry in walk_index(index_handle)]
all_entries.sort(key=lambda unsorted_entry: unsorted_entry.offset)
removed_entries = []
for removed_file in removed_file_generator:
def lookup_function(entry):
"""Helper lookup function according to the type of the removed file"""
if is_sha1(removed_file):
return entry.checksum == removed_file
else:
return entry.path == removed_file
if remove_all:
removed_entries.append(
lookup_all_entries_within_index(index_handle, lookup_function)
)
else:
removed_entries.extend([lookup_entry_within_index(index_handle, lookup_function)])
perun_log.info("deregistered: {}".format(removed_file))
# Update number of entries
index_handle.seek(helpers.INDEX_NUMBER_OF_ENTRIES_OFFSET)
index_handle.write(struct.pack('i', len(all_entries) - len(removed_entries)))
# For each entry remove from the index, starting from the greatest offset
for entry in all_entries:
if entry in removed_entries:
continue
write_entry(index_handle, entry)
index_handle.truncate()
def _locate_workload_candidates():
"""Iterates through the filesystem tree under the current directory and tries to locate
candidate workloads.
The actual lookup is limited to subfolders such as 'workload', 'examples', 'payload' etc.
:return: list of candidate workloads
"""
log.info("Looking up candidate workloads")
workload_candidates = []
for file in UserConfiguration._all_candidate_files(
UserConfiguration.WORKLOAD_FOLDERS):
workload_candidates.append(file)
return workload_candidates
def clusterize(sorted_resources, window_width, width_measure, window_height,
height_measure, **_):
"""Clusterize the list of sorted resources w.r.t sliding window
Iterates through sorted resources, and classifies each of the resource to appropriate cluster
according to the sliding window. Each time, we fell out of the window, we recompute the sizes
(width and height) of the window and increment the cluster
:param list sorted_resources: list of sorted resource
:param float window_width: the rate of the window width, dependent on width_measure,
either percents or absolute number
:param str width_measure: type of the width measure (absolute, relative or weighted)
:param int window_height: height of the sliding window
:param str height_measure: type of the height measure (absolute or relative)
:param _: rest of the keyword arguments, not used in the function
"""
# Initialize the cluster and width
resource_number = len(sorted_resources)
current_cluster = 1
resource_width = 0
# Initialize the width and height of the window w.r.t params and first resource
current_width = compute_window_width(sorted_resources[0]['amount'],
window_width, width_measure,
resource_number)
current_height = compute_window_height(sorted_resources[0]['amount'],
window_height, height_measure)
log.info("clustering with window of ({}, {})".format(
current_width, current_height))
# Iterate through all of the resources
for resource in sorted_resources:
resource_height = resource['amount']
# If we are out of the window, we recompute the width and height and move to next cluster
if resource_width >= current_width or resource_height > current_height:
current_width = compute_window_width(resource_height, window_width,
width_measure,
resource_number)
current_height = compute_window_height(resource_height,
window_height,
height_measure)
current_cluster += 1
resource_width = 0
log.info("creating new cluster of ({}, {})".format(
current_width, current_height))
# Update the cluster of the resource
resource_width += 1
resource['cluster'] = current_cluster
def remove(profile_generator, minor_version, **kwargs):
"""Removes @p profile from the @p minor_version inside the @p pcs
:param generator profile_generator: profile that will be stored for the minor version
:param str minor_version: SHA-1 representation of the minor version
:param dict kwargs: dictionary with additional options
:raisesEntryNotFoundException: when the given profile_generator points to non-tracked profile
"""
perun_log.msg_to_stdout("Running inner wrapper of the 'perun rm'", 2)
object_directory = pcs.get_object_directory()
store.remove_from_index(object_directory, minor_version, profile_generator,
**kwargs)
perun_log.info("successfully removed {} from index".format(
len(profile_generator)))
def store_generated_profile(pcs, prof, job):
"""Stores the generated profile in the pending jobs directory.
Arguments:
pcs(PCS): object with performance control system wrapper
prof(dict): profile that we are storing in the repository
job(Job): job with additional information about generated profiles
"""
full_profile = profile.finalize_profile_for_job(pcs, prof, job)
full_profile_name = profile.generate_profile_name(job)
profile_directory = pcs.get_job_directory()
full_profile_path = os.path.join(profile_directory, full_profile_name)
profile.store_profile_at(full_profile, full_profile_path)
log.info("stored profile at: {}".format(
os.path.relpath(full_profile_path)))
def rm(profile, minor, **kwargs):
"""Unlinks the profile from the given minor version, keeping the contents
stored in ``.perun`` directory.
<profile> is unlinked in the following steps:
1. <profile> is looked up in the <hash> minor version's internal index.
2. In case <profile> is not found. An error is raised.
3. Otherwise, the record corresponding to <hash> is erased. However,
the original blob is kept in ``.perun/objects``.
If no `<hash>` is specified, then current ``HEAD`` of the wrapped version
control system is used instead. Massaging of <hash> is taken care of by
underlying version control system (e.g. git uses ``git rev-parse``).
<profile> can either be a ``index tag`` or a path specifying the profile.
``Index tags`` are in form of ``i@i``, where ``i`` stands for an index in
the minor version's index and ``@i`` is literal suffix. Run ``perun
status`` to see the `tags` of current ``HEAD``'s index.
Examples of removing profiles:
.. code-block:: bash
$ perun rm 2@i
This commands removes the third (we index from zero) profile in the index
of registered profiles of current ``HEAD``.
An error is raised if the command is executed outside of range of any
Perun or if <profile> is not found inside the <hash> index.
See :doc:`internals` for information how perun handles profiles internally.
"""
try:
commands.remove(profile, minor, **kwargs)
except (NotPerunRepositoryException, EntryNotFoundException) as exception:
perun_log.error(str(exception))
finally:
perun_log.info("removed '{}'".format(profile))
def store_generated_profile(prof, job):
"""Stores the generated profile in the pending jobs directory.
:param dict prof: profile that we are storing in the repository
:param Job job: job with additional information about generated profiles
"""
full_profile = profile.finalize_profile_for_job(prof, job)
full_profile_name = profile.generate_profile_name(full_profile)
profile_directory = pcs.get_job_directory()
full_profile_path = os.path.join(profile_directory, full_profile_name)
profile.store_profile_at(full_profile, full_profile_path)
log.info("stored profile at: {}".format(
os.path.relpath(full_profile_path)))
if dutils.strtobool(
str(
config.lookup_key_recursively("profiles.register_after_run",
"false"))):
# We either store the profile according to the origin, or we use the current head
dst = prof.get('origin', vcs.get_minor_head())
commands.add([full_profile_path], dst, keep_profile=False)
def postprocess(profile, strategy, **kwargs):
"""Takes the given profile and according to the set strategy computes clusters of resources
All of the resources are first sorted according to their uid and amounts. Then they are group
according to their uid and for each group we compute the clusters in isolation
:param dict profile: performance profile that will be clusterized
:param str strategy: name of the used strategy
(one of clustering.SUPPORTED_STRATEGIES
:param kwargs:
:return:
"""
# Flatten the resources to sorted list of resources
resources = list(
map(operator.itemgetter(1), query.all_resources_of(profile)))
resources.sort(key=resource_sort_key)
# For debug purposes, print the results
if log.is_verbose_enough(log.VERBOSE_INFO):
print_groups(resources)
# Call the concrete strategy, for each group of resources
groups = itertools.groupby(resources, resource_group_key)
for group, members in groups:
log.info("clusterizing group {}{}@{}".format(
group[0], "({})".format(group[1]) if group[1] else "", group[2]))
utils.dynamic_module_function_call('perun.postprocess.clusterizer',
strategy, 'clusterize',
list(members), **kwargs)
# For debug purposes, print the results
if log.is_verbose_enough(log.VERBOSE_INFO):
print_groups(resources)
# Return that everything is ok
return PostprocessStatus.OK, "Sucessfully clusterized the profile", dict(
kwargs)
def profile_lookup_callback(ctx, _, value):
"""
Arguments:
ctx(click.core.Context): context
_(click.core.Argument): param
value(str): value of the profile parameter
"""
# 0) First check if the value is tag or not
index_tag_match = store.INDEX_TAG_REGEX.match(value)
if index_tag_match:
index_profile = commands.get_nth_profile_of(
int(index_tag_match.group(1)), ctx.params['minor'])
return profiles.load_profile_from_file(index_profile,
is_raw_profile=False)
pending_tag_match = store.PENDING_TAG_REGEX.match(value)
if pending_tag_match:
pending_profile = lookup_nth_pending_filename(
int(pending_tag_match.group(1)))
return profiles.load_profile_from_file(pending_profile,
is_raw_profile=True)
# 1) Check the index, if this is registered
profile_from_index = commands.load_profile_from_args(
value, ctx.params['minor'])
if profile_from_index:
return profile_from_index
perun_log.info(
"file '{}' not found in index. Checking filesystem...".format(value))
# 2) Else lookup filenames and load the profile
abs_path = lookup_profile_filename(value)
if not os.path.exists(abs_path):
perun_log.error("could not find the file '{}'".format(abs_path))
return profiles.load_profile_from_file(abs_path, is_raw_profile=True)
def _locate_executable_candidates():
"""Iterates through the filesystem tree under the current directory and tries to locate
candidate executables.
The actual lookup is limited to subfolders such as 'build', '_build', 'dist' etc.
:return: list of candidate executables
"""
# Execute make before, in case there is nothing to make, then beat it
log.info("Looking up candidate executables")
log.info("Try to compile binaries for the project by running make")
try:
utils.run_safely_list_of_commands(['make'])
except subprocess.CalledProcessError:
log.info("Nothing to make...")
executable_candidates = []
for file in UserConfiguration._all_candidate_files(
UserConfiguration.EXECUTABLE_FOLDERS):
if os.path.isfile(file) and os.access(file, os.X_OK):
executable_candidates.append(file)
return executable_candidates
def add(profile_names, minor_version, keep_profile=False):
"""Appends @p profile to the @p minor_version inside the @p pcs
:param generator profile_names: generator of profiles that will be stored for the minor version
:param str minor_version: SHA-1 representation of the minor version
:param bool keep_profile: if true, then the profile that is about to be added will be not
deleted, and will be kept as it is. By default false, i.e. profile is deleted.
"""
added_profile_count = 0
for profile_name in profile_names:
# Test if the given profile exists (This should hold always, or not?)
if not os.path.exists(profile_name):
perun_log.error("{} does not exists".format(profile_name),
recoverable=True)
continue
# Load profile content
# Unpack to JSON representation
unpacked_profile = profile.load_profile_from_file(profile_name, True)
if unpacked_profile['origin'] != minor_version:
error_msg = "cannot add profile '{}' to minor index of '{}':".format(
profile_name, minor_version)
error_msg += "profile originates from minor version '{}'".format(
unpacked_profile['origin'])
perun_log.error(error_msg, recoverable=True)
continue
# Remove origin from file
unpacked_profile.pop('origin')
profile_content = profile.to_string(unpacked_profile)
# Append header to the content of the file
header = "profile {} {}\0".format(unpacked_profile['header']['type'],
len(profile_content))
profile_content = (header + profile_content).encode('utf-8')
# Transform to internal representation - file as sha1 checksum and content packed with zlib
profile_sum = store.compute_checksum(profile_content)
compressed_content = store.pack_content(profile_content)
# Add to control
object_dir = pcs.get_object_directory()
store.add_loose_object_to_dir(object_dir, profile_sum,
compressed_content)
# Register in the minor_version index
store.register_in_index(object_dir, minor_version, profile_name,
profile_sum)
# Remove the file
if not keep_profile:
os.remove(profile_name)
added_profile_count += 1
profile_names_len = len(profile_names)
if added_profile_count != profile_names_len:
perun_log.error(
"only {}/{} profiles were successfully registered in index".format(
added_profile_count, profile_names_len))
perun_log.info("successfully registered {} profiles in index".format(
added_profile_count))