def init_perun_at(perun_path, is_reinit, vcs_config, config_template='master'):
"""Initialize the .perun directory at given path
Initializes or reinitializes the .perun directory at the given path.
:param path perun_path: path where new perun performance control system will be stored
:param bool is_reinit: true if this is existing perun, that will be reinitialized
:param dict vcs_config: dictionary of form {'vcs': {'type', 'url'}} for local config init
:param str config_template: name of the configuration template
"""
# Initialize the basic structure of the .perun directory
perun_full_path = os.path.join(perun_path, '.perun')
store.touch_dir(perun_full_path)
store.touch_dir(os.path.join(perun_full_path, 'objects'))
store.touch_dir(os.path.join(perun_full_path, 'jobs'))
store.touch_dir(os.path.join(perun_full_path, 'logs'))
store.touch_dir(os.path.join(perun_full_path, 'cache'))
# If the config does not exist, we initialize the new version
if not os.path.exists(os.path.join(perun_full_path, 'local.yml')):
perun_config.init_local_config_at(perun_full_path, vcs_config,
config_template)
else:
perun_log.info(
'configuration file already exists. Run ``perun config reset`` to reset'
' configuration to default state.')
# Perun successfully created
msg_prefix = "Reinitialized existing" if is_reinit else "Initialized empty"
perun_log.msg_to_stdout(
msg_prefix + " Perun repository in {}".format(perun_path), 0)
def log(pcs, minor_version, short=False, **_):
"""Prints the log of the performance control system
Either prints the short or longer version. In short version, only header and short
list according to the formatting string from stored in the configuration. Prints
the number of profiles associated with each of the minor version and some basic
information about minor versions, like e.g. description, hash, etc.
Arguments:
pcs(PCS): object with performance control system wrapper
minor_version(str): representation of the head version
short(bool): true if the log should be in short format
"""
perun_log.msg_to_stdout("Running inner wrapper of the 'perun log '", 2)
# Print header for --short-minors
if short:
print_short_minor_info_header()
# Walk the minor versions and print them
for minor in vcs.walk_minor_versions(pcs.vcs_type, pcs.vcs_path,
minor_version):
if short:
print_short_minor_version_info(pcs, minor)
else:
cprintln("Minor Version {}".format(minor.checksum),
TEXT_EMPH_COLOUR,
attrs=TEXT_ATTRS)
base_dir = pcs.get_object_directory()
tracked_profiles = store.get_profile_number_for_minor(
base_dir, minor.checksum)
print_profile_numbers(tracked_profiles, 'tracked')
print_minor_version_info(minor, indent=1)
def write_config_to(path, config_data):
"""Stores the config data on the path
:param str path: path where the config will be stored to
:param dict config_data: dictionary with contents of the configuration
"""
perun_log.msg_to_stdout(
"Writing config '{}' at {}".format(config_data, path), 2)
with open(path, 'w') as yaml_file:
YAML().dump(config_data, yaml_file)
def write_config_file(config, path):
"""
Arguments:
config(yaml): configuration dictionary
path(str): path, where the configuration will be stored
"""
perun_log.msg_to_stdout("Writing config '{}' at {}".format(config, path),
2)
with open(path, 'w') as yaml_file:
YAML().dump(config, yaml_file)
def walk_major_versions():
"""Generator of major versions for the current wrapped repository.
This function is currently unused, but will be needed in the future.
:returns: iterable stream of major version representation
"""
vcs_type, vcs_path = pcs.get_vcs_type(), pcs.get_vcs_path()
perun_log.msg_to_stdout(
"Walking major versions of type {}".format(vcs_type), 1)
return dynamic_module_function_call('perun.vcs', vcs_type,
'_walk_major_versions', vcs_path)
def get_head_major_version():
"""Returns the string representation of current major version of the
wrapped repository.
Major version is displayed during the ``perun status`` output, which shows
the current working major version of the project.
:returns: string representation of the major version
"""
vcs_type, vcs_path = pcs.get_vcs_type(), pcs.get_vcs_path()
perun_log.msg_to_stdout(
"Getting head major version of type {}".format(vcs_type), 1)
return dynamic_module_function_call('perun.vcs', vcs_type,
'_get_head_major_version', vcs_path)
def remove(profile_generator, minor_version, **kwargs):
"""Removes @p profile from the @p minor_version inside the @p pcs
:param generator profile_generator: profile that will be stored for the minor version
:param str minor_version: SHA-1 representation of the minor version
:param dict kwargs: dictionary with additional options
:raisesEntryNotFoundException: when the given profile_generator points to non-tracked profile
"""
perun_log.msg_to_stdout("Running inner wrapper of the 'perun rm'", 2)
object_directory = pcs.get_object_directory()
store.remove_from_index(object_directory, minor_version, profile_generator,
**kwargs)
perun_log.info("successfully removed {} from index".format(
len(profile_generator)))
def write_entry_to_index(index_file, file_entry):
"""Writes the file_entry to its appropriate position within the index.
Given the file entry, writes the entry within the file, moving everything by the given offset
and then incrementing the number of entries within the index.
:param str index_file: path to the index file
:param IndexEntry file_entry: index entry that will be written to the file
"""
with open(index_file, 'rb+') as index_handle:
# Lookup the position of the registered file within the index
if file_entry.offset == -1:
try:
predicate = (
lambda entry: entry.path > file_entry.path or (
entry.path == file_entry.path and entry.time >= file_entry.time
)
)
looked_up_entry = lookup_entry_within_index(index_handle, predicate)
# If there is an exact match, we do not add the entry to the index
if looked_up_entry.path == file_entry.path and \
looked_up_entry.time == file_entry.time:
perun_log.msg_to_stdout("{0.path} ({0.time}) already registered in {1}".format(
file_entry, index_file
), 0)
return
offset_in_file = looked_up_entry.offset
except EntryNotFoundException:
# Move to end of the file and set the offset to the end of the file
index_handle.seek(0, 2)
offset_in_file = index_handle.tell()
else:
offset_in_file = file_entry.offset
# Modify the number of entries in index and return to position
modify_number_of_entries_in_index(index_handle, lambda x: x + 1)
index_handle.seek(offset_in_file)
# Read previous entries to buffer and return back to the position
buffer = index_handle.read()
index_handle.seek(offset_in_file)
# Write the index_file entry to index
write_entry(index_handle, file_entry)
# Write the stuff stored in buffer
index_handle.write(buffer)
def walk_major_versions(vcs_type, *args, **kwargs):
"""Generator of major versions for the current wrapped repository.
This function is currently unused, but will be needed in the future.
:param str vcs_type: type of the underlying wrapped version control system
:param list args: list of non-keyword arguments passed to implemenetation
of the wrapped vcs layer
:param dict kwargs: dictionary of keyword arguments passed to
implementation of the wrapped vcs layer
:returns: interable stream of major version representation
"""
perun_log.msg_to_stdout(
"Walking major versions of type {}".format(vcs_type), 1)
return dynamic_module_function_call('perun.vcs', vcs_type,
'_walk_major_versions', *args,
**kwargs)
def init(vcs_init_params):
"""Calls the implementation of initialization of wrapped underlying version
control system.
The initialization should take care of both reinitialization of existing
version control system instances and newly created instances. Init is
called during the ``perun init`` command from command line interface.
:param dict vcs_init_params: dictionary of keyword arguments passed to
initialization method of the underlying vcs module
:return: true if the underlying vcs was successfully initialized
"""
vcs_type, vcs_path = pcs.get_vcs_type(), pcs.get_vcs_path()
perun_log.msg_to_stdout(
"Initializing {} version control params {} and {}".format(
vcs_type, vcs_path, vcs_init_params), 1)
return dynamic_module_function_call('perun.vcs', vcs_type, '_init',
vcs_path, vcs_init_params)
def remove(pcs, profile_name, minor_version, **kwargs):
"""Removes @p profile from the @p minor_version inside the @p pcs
Arguments:
pcs(PCS): object with performance control system wrapper
profile_name(str): profile that will be stored for the minor version
minor_version(str): SHA-1 representation of the minor version
kwargs(dict): dictionary with additional options
Raises:
EntryNotFoundException: when the given profile_name points to non-tracked profile
"""
assert minor_version is not None and "Missing minor version specification"
perun_log.msg_to_stdout("Running inner wrapper of the 'perun rm'", 2)
object_directory = pcs.get_object_directory()
store.remove_from_index(object_directory, minor_version, profile_name,
**kwargs)
def get_head_major_version(vcs_type, *args, **kwargs):
"""Returns the string representation of current major version of the
wrapped repository.
Major version is displayed during the ``perun status`` output, which shows
the current working major version of the project.
:param str vcs_type: type of the underlying wrapped version control system
:param list args: list of non-keyword arguments passed to implemenetation
of the wrapped vcs layer
:param dict kwargs: dictionary of keyword arguments passed to
implementation of the wrapped vcs layer
:returns: string representation of the major version
"""
perun_log.msg_to_stdout(
"Getting head major version of type {}".format(vcs_type), 1)
return dynamic_module_function_call('perun.vcs', vcs_type,
'_get_head_major_version', *args,
**kwargs)
def init(dst, configuration_template='master', **kwargs):
"""Initializes the performance and version control systems
Inits the performance control system at a given directory. Optionally inits the
wrapper of the Version Control System that is used as tracking point.
:param path dst: path where the pcs will be initialized
:param dict kwargs: keyword arguments of the initialization
:param str configuration_template: name of the template that will be used for initialization
of local configuration
"""
perun_log.msg_to_stdout("call init({}, {})".format(dst, kwargs), 2)
# First init the wrapping repository well
vcs_type = kwargs['vcs_type']
vcs_path = kwargs['vcs_path'] or dst
vcs_params = kwargs['vcs_params']
# Construct local config
vcs_config = {'vcs': {'url': vcs_path, 'type': vcs_type}}
# Check if there exists perun directory above and initialize the new pcs
try:
super_perun_dir = store.locate_perun_dir_on(dst)
is_pcs_reinitialized = (super_perun_dir == dst)
if not is_pcs_reinitialized:
perun_log.warn("There exists super perun directory at {}".format(
super_perun_dir))
except NotPerunRepositoryException:
is_pcs_reinitialized = False
init_perun_at(dst, is_pcs_reinitialized, vcs_config,
configuration_template)
# If the wrapped repo could not be initialized we end with error. The user should adjust this
# himself and fix it in the config. Note that this decision was made after tagit design,
# where one needs to further adjust some options in initialized directory.
if vcs_type and not vcs.init(vcs_params):
err_msg = "Could not initialize empty {0} repository at {1}.\n".format(
vcs_type, vcs_path)
err_msg += "Either reinitialize perun with 'perun init' or initialize {0} repository"
err_msg += "manually and fix the path to vcs in 'perun config --edit'"
perun_log.error(err_msg)
def walk_minor_versions(head_minor_version):
"""Generator of minor versions for the given major version, which yields
the ``MinorVersion`` named tuples containing the following information:
``date``, ``author``, ``email``, ``checksum`` (i.e. the hash representation
of the minor version), ``commit_description`` and ``commit_parents`` (i.e.
other minor versions).
Minor versions are walked through this function during the ``perun log``
command.
:param str head_minor_version: the root minor versions which is the root
of the walk.
:returns: iterable stream of minor version representation
"""
vcs_type, vcs_path = pcs.get_vcs_type(), pcs.get_vcs_path()
perun_log.msg_to_stdout(
"Walking minor versions of type {}".format(vcs_type), 1)
return dynamic_module_function_call('perun.vcs', vcs_type,
'_walk_minor_versions', vcs_path,
head_minor_version)
def init(vcs_type, *args, **kwargs):
"""Calls the implementation of initialization of wrapped underlying version
control system.
The initialization should take care of both reinitialization of existing
version control system instances and newly created instances. Init is
called during the ``perun init`` command from command line interface.
:param str vcs_type: type of the underlying wrapped version control system
:param list args: list of non-keyword arguments passed to implemenetation
of the wrapped vcs layer
:param dict kwargs: dictionary of keyword arguments passed to
implementation of the wrapped vcs layer
:return: true if the underlying vcs was successfully initialized
"""
perun_log.msg_to_stdout(
"Initializing {} version control with params {} and {}".format(
vcs_type, args, kwargs), 1)
return dynamic_module_function_call('perun.vcs', vcs_type, '_init', *args,
**kwargs)
def init_perun_at(perun_path, is_reinit, vcs_config):
"""Initialize the .perun directory at given path
Initializes or reinitializes the .perun directory at the given path.
Arguments:
perun_path(path): path where new perun performance control system will be stored
is_reinit(bool): true if this is existing perun, that will be reinitialized
vcs_config(dict): dictionary of form {'vcs': {'type', 'url'}} for local config init
"""
# Initialize the basic structure of the .perun directory
perun_full_path = os.path.join(perun_path, '.perun')
store.touch_dir(perun_full_path)
store.touch_dir(os.path.join(perun_full_path, 'objects'))
store.touch_dir(os.path.join(perun_full_path, 'jobs'))
store.touch_dir(os.path.join(perun_full_path, 'cache'))
perun_config.init_local_config_at(perun_full_path, vcs_config)
# Perun successfully created
msg_prefix = "Reinitialized existing" if is_reinit else "Initialized empty"
perun_log.msg_to_stdout(
msg_prefix + " Perun repository in {}".format(perun_path), 0)
def get_minor_version_info(minor_version):
"""Yields the specification of concrete minor version in form of
the ``MinorVersion`` named tuples containing the following information:
``date``, ``author``, ``email``, ``checksum`` (i.e. the hash representation
of the minor version), ``commit_description`` and ``commit_parents`` (i.e.
other minor versions).
This function is a non-generator alternative of
:func:`perun.vcs.walk_minor_versions` and is used during the ``perun
status`` output to display the specifics of minor version.
:param str minor_version: the specification of minor version (in form of
sha e.g.) for which we are retrieving the details
:returns: minor version named tuple
"""
vcs_type, vcs_path = pcs.get_vcs_type(), pcs.get_vcs_path()
perun_log.msg_to_stdout(
"Getting minor version info of type {} and args {}, {}".format(
vcs_type, vcs_path, minor_version), 1)
return dynamic_module_function_call('perun.vcs', vcs_type,
'_get_minor_version_info', vcs_path,
minor_version)
def walk_minor_versions(vcs_type, *args, **kwargs):
"""Generator of minor versions for the given major version, which yields
the ``MinorVersion`` named tuples containing the following information:
``date``, ``author``, ``email``, ``checksum`` (i.e. the hash representation
of the minor version), ``commit_description`` and ``commit_parents`` (i.e.
other minor versions).
Minor versions are walked through this function during the ``perun log``
command.
:param str vcs_type: type of the underlying wrapped version control system
:param list args: list of non-keyword arguments passed to implemenetation
of the wrapped vcs layer
:param dict kwargs: dictionary of keyword arguments passed to
implementation of the wrapped vcs layer
:returns: iterable stream of minor version representation
"""
perun_log.msg_to_stdout(
"Walking minor versions of type {}".format(vcs_type), 1)
return dynamic_module_function_call('perun.vcs', vcs_type,
'_walk_minor_versions', *args,
**kwargs)
def get_minor_version_info(vcs_type, *args, **kwargs):
"""Yields the specification of concrete minor version in form of
the ``MinorVersion`` named tuples containing the following information:
``date``, ``author``, ``email``, ``checksum`` (i.e. the hash representation
of the minor version), ``commit_description`` and ``commit_parents`` (i.e.
other minor versions).
This function is a non-generator alternative of
:func:`perun.vcs.walk_minor_versions` and is used during the ``perun
status`` output to display the specifics of minor version.
:param str vcs_type: type of the underlying wrapped version control system
:param list args: list of non-keyword arguments passed to implemenetation
of the wrapped vcs layer
:param dict kwargs: dictionary of keyword arguments passed to
implementation of the wrapped vcs layer
:returns: minor version named tuple
"""
perun_log.msg_to_stdout(
"Getting minor version info of type {} and args {}, {}".format(
vcs_type, args, kwargs), 1)
return dynamic_module_function_call('perun.vcs', vcs_type,
'_get_minor_version_info', *args,
**kwargs)
def log(minor_version, short=False, **_):
"""Prints the log of the performance control system
Either prints the short or longer version. In short version, only header and short
list according to the formatting string from stored in the configuration. Prints
the number of profiles associated with each of the minor version and some basic
information about minor versions, like e.g. description, hash, etc.
:param str minor_version: representation of the head version
:param bool short: true if the log should be in short format
"""
perun_log.msg_to_stdout("Running inner wrapper of the 'perun log '", 2)
# Print header for --short-minors
if short:
minor_versions = list(vcs.walk_minor_versions(minor_version))
# Reduce the descriptions of minor version to one liners
for mv_no, minor_version in enumerate(minor_versions):
minor_versions[mv_no] = minor_version._replace(
desc=minor_version.desc.split("\n")[0])
minor_version_maxima = calculate_maximal_lengths_for_object_list(
minor_versions, MinorVersion._fields)
# Update manually the maxima for the printed supported profile types, each requires two
# characters and 9 stands for " profiles" string
def minor_stat_retriever(minor_v):
"""Helper function for picking stats of the given minor version
:param MinorVersion minor_v: minor version for which we are retrieving the stats
:return: dictionary with stats for minor version
"""
return store.get_profile_number_for_minor(
pcs.get_object_directory(), minor_v.checksum)
def deg_count_retriever(minor_v):
"""Helper function for picking stats of the degradation strings of form ++--
:param MinorVersion minor_v: minor version for which we are retrieving the stats
:return: dictionary with stats for minor version
"""
counts = perun_log.count_degradations_per_group(
store.load_degradation_list_for(pcs.get_object_directory(),
minor_v.checksum))
return {
'changes':
counts.get('Optimization', 0) * '+' +
counts.get('Degradation', 0) * '-'
}
minor_version_maxima.update(
calculate_maximal_lengths_for_stats(minor_versions,
minor_stat_retriever))
minor_version_maxima.update(
calculate_maximal_lengths_for_stats(minor_versions,
deg_count_retriever,
" changes "))
print_short_minor_version_info_list(minor_versions,
minor_version_maxima)
else:
# Walk the minor versions and print them
for minor in vcs.walk_minor_versions(minor_version):
cprintln("Minor Version {}".format(minor.checksum),
TEXT_EMPH_COLOUR,
attrs=TEXT_ATTRS)
base_dir = pcs.get_object_directory()
tracked_profiles = store.get_profile_number_for_minor(
base_dir, minor.checksum)
print_profile_numbers(tracked_profiles, 'tracked')
print_minor_version_info(minor, indent=1)