def init_perun_at(perun_path, is_reinit, vcs_config, config_template='master'): """Initialize the .perun directory at given path Initializes or reinitializes the .perun directory at the given path. :param path perun_path: path where new perun performance control system will be stored :param bool is_reinit: true if this is existing perun, that will be reinitialized :param dict vcs_config: dictionary of form {'vcs': {'type', 'url'}} for local config init :param str config_template: name of the configuration template """ # Initialize the basic structure of the .perun directory perun_full_path = os.path.join(perun_path, '.perun') store.touch_dir(perun_full_path) store.touch_dir(os.path.join(perun_full_path, 'objects')) store.touch_dir(os.path.join(perun_full_path, 'jobs')) store.touch_dir(os.path.join(perun_full_path, 'logs')) store.touch_dir(os.path.join(perun_full_path, 'cache')) # If the config does not exist, we initialize the new version if not os.path.exists(os.path.join(perun_full_path, 'local.yml')): perun_config.init_local_config_at(perun_full_path, vcs_config, config_template) else: perun_log.info( 'configuration file already exists. Run ``perun config reset`` to reset' ' configuration to default state.') # Perun successfully created msg_prefix = "Reinitialized existing" if is_reinit else "Initialized empty" perun_log.msg_to_stdout( msg_prefix + " Perun repository in {}".format(perun_path), 0)
def log(pcs, minor_version, short=False, **_): """Prints the log of the performance control system Either prints the short or longer version. In short version, only header and short list according to the formatting string from stored in the configuration. Prints the number of profiles associated with each of the minor version and some basic information about minor versions, like e.g. description, hash, etc. Arguments: pcs(PCS): object with performance control system wrapper minor_version(str): representation of the head version short(bool): true if the log should be in short format """ perun_log.msg_to_stdout("Running inner wrapper of the 'perun log '", 2) # Print header for --short-minors if short: print_short_minor_info_header() # Walk the minor versions and print them for minor in vcs.walk_minor_versions(pcs.vcs_type, pcs.vcs_path, minor_version): if short: print_short_minor_version_info(pcs, minor) else: cprintln("Minor Version {}".format(minor.checksum), TEXT_EMPH_COLOUR, attrs=TEXT_ATTRS) base_dir = pcs.get_object_directory() tracked_profiles = store.get_profile_number_for_minor( base_dir, minor.checksum) print_profile_numbers(tracked_profiles, 'tracked') print_minor_version_info(minor, indent=1)
def write_config_to(path, config_data): """Stores the config data on the path :param str path: path where the config will be stored to :param dict config_data: dictionary with contents of the configuration """ perun_log.msg_to_stdout( "Writing config '{}' at {}".format(config_data, path), 2) with open(path, 'w') as yaml_file: YAML().dump(config_data, yaml_file)
def write_config_file(config, path): """ Arguments: config(yaml): configuration dictionary path(str): path, where the configuration will be stored """ perun_log.msg_to_stdout("Writing config '{}' at {}".format(config, path), 2) with open(path, 'w') as yaml_file: YAML().dump(config, yaml_file)
def walk_major_versions(): """Generator of major versions for the current wrapped repository. This function is currently unused, but will be needed in the future. :returns: iterable stream of major version representation """ vcs_type, vcs_path = pcs.get_vcs_type(), pcs.get_vcs_path() perun_log.msg_to_stdout( "Walking major versions of type {}".format(vcs_type), 1) return dynamic_module_function_call('perun.vcs', vcs_type, '_walk_major_versions', vcs_path)
def get_head_major_version(): """Returns the string representation of current major version of the wrapped repository. Major version is displayed during the ``perun status`` output, which shows the current working major version of the project. :returns: string representation of the major version """ vcs_type, vcs_path = pcs.get_vcs_type(), pcs.get_vcs_path() perun_log.msg_to_stdout( "Getting head major version of type {}".format(vcs_type), 1) return dynamic_module_function_call('perun.vcs', vcs_type, '_get_head_major_version', vcs_path)
def remove(profile_generator, minor_version, **kwargs): """Removes @p profile from the @p minor_version inside the @p pcs :param generator profile_generator: profile that will be stored for the minor version :param str minor_version: SHA-1 representation of the minor version :param dict kwargs: dictionary with additional options :raisesEntryNotFoundException: when the given profile_generator points to non-tracked profile """ perun_log.msg_to_stdout("Running inner wrapper of the 'perun rm'", 2) object_directory = pcs.get_object_directory() store.remove_from_index(object_directory, minor_version, profile_generator, **kwargs) perun_log.info("successfully removed {} from index".format( len(profile_generator)))
def write_entry_to_index(index_file, file_entry): """Writes the file_entry to its appropriate position within the index. Given the file entry, writes the entry within the file, moving everything by the given offset and then incrementing the number of entries within the index. :param str index_file: path to the index file :param IndexEntry file_entry: index entry that will be written to the file """ with open(index_file, 'rb+') as index_handle: # Lookup the position of the registered file within the index if file_entry.offset == -1: try: predicate = ( lambda entry: entry.path > file_entry.path or ( entry.path == file_entry.path and entry.time >= file_entry.time ) ) looked_up_entry = lookup_entry_within_index(index_handle, predicate) # If there is an exact match, we do not add the entry to the index if looked_up_entry.path == file_entry.path and \ looked_up_entry.time == file_entry.time: perun_log.msg_to_stdout("{0.path} ({0.time}) already registered in {1}".format( file_entry, index_file ), 0) return offset_in_file = looked_up_entry.offset except EntryNotFoundException: # Move to end of the file and set the offset to the end of the file index_handle.seek(0, 2) offset_in_file = index_handle.tell() else: offset_in_file = file_entry.offset # Modify the number of entries in index and return to position modify_number_of_entries_in_index(index_handle, lambda x: x + 1) index_handle.seek(offset_in_file) # Read previous entries to buffer and return back to the position buffer = index_handle.read() index_handle.seek(offset_in_file) # Write the index_file entry to index write_entry(index_handle, file_entry) # Write the stuff stored in buffer index_handle.write(buffer)
def walk_major_versions(vcs_type, *args, **kwargs): """Generator of major versions for the current wrapped repository. This function is currently unused, but will be needed in the future. :param str vcs_type: type of the underlying wrapped version control system :param list args: list of non-keyword arguments passed to implemenetation of the wrapped vcs layer :param dict kwargs: dictionary of keyword arguments passed to implementation of the wrapped vcs layer :returns: interable stream of major version representation """ perun_log.msg_to_stdout( "Walking major versions of type {}".format(vcs_type), 1) return dynamic_module_function_call('perun.vcs', vcs_type, '_walk_major_versions', *args, **kwargs)
def init(vcs_init_params): """Calls the implementation of initialization of wrapped underlying version control system. The initialization should take care of both reinitialization of existing version control system instances and newly created instances. Init is called during the ``perun init`` command from command line interface. :param dict vcs_init_params: dictionary of keyword arguments passed to initialization method of the underlying vcs module :return: true if the underlying vcs was successfully initialized """ vcs_type, vcs_path = pcs.get_vcs_type(), pcs.get_vcs_path() perun_log.msg_to_stdout( "Initializing {} version control params {} and {}".format( vcs_type, vcs_path, vcs_init_params), 1) return dynamic_module_function_call('perun.vcs', vcs_type, '_init', vcs_path, vcs_init_params)
def remove(pcs, profile_name, minor_version, **kwargs): """Removes @p profile from the @p minor_version inside the @p pcs Arguments: pcs(PCS): object with performance control system wrapper profile_name(str): profile that will be stored for the minor version minor_version(str): SHA-1 representation of the minor version kwargs(dict): dictionary with additional options Raises: EntryNotFoundException: when the given profile_name points to non-tracked profile """ assert minor_version is not None and "Missing minor version specification" perun_log.msg_to_stdout("Running inner wrapper of the 'perun rm'", 2) object_directory = pcs.get_object_directory() store.remove_from_index(object_directory, minor_version, profile_name, **kwargs)
def get_head_major_version(vcs_type, *args, **kwargs): """Returns the string representation of current major version of the wrapped repository. Major version is displayed during the ``perun status`` output, which shows the current working major version of the project. :param str vcs_type: type of the underlying wrapped version control system :param list args: list of non-keyword arguments passed to implemenetation of the wrapped vcs layer :param dict kwargs: dictionary of keyword arguments passed to implementation of the wrapped vcs layer :returns: string representation of the major version """ perun_log.msg_to_stdout( "Getting head major version of type {}".format(vcs_type), 1) return dynamic_module_function_call('perun.vcs', vcs_type, '_get_head_major_version', *args, **kwargs)
def init(dst, configuration_template='master', **kwargs): """Initializes the performance and version control systems Inits the performance control system at a given directory. Optionally inits the wrapper of the Version Control System that is used as tracking point. :param path dst: path where the pcs will be initialized :param dict kwargs: keyword arguments of the initialization :param str configuration_template: name of the template that will be used for initialization of local configuration """ perun_log.msg_to_stdout("call init({}, {})".format(dst, kwargs), 2) # First init the wrapping repository well vcs_type = kwargs['vcs_type'] vcs_path = kwargs['vcs_path'] or dst vcs_params = kwargs['vcs_params'] # Construct local config vcs_config = {'vcs': {'url': vcs_path, 'type': vcs_type}} # Check if there exists perun directory above and initialize the new pcs try: super_perun_dir = store.locate_perun_dir_on(dst) is_pcs_reinitialized = (super_perun_dir == dst) if not is_pcs_reinitialized: perun_log.warn("There exists super perun directory at {}".format( super_perun_dir)) except NotPerunRepositoryException: is_pcs_reinitialized = False init_perun_at(dst, is_pcs_reinitialized, vcs_config, configuration_template) # If the wrapped repo could not be initialized we end with error. The user should adjust this # himself and fix it in the config. Note that this decision was made after tagit design, # where one needs to further adjust some options in initialized directory. if vcs_type and not vcs.init(vcs_params): err_msg = "Could not initialize empty {0} repository at {1}.\n".format( vcs_type, vcs_path) err_msg += "Either reinitialize perun with 'perun init' or initialize {0} repository" err_msg += "manually and fix the path to vcs in 'perun config --edit'" perun_log.error(err_msg)
def walk_minor_versions(head_minor_version): """Generator of minor versions for the given major version, which yields the ``MinorVersion`` named tuples containing the following information: ``date``, ``author``, ``email``, ``checksum`` (i.e. the hash representation of the minor version), ``commit_description`` and ``commit_parents`` (i.e. other minor versions). Minor versions are walked through this function during the ``perun log`` command. :param str head_minor_version: the root minor versions which is the root of the walk. :returns: iterable stream of minor version representation """ vcs_type, vcs_path = pcs.get_vcs_type(), pcs.get_vcs_path() perun_log.msg_to_stdout( "Walking minor versions of type {}".format(vcs_type), 1) return dynamic_module_function_call('perun.vcs', vcs_type, '_walk_minor_versions', vcs_path, head_minor_version)
def init(vcs_type, *args, **kwargs): """Calls the implementation of initialization of wrapped underlying version control system. The initialization should take care of both reinitialization of existing version control system instances and newly created instances. Init is called during the ``perun init`` command from command line interface. :param str vcs_type: type of the underlying wrapped version control system :param list args: list of non-keyword arguments passed to implemenetation of the wrapped vcs layer :param dict kwargs: dictionary of keyword arguments passed to implementation of the wrapped vcs layer :return: true if the underlying vcs was successfully initialized """ perun_log.msg_to_stdout( "Initializing {} version control with params {} and {}".format( vcs_type, args, kwargs), 1) return dynamic_module_function_call('perun.vcs', vcs_type, '_init', *args, **kwargs)
def init_perun_at(perun_path, is_reinit, vcs_config): """Initialize the .perun directory at given path Initializes or reinitializes the .perun directory at the given path. Arguments: perun_path(path): path where new perun performance control system will be stored is_reinit(bool): true if this is existing perun, that will be reinitialized vcs_config(dict): dictionary of form {'vcs': {'type', 'url'}} for local config init """ # Initialize the basic structure of the .perun directory perun_full_path = os.path.join(perun_path, '.perun') store.touch_dir(perun_full_path) store.touch_dir(os.path.join(perun_full_path, 'objects')) store.touch_dir(os.path.join(perun_full_path, 'jobs')) store.touch_dir(os.path.join(perun_full_path, 'cache')) perun_config.init_local_config_at(perun_full_path, vcs_config) # Perun successfully created msg_prefix = "Reinitialized existing" if is_reinit else "Initialized empty" perun_log.msg_to_stdout( msg_prefix + " Perun repository in {}".format(perun_path), 0)
def get_minor_version_info(minor_version): """Yields the specification of concrete minor version in form of the ``MinorVersion`` named tuples containing the following information: ``date``, ``author``, ``email``, ``checksum`` (i.e. the hash representation of the minor version), ``commit_description`` and ``commit_parents`` (i.e. other minor versions). This function is a non-generator alternative of :func:`perun.vcs.walk_minor_versions` and is used during the ``perun status`` output to display the specifics of minor version. :param str minor_version: the specification of minor version (in form of sha e.g.) for which we are retrieving the details :returns: minor version named tuple """ vcs_type, vcs_path = pcs.get_vcs_type(), pcs.get_vcs_path() perun_log.msg_to_stdout( "Getting minor version info of type {} and args {}, {}".format( vcs_type, vcs_path, minor_version), 1) return dynamic_module_function_call('perun.vcs', vcs_type, '_get_minor_version_info', vcs_path, minor_version)
def walk_minor_versions(vcs_type, *args, **kwargs): """Generator of minor versions for the given major version, which yields the ``MinorVersion`` named tuples containing the following information: ``date``, ``author``, ``email``, ``checksum`` (i.e. the hash representation of the minor version), ``commit_description`` and ``commit_parents`` (i.e. other minor versions). Minor versions are walked through this function during the ``perun log`` command. :param str vcs_type: type of the underlying wrapped version control system :param list args: list of non-keyword arguments passed to implemenetation of the wrapped vcs layer :param dict kwargs: dictionary of keyword arguments passed to implementation of the wrapped vcs layer :returns: iterable stream of minor version representation """ perun_log.msg_to_stdout( "Walking minor versions of type {}".format(vcs_type), 1) return dynamic_module_function_call('perun.vcs', vcs_type, '_walk_minor_versions', *args, **kwargs)
def get_minor_version_info(vcs_type, *args, **kwargs): """Yields the specification of concrete minor version in form of the ``MinorVersion`` named tuples containing the following information: ``date``, ``author``, ``email``, ``checksum`` (i.e. the hash representation of the minor version), ``commit_description`` and ``commit_parents`` (i.e. other minor versions). This function is a non-generator alternative of :func:`perun.vcs.walk_minor_versions` and is used during the ``perun status`` output to display the specifics of minor version. :param str vcs_type: type of the underlying wrapped version control system :param list args: list of non-keyword arguments passed to implemenetation of the wrapped vcs layer :param dict kwargs: dictionary of keyword arguments passed to implementation of the wrapped vcs layer :returns: minor version named tuple """ perun_log.msg_to_stdout( "Getting minor version info of type {} and args {}, {}".format( vcs_type, args, kwargs), 1) return dynamic_module_function_call('perun.vcs', vcs_type, '_get_minor_version_info', *args, **kwargs)
def log(minor_version, short=False, **_): """Prints the log of the performance control system Either prints the short or longer version. In short version, only header and short list according to the formatting string from stored in the configuration. Prints the number of profiles associated with each of the minor version and some basic information about minor versions, like e.g. description, hash, etc. :param str minor_version: representation of the head version :param bool short: true if the log should be in short format """ perun_log.msg_to_stdout("Running inner wrapper of the 'perun log '", 2) # Print header for --short-minors if short: minor_versions = list(vcs.walk_minor_versions(minor_version)) # Reduce the descriptions of minor version to one liners for mv_no, minor_version in enumerate(minor_versions): minor_versions[mv_no] = minor_version._replace( desc=minor_version.desc.split("\n")[0]) minor_version_maxima = calculate_maximal_lengths_for_object_list( minor_versions, MinorVersion._fields) # Update manually the maxima for the printed supported profile types, each requires two # characters and 9 stands for " profiles" string def minor_stat_retriever(minor_v): """Helper function for picking stats of the given minor version :param MinorVersion minor_v: minor version for which we are retrieving the stats :return: dictionary with stats for minor version """ return store.get_profile_number_for_minor( pcs.get_object_directory(), minor_v.checksum) def deg_count_retriever(minor_v): """Helper function for picking stats of the degradation strings of form ++-- :param MinorVersion minor_v: minor version for which we are retrieving the stats :return: dictionary with stats for minor version """ counts = perun_log.count_degradations_per_group( store.load_degradation_list_for(pcs.get_object_directory(), minor_v.checksum)) return { 'changes': counts.get('Optimization', 0) * '+' + counts.get('Degradation', 0) * '-' } minor_version_maxima.update( calculate_maximal_lengths_for_stats(minor_versions, minor_stat_retriever)) minor_version_maxima.update( calculate_maximal_lengths_for_stats(minor_versions, deg_count_retriever, " changes ")) print_short_minor_version_info_list(minor_versions, minor_version_maxima) else: # Walk the minor versions and print them for minor in vcs.walk_minor_versions(minor_version): cprintln("Minor Version {}".format(minor.checksum), TEXT_EMPH_COLOUR, attrs=TEXT_ATTRS) base_dir = pcs.get_object_directory() tracked_profiles = store.get_profile_number_for_minor( base_dir, minor.checksum) print_profile_numbers(tracked_profiles, 'tracked') print_minor_version_info(minor, indent=1)