def html_checkbox(caravel=False, checked=False): """ Create argument for type parameter on argparse.ArgumentParser.add_argument. :param bool caravel: whether this is being used in the caravel context :param bool checked: whether to add a particular key-value entry to a collection used by caravel :return callable: argument to the type parameter of an argparse.ArgumentParser's add_argument method. """ caravel_data = \ PathExAttMap({"element_type": "checkbox", "element_args": {}}) if checked: caravel_data.add_entries({"element_args": {"checked": True}}) def fun(x=None, caravel_data=caravel_data, caravel=caravel): return caravel_data if caravel else eval(x) return fun
class ComputingConfiguration(PathExAttMap): """ Represents computing configuration objects. The ComputingConfiguration class provides a computing configuration object that is an *in memory* representation of a `divvy` computing configuration file. This object has various functions to allow a user to activate, modify, and retrieve computing configuration files, and use these values to populate job submission script templates. :param str config_file: YAML file specifying computing package data (The `DIVCFG` file). :param type no_env_error: type of exception to raise if divvy settings can't be established, optional; if null (the default), a warning message will be logged, and no exception will be raised. :param type no_compute_exception: type of exception to raise if compute settings can't be established, optional; if null (the default), a warning message will be logged, and no exception will be raised. """ def __init__(self, config_file=None, no_env_error=None, no_compute_exception=None): super(ComputingConfiguration, self).__init__() self.compute_packages = None if config_file: if os.path.isfile(config_file): self.config_file = config_file else: _LOGGER.error( "Config file path isn't a file: {}".format(config_file)) raise IOError(config_file) else: _LOGGER.debug("No local config file was provided") _LOGGER.debug( "Checking this set of environment variables: {}".format( self.compute_env_var)) divcfg_env_var, divcfg_file = get_first_env_var( self.compute_env_var) or ["", ""] if os.path.isfile(divcfg_file): _LOGGER.debug("Found global config file in {}: {}".format( divcfg_env_var, divcfg_file)) self.config_file = divcfg_file else: _LOGGER.info( "Using default config file, no global config file provided in environment " "variable(s): {}".format(str(self.compute_env_var))) self.config_file = self.default_config_file try: self.update_packages(self.config_file) except Exception as e: _LOGGER.error("Can't load config file '%s'", str(self.config_file)) _LOGGER.error(str(type(e).__name__) + str(e)) self._handle_missing_env_attrs(self.config_file, when_missing=no_env_error) # Initialize default compute settings. _LOGGER.debug("Establishing project compute settings") self.compute = None self.activate_package(DEFAULT_COMPUTE_RESOURCES_NAME) # Either warn or raise exception if the compute is null. if self.compute is None: message = "Failed to establish compute settings." if no_compute_exception: no_compute_exception(message) else: _LOGGER.warning(message) else: _LOGGER.debug("Compute: %s", str(self.compute)) @property def compute_env_var(self): """ Environment variable through which to access compute settings. :return list[str]: names of candidate environment variables, for which value may be path to compute settings file; first found is used. """ return COMPUTE_SETTINGS_VARNAME @property def default_config_file(self): """ Path to default compute environment settings file. :return str: Path to default compute settings file """ return os.path.join(self.templates_folder, "default_compute_settings.yaml") @property def template(self): """ Get the currently active submission template. :return str: submission script content template for current state """ with open(self.compute.submission_template, 'r') as f: return f.read() @property def templates_folder(self): """ Path to folder with default submission templates. :return str: path to folder with default submission templates """ return os.path.join(os.path.dirname(__file__), "submit_templates") def activate_package(self, package_name): """ Activates a compute package. This copies the computing attributes from the configuration file into the `compute` attribute, where the class stores current compute settings. :param str package_name: name for non-resource compute bundle, the name of a subsection in an environment configuration file :return bool: success flag for attempt to establish compute settings """ # Hope that environment & environment compute are present. _LOGGER.info("Activating compute package '{}'".format(package_name)) if package_name and self.compute_packages and package_name in self.compute_packages: # Augment compute, creating it if needed. if self.compute is None: _LOGGER.debug("Creating Project compute") self.compute = PathExAttMap() _LOGGER.debug("Adding entries for package_name '%s'", package_name) self.compute.add_entries(self.compute_packages[package_name]) # Ensure submission template is absolute. # This is handled at update. # if not os.path.isabs(self.compute.submission_template): # try: # self.compute.submission_template = os.path.join( # os.path.dirname(self.config_file), # self.compute.submission_template) # except AttributeError as e: # # Environment and environment compute should at least have been # # set as null-valued attributes, so execution here is an error. # _LOGGER.error(str(e)) return True else: # Scenario in which environment and environment compute are # both present--but don't evaluate to True--is fairly harmless. _LOGGER.debug("Environment = {}".format(self.compute_packages)) return False def clean_start(self, package_name): """ Clear current active settings and then activate the given package. :param str package_name: name of the resource package to activate :return bool: success flag """ self.reset_active_settings() return self.activate_package(package_name) def get_active_package(self): """ Returns settings for the currently active compute package :return PathExAttMap: data defining the active compute package """ return self.compute def list_compute_packages(self): """ Returns a list of available compute packages. :return set[str]: names of available compute packages """ return set(self.compute_packages.keys()) def reset_active_settings(self): """ Clear out current compute settings. :return bool: success flag """ self.compute = PathExAttMap() return True def update_packages(self, config_file): """ Parse data from divvy configuration file. Given a divvy configuration file, this function will update (not overwrite) existing compute packages with existing values. It does not affect any currently active settings. :param str config_file: path to file with new divvy configuration data """ env_settings = parse_config_file(config_file) loaded_packages = env_settings[NEW_COMPUTE_KEY] for key, value in loaded_packages.items(): if type(loaded_packages[key]) is dict: for key2, value2 in loaded_packages[key].items(): if key2 == "submission_template": if not os.path.isabs(loaded_packages[key][key2]): loaded_packages[key][key2] = os.path.join( os.path.dirname(config_file), loaded_packages[key][key2]) if self.compute_packages is None: self.compute_packages = PathExAttMap(loaded_packages) else: self.compute_packages.add_entries(loaded_packages) _LOGGER.debug("Available divvy packages: {}".format(', '.join( self.list_compute_packages()))) self.config_file = config_file def write_script(self, output_path, extra_vars=None): """ Given currently active settings, populate the active template to write a submission script. :param str output_path: Path to file to write as submission script :param Iterable[Mapping] extra_vars: A list of Dict objects with key-value pairs with which to populate template fields. These will override any values in the currently active compute package. :return str: Path to the submission script file """ from copy import deepcopy variables = deepcopy(self.compute) if extra_vars: if not isinstance(extra_vars, list): extra_vars = [extra_vars] for kvs in reversed(extra_vars): variables.update(kvs) _LOGGER.info("Writing script to {}".format( os.path.abspath(output_path))) return write_submit_script(output_path, self.template, variables) def _handle_missing_env_attrs(self, config_file, when_missing): """ Default environment settings aren't required; warn, though. """ missing_env_attrs = \ [attr for attr in [NEW_COMPUTE_KEY, "config_file"] if getattr(self, attr, None) is None] if not missing_env_attrs: return message = "'{}' lacks environment attributes: {}". \ format(config_file, missing_env_attrs) if when_missing is None: _LOGGER.warning(message) else: when_missing(message)