def _update_icon(self, problems, project_file, conda_meta_file): icon = project_file.get_value('icon', None) if icon is not None and not is_string(icon): problems.append( "%s: icon: field should have a string value not %r" % (project_file.filename, icon)) icon = None if icon is None: icon = conda_meta_file.icon if icon is not None and not is_string(icon): problems.append( "%s: app: icon: field should have a string value not %r" % (conda_meta_file.filename, icon)) icon = None if icon is not None: # relative to conda.recipe icon = os.path.join(META_DIRECTORY, icon) if icon is not None: icon = os.path.join(self.directory_path, icon) if not os.path.isfile(icon): problems.append("Icon file %s does not exist." % icon) icon = None self.icon = icon
def _update_name(self, problems, project_file, conda_meta_file): name = project_file.get_value('name', None) if name is not None: if not is_string(name): problems.append( "%s: name: field should have a string value not %r" % (project_file.filename, name)) name = None elif len(name.strip()) == 0: problems.append( "%s: name: field is an empty or all-whitespace string." % (project_file.filename)) name = None if name is None: name = conda_meta_file.name if name is not None and not is_string(name): problems.append( "%s: package: name: field should have a string value not %r" % (conda_meta_file.filename, name)) name = None if name is None: name = os.path.basename(self.directory_path) self.name = name
def _load_environment_yml(filename): """Load an environment.yml as an EnvSpec, or None if not loaded.""" try: with codecs.open(filename, 'r', 'utf-8') as file: contents = file.read() yaml = _load_string(contents) except (IOError, _YAMLError): return None name = None if 'name' in yaml: name = yaml['name'] if not name: if 'prefix' in yaml and yaml['prefix']: name = os.path.basename(yaml['prefix']) if not name: name = os.path.basename(filename) # We don't do too much validation here because we end up doing it # later if we import this into the project, and then load it from # the project file. We will do the import such that we don't end up # keeping the new project file if it's messed up. # # However we do try to avoid crashing on None or type errors here. raw_dependencies = yaml.get('dependencies', []) if not isinstance(raw_dependencies, list): raw_dependencies = [] raw_channels = yaml.get('channels', []) if not isinstance(raw_channels, list): raw_channels = [] conda_packages = [] pip_packages = [] for dep in raw_dependencies: if is_string(dep): conda_packages.append(dep) elif isinstance(dep, dict) and 'pip' in dep and isinstance(dep['pip'], list): for pip_dep in dep['pip']: if is_string(pip_dep): pip_packages.append(pip_dep) channels = [] for channel in raw_channels: if is_string(channel): channels.append(channel) return EnvSpec(name=name, conda_packages=conda_packages, channels=channels, pip_packages=pip_packages)
def _parse_default(self, options, env_var, problems): assert (isinstance(options, dict)) raw_default = options.get('default', None) if raw_default is None: good_default = True elif isinstance(raw_default, bool): # we have to check bool since it's considered an int apparently good_default = False elif is_string(raw_default) or isinstance(raw_default, (int, float)): good_default = True else: good_default = False if 'default' in options and raw_default is None: # convert null to be the same as simply missing del options['default'] if good_default: return True else: problems.append( "default value for variable {env_var} must be null, a string, or a number, not {value}." .format(env_var=env_var, value=raw_default)) return False
def _parse(cls, registry, varname, item, problems, requirements): """Parse an item from the services: section.""" service_type = None if is_string(item): service_type = item options = dict(type=service_type) elif isinstance(item, dict): service_type = item.get('type', None) if service_type is None: problems.append( "Service {} doesn't contain a 'type' field.".format( varname)) return options = deepcopy(item) else: problems.append( "Service {} should have a service type string or a dictionary as its value." .format(varname)) return if not EnvVarRequirement._parse_default(options, varname, problems): return requirement = registry.find_requirement_by_service_type( service_type=service_type, env_var=varname, options=options) if requirement is None: problems.append("Service {} has an unknown type '{}'.".format( varname, service_type)) else: assert isinstance(requirement, ServiceRequirement) assert 'type' in requirement.options requirements.append(requirement)
def _parse_default(self, options, env_var, problems): assert (isinstance(options, dict)) raw_default = options.get('default', None) if raw_default is None: good_default = True elif isinstance(raw_default, bool): # we have to check bool since it's considered an int apparently good_default = False elif is_string(raw_default) or isinstance(raw_default, (int, float)): good_default = True else: good_default = False if 'default' in options and raw_default is None: # convert null to be the same as simply missing del options['default'] if good_default: return True else: problems.append( "default value for variable {env_var} must be null, a string, or a number, not {value}.".format( env_var=env_var, value=raw_default)) return False
def _parse(cls, registry, varname, item, problems, requirements): """Parse an item from the services: section.""" service_type = None if is_string(item): service_type = item options = dict(type=service_type) elif isinstance(item, dict): service_type = item.get('type', None) if service_type is None: problems.append("Service {} doesn't contain a 'type' field.".format(varname)) return options = deepcopy(item) else: problems.append("Service {} should have a service type string or a dictionary as its value.".format( varname)) return if not EnvVarRequirement._parse_default(options, varname, problems): return requirement = registry.find_requirement_by_service_type(service_type=service_type, env_var=varname, options=options) if requirement is None: problems.append("Service {} has an unknown type '{}'.".format(varname, service_type)) else: assert isinstance(requirement, ServiceRequirement) assert 'type' in requirement.options requirements.append(requirement)
def _path(cls, path): if is_string(path): return (path, ) else: try: return list(element for element in path) except TypeError: raise ValueError("YAML file path must be a string or an iterable of strings")
def status_for(self, env_var_or_class): """Get status for the given env var or class, or None if unknown.""" for status in self.statuses: if is_string(env_var_or_class): if isinstance(status.requirement, EnvVarRequirement) and \ status.requirement.env_var == env_var_or_class: return status elif isinstance(status.requirement, env_var_or_class): return status return None
def status_for(self, env_var_or_class): """Get status for the given env var or class, or None if unknown.""" for status in self.statuses: if is_string(env_var_or_class): if isinstance(status.requirement, EnvVarRequirement) and \ status.requirement.env_var == env_var_or_class: return status elif isinstance(status.requirement, env_var_or_class): return status return None
def _path(cls, path): if is_string(path): return (path, ) else: try: return list(element for element in path) except TypeError: raise ValueError( "YAML file path must be a string or an iterable of strings" )
def _update_description(self, problems, project_file): desc = project_file.get_value('description', None) if desc is not None and not is_string(desc): problems.append("%s: description: field should have a string value not %r" % (project_file.filename, desc)) desc = None if desc is None: desc = '' self.description = desc
def _update_variables(self, requirements, problems, project_file): variables = project_file.get_value("variables") def check_conda_reserved(key): if key in ('CONDA_DEFAULT_ENV', 'CONDA_ENV_PATH', 'CONDA_PREFIX'): problems.append(("Environment variable %s is reserved for Conda's use, " + "so it can't appear in the variables section.") % key) return True else: return False # variables: section can contain a list of var names or a dict from # var names to options OR default values. it can also be missing # entirely which is the same as empty. if variables is None: pass elif isinstance(variables, dict): for key in variables.keys(): if check_conda_reserved(key): continue if key.strip() == '': problems.append("Variable name cannot be empty string, found: '{}' as name".format(key)) continue raw_options = variables[key] if raw_options is None: options = {} elif isinstance(raw_options, dict): options = deepcopy(raw_options) # so we can modify it below else: options = dict(default=raw_options) assert (isinstance(options, dict)) if EnvVarRequirement._parse_default(options, key, problems): requirement = self.registry.find_requirement_by_env_var(key, options) requirements.append(requirement) elif isinstance(variables, list): for item in variables: if is_string(item): if item.strip() == '': problems.append("Variable name cannot be empty string, found: '{}' as name".format(item)) continue if check_conda_reserved(item): continue requirement = self.registry.find_requirement_by_env_var(item, options=dict()) requirements.append(requirement) else: problems.append( "variables section should contain environment variable names, {item} is not a string".format( item=item)) else: problems.append( "variables section contains wrong value type {value}, should be dict or list of requirements".format( value=variables))
def _update_description(self, problems, project_file): desc = project_file.get_value('description', None) if desc is not None and not is_string(desc): problems.append( "%s: description: field should have a string value not %r" % (project_file.filename, desc)) desc = None if desc is None: desc = '' self.description = desc
def _update_name(self, problems, project_file, conda_meta_file): name = project_file.get_value('name', None) if name is not None: if not is_string(name): problems.append("%s: name: field should have a string value not %r" % (project_file.filename, name)) name = None elif len(name.strip()) == 0: problems.append("%s: name: field is an empty or all-whitespace string." % (project_file.filename)) name = None if name is None: name = conda_meta_file.name if name is not None and not is_string(name): problems.append("%s: package: name: field should have a string value not %r" % (conda_meta_file.filename, name)) name = None if name is None: name = os.path.basename(self.directory_path) self.name = name
def _in_provide_whitelist(provide_whitelist, requirement): if provide_whitelist is None: # whitelist of None means "everything" return True for env_var_or_class in provide_whitelist: if is_string(env_var_or_class): if isinstance(requirement, EnvVarRequirement) and requirement.env_var == env_var_or_class: return True else: if isinstance(requirement, env_var_or_class): return True return False
def _in_provide_whitelist(provide_whitelist, requirement): if provide_whitelist is None: # whitelist of None means "everything" return True for env_var_or_class in provide_whitelist: if is_string(env_var_or_class): if isinstance(requirement, EnvVarRequirement ) and requirement.env_var == env_var_or_class: return True else: if isinstance(requirement, env_var_or_class): return True return False
def _update_icon(self, problems, project_file, conda_meta_file): icon = project_file.get_value('icon', None) if icon is not None and not is_string(icon): problems.append("%s: icon: field should have a string value not %r" % (project_file.filename, icon)) icon = None if icon is None: icon = conda_meta_file.icon if icon is not None and not is_string(icon): problems.append("%s: app: icon: field should have a string value not %r" % (conda_meta_file.filename, icon)) icon = None if icon is not None: # relative to conda.recipe icon = os.path.join(META_DIRECTORY, icon) if icon is not None: icon = os.path.join(self.directory_path, icon) if not os.path.isfile(icon): problems.append("Icon file %s does not exist." % icon) icon = None self.icon = icon
def _parse_string_list_with_special(parent_dict, key, what, special_filter): items = parent_dict.get(key, []) if not isinstance(items, (list, tuple)): problems.append("%s: %s: value should be a list of %ss, not '%r'" % (project_file.filename, key, what, items)) return ([], []) cleaned = [] special = [] for item in items: if is_string(item): cleaned.append(item.strip()) elif special_filter(item): special.append(item) else: problems.append("%s: %s: value should be a %s (as a string) not '%r'" % (project_file.filename, key, what, item)) return (cleaned, special)
def __init__(self, registry, options): """Construct a Requirement. Args: registry (PluginRegistry): the plugin registry we came from options (dict): dict of requirement options from the project config """ self.registry = registry if options is None: self.options = dict() else: self.options = deepcopy(options) # always convert the default to a string (it's allowed to be a number # in the config file, but env vars have to be strings), unless # it's a dict because we use a dict for encrypted defaults if 'default' in self.options and not (is_string(self.options['default']) or isinstance( self.options['default'], dict)): self.options['default'] = str(self.options['default'])
def __init__(self, registry, options): """Construct a Requirement. Args: registry (PluginRegistry): the plugin registry we came from options (dict): dict of requirement options from the project config """ self.registry = registry if options is None: self.options = dict() else: self.options = deepcopy(options) # always convert the default to a string (it's allowed to be a number # in the config file, but env vars have to be strings), unless # it's a dict because we use a dict for encrypted defaults if 'default' in self.options and not ( is_string(self.options['default']) or isinstance(self.options['default'], dict)): self.options['default'] = str(self.options['default'])
def _parse_string_list_with_special(parent_dict, key, what, special_filter): items = parent_dict.get(key, []) if not isinstance(items, (list, tuple)): problems.append( "%s: %s: value should be a list of %ss, not '%r'" % (project_file.filename, key, what, items)) return ([], []) cleaned = [] special = [] for item in items: if is_string(item): cleaned.append(item.strip()) elif special_filter(item): special.append(item) else: problems.append( "%s: %s: value should be a %s (as a string) not '%r'" % (project_file.filename, key, what, item)) return (cleaned, special)
def _update_env_specs(self, problems, project_file): def _parse_string_list_with_special(parent_dict, key, what, special_filter): items = parent_dict.get(key, []) if not isinstance(items, (list, tuple)): problems.append( "%s: %s: value should be a list of %ss, not '%r'" % (project_file.filename, key, what, items)) return ([], []) cleaned = [] special = [] for item in items: if is_string(item): cleaned.append(item.strip()) elif special_filter(item): special.append(item) else: problems.append( "%s: %s: value should be a %s (as a string) not '%r'" % (project_file.filename, key, what, item)) return (cleaned, special) def _parse_string_list(parent_dict, key, what): return _parse_string_list_with_special( parent_dict, key, what, special_filter=lambda x: False)[0] def _parse_channels(parent_dict): return _parse_string_list(parent_dict, 'channels', 'channel name') def _parse_packages(parent_dict): (deps, pip_dicts) = _parse_string_list_with_special( parent_dict, 'packages', 'package name', lambda x: isinstance(x, dict) and ('pip' in x)) for dep in deps: parsed = conda_api.parse_spec(dep) if parsed is None: problems.append("%s: invalid package specification: %s" % (project_file.filename, dep)) # note that multiple "pip:" dicts are allowed pip_deps = [] for pip_dict in pip_dicts: pip_list = _parse_string_list(pip_dict, 'pip', 'pip package name') pip_deps.extend(pip_list) for dep in pip_deps: parsed = pip_api.parse_spec(dep) if parsed is None: problems.append("%s: invalid pip package specifier: %s" % (project_file.filename, dep)) return (deps, pip_deps) self.env_specs = dict() (shared_deps, shared_pip_deps) = _parse_packages(project_file.root) shared_channels = _parse_channels(project_file.root) env_specs = project_file.get_value('env_specs', default={}) first_env_spec_name = None env_specs_is_empty_or_missing = False # this should be iff it's an empty dict or absent entirely if isinstance(env_specs, dict): if len(env_specs) == 0: env_specs_is_empty_or_missing = True for (name, attrs) in env_specs.items(): if name.strip() == '': problems.append( "Environment spec name cannot be empty string, found: '{}' as name" .format(name)) continue description = attrs.get('description', None) if description is not None and not is_string(description): problems.append( "{}: 'description' field of environment {} must be a string" .format(project_file.filename, name)) continue (deps, pip_deps) = _parse_packages(attrs) channels = _parse_channels(attrs) # ideally we would merge same-name packages here, choosing the # highest of the two versions or something. maybe conda will # do that for us anyway? all_deps = shared_deps + deps all_pip_deps = shared_pip_deps + pip_deps all_channels = shared_channels + channels self.env_specs[name] = EnvSpec(name=name, conda_packages=all_deps, pip_packages=all_pip_deps, channels=all_channels, description=description) if first_env_spec_name is None: first_env_spec_name = name else: problems.append( "%s: env_specs should be a dictionary from environment name to environment attributes, not %r" % (project_file.filename, env_specs)) if env_specs_is_empty_or_missing: # we do NOT want to add this problem if we merely # failed to parse individual env specs; it must be # safe to overwrite the env_specs key, so it has to # be empty or missing entirely. def add_default_env_spec(project): project.project_file.set_value(['env_specs', 'default'], dict(packages=[], channels=[])) project.project_file.save() problems.append( ProjectProblem( text=("%s has an empty env_specs section." % project_file.filename), fix_prompt=("Add an environment spec to %s?" % os.path.basename(project_file.filename)), fix_function=add_default_env_spec)) # this is only used for commands that don't specify anything # (when/if we require all commands to specify, then remove this.) if 'default' in self.env_specs: self.default_env_spec_name = 'default' else: self.default_env_spec_name = first_env_spec_name
def _update_commands(self, problems, project_file, conda_meta_file, requirements): failed = False app_entry_from_meta_yaml = conda_meta_file.app_entry if app_entry_from_meta_yaml is not None: if not is_string(app_entry_from_meta_yaml): problems.append( "%s: app: entry: should be a string not '%r'" % (conda_meta_file.filename, app_entry_from_meta_yaml)) app_entry_from_meta_yaml = None failed = True first_command_name = None commands = dict() commands_section = project_file.get_value('commands', None) if commands_section is not None and not isinstance( commands_section, dict): problems.append( "%s: 'commands:' section should be a dictionary from command names to attributes, not %r" % (project_file.filename, commands_section)) failed = True elif commands_section is not None: for (name, attrs) in commands_section.items(): if name.strip() == '': problems.append( "Command variable name cannot be empty string, found: '{}' as name" .format(name)) failed = True continue if first_command_name is None: first_command_name = name if not isinstance(attrs, dict): problems.append( "%s: command name '%s' should be followed by a dictionary of attributes not %r" % (project_file.filename, name, attrs)) failed = True continue if 'description' in attrs and not is_string( attrs['description']): problems.append( "{}: 'description' field of command {} must be a string" .format(project_file.filename, name)) failed = True if 'env_spec' in attrs: if not is_string(attrs['env_spec']): problems.append( "{}: 'env_spec' field of command {} must be a string (an environment spec name)" .format(project_file.filename, name)) failed = True elif attrs['env_spec'] not in self.env_specs: problems.append( "%s: env_spec '%s' for command '%s' does not appear in the env_specs section" % (project_file.filename, attrs['env_spec'], name)) failed = True copied_attrs = deepcopy(attrs) if 'env_spec' not in copied_attrs: copied_attrs['env_spec'] = self.default_env_spec_name command_types = [] for attr in ALL_COMMAND_TYPES: if attr not in copied_attrs: continue # be sure we add this even if the command is broken, since it's # confusing to say "does not have a command line in it" below # if the issue is that the command line is broken. command_types.append(attr) if not is_string(copied_attrs[attr]): problems.append( "%s: command '%s' attribute '%s' should be a string not '%r'" % (project_file.filename, name, attr, copied_attrs[attr])) failed = True if len(command_types) == 0: problems.append( "%s: command '%s' does not have a command line in it" % (project_file.filename, name)) failed = True if ('notebook' in copied_attrs or 'bokeh_app' in copied_attrs) and (len(command_types) > 1): label = 'bokeh_app' if 'bokeh_app' in copied_attrs else 'notebook' others = copy(command_types) others.remove(label) others = [("'%s'" % other) for other in others] problems.append( "%s: command '%s' has multiple commands in it, '%s' can't go with %s" % (project_file.filename, name, label, ", ".join(others))) failed = True # note that once one command fails, we don't add any more if not failed: commands[name] = ProjectCommand(name=name, attributes=copied_attrs) self._add_notebook_commands(commands, problems, requirements) if failed: self.commands = dict() self.default_command_name = None else: # if no commands and we have a meta.yaml app entry, use the meta.yaml if app_entry_from_meta_yaml is not None and len(commands) == 0: commands['default'] = ProjectCommand( name='default', attributes=dict(conda_app_entry=app_entry_from_meta_yaml, auto_generated=True, env_spec=self.default_env_spec_name)) self.commands = commands if first_command_name is None and len(commands) > 0: # this happens if we created a command automatically # from a notebook file or conda meta.yaml first_command_name = sorted(commands.keys())[0] if 'default' in self.commands: self.default_command_name = 'default' else: # 'default' is always mapped to the first-listed if none is named 'default' # note: this may be None self.default_command_name = first_command_name
def _parse(cls, registry, varname, item, problems, requirements): """Parse an item from the downloads: section.""" url = None filename = None hash_algorithm = None hash_value = None unzip = None description = None if is_string(item): url = item elif isinstance(item, dict): url = item.get('url', None) if url is None: problems.append("Download item {} doesn't contain a 'url' field.".format(varname)) return description = item.get('description', None) if description is not None and not is_string(description): problems.append("'description' field for download item {} is not a string".format(varname)) return for method in _hash_algorithms: if method not in item: continue if hash_algorithm is not None: problems.append("Multiple checksums for download {}: {} and {}.".format(varname, hash_algorithm, method)) return else: hash_value = item[method] if is_string(hash_value): hash_algorithm = method else: problems.append("Checksum value for {} should be a string not {}.".format(varname, hash_value)) return filename = item.get('filename', None) unzip = item.get('unzip', None) if unzip is not None and not isinstance(unzip, bool): problems.append("Value of 'unzip' for download item {} should be a boolean, not {}.".format(varname, unzip)) return if url is None or not is_string(url): problems.append(("Download name {} should be followed by a URL string or a dictionary " + "describing the download.").format(varname)) return if url == '': problems.append("Download item {} has an empty 'url' field.".format(varname)) return # urlsplit doesn't seem to ever throw an exception, but it can # return pretty nonsensical stuff on invalid urls, in particular # an empty path is very possible url_path = os.path.basename(urlparse.urlsplit(url).path) url_path_is_zip = url_path.lower().endswith(".zip") if filename is None: if url_path != '': filename = url_path if url_path_is_zip: if unzip is None: # url is a zip and neither filename nor unzip specified, assume unzip unzip = True if unzip: # unzip specified True, or we guessed True, and url ends in zip; # take the .zip off the filename we invented based on the url. filename = filename[:-4] elif url_path_is_zip and unzip is None and not filename.lower().endswith(".zip"): # URL is a zip, filename is not a zip, unzip was not specified, so assume # we want to unzip unzip = True if filename is None: filename = varname if unzip is None: unzip = False requirements.append(DownloadRequirement(registry, env_var=varname, url=url, filename=filename, hash_algorithm=hash_algorithm, hash_value=hash_value, unzip=unzip, description=description))
def _update_variables(self, requirements, problems, project_file): variables = project_file.get_value("variables") def check_conda_reserved(key): if key in ('CONDA_DEFAULT_ENV', 'CONDA_ENV_PATH', 'CONDA_PREFIX'): problems.append( ("Environment variable %s is reserved for Conda's use, " + "so it can't appear in the variables section.") % key) return True else: return False # variables: section can contain a list of var names or a dict from # var names to options OR default values. it can also be missing # entirely which is the same as empty. if variables is None: pass elif isinstance(variables, dict): for key in variables.keys(): if check_conda_reserved(key): continue if key.strip() == '': problems.append( "Variable name cannot be empty string, found: '{}' as name" .format(key)) continue raw_options = variables[key] if raw_options is None: options = {} elif isinstance(raw_options, dict): options = deepcopy( raw_options) # so we can modify it below else: options = dict(default=raw_options) assert (isinstance(options, dict)) if EnvVarRequirement._parse_default(options, key, problems): requirement = self.registry.find_requirement_by_env_var( key, options) requirements.append(requirement) elif isinstance(variables, list): for item in variables: if is_string(item): if item.strip() == '': problems.append( "Variable name cannot be empty string, found: '{}' as name" .format(item)) continue if check_conda_reserved(item): continue requirement = self.registry.find_requirement_by_env_var( item, options=dict()) requirements.append(requirement) else: problems.append( "variables section should contain environment variable names, {item} is not a string" .format(item=item)) else: problems.append( "variables section contains wrong value type {value}, should be dict or list of requirements" .format(value=variables))
def _update_commands(self, problems, project_file, conda_meta_file, requirements): failed = False first_command_name = None commands = dict() commands_section = project_file.get_value('commands', None) if commands_section is not None and not isinstance(commands_section, dict): problems.append("%s: 'commands:' section should be a dictionary from command names to attributes, not %r" % (project_file.filename, commands_section)) failed = True elif commands_section is not None: for (name, attrs) in commands_section.items(): if name.strip() == '': problems.append("Command variable name cannot be empty string, found: '{}' as name".format(name)) failed = True continue if first_command_name is None: first_command_name = name if not isinstance(attrs, dict): problems.append("%s: command name '%s' should be followed by a dictionary of attributes not %r" % (project_file.filename, name, attrs)) failed = True continue if 'description' in attrs and not is_string(attrs['description']): problems.append("{}: 'description' field of command {} must be a string".format( project_file.filename, name)) failed = True if 'supports_http_options' in attrs and not isinstance(attrs['supports_http_options'], bool): problems.append("{}: 'supports_http_options' field of command {} must be a boolean".format( project_file.filename, name)) failed = True if 'env_spec' in attrs: if not is_string(attrs['env_spec']): problems.append( "{}: 'env_spec' field of command {} must be a string (an environment spec name)".format( project_file.filename, name)) failed = True elif attrs['env_spec'] not in self.env_specs: problems.append("%s: env_spec '%s' for command '%s' does not appear in the env_specs section" % (project_file.filename, attrs['env_spec'], name)) failed = True copied_attrs = deepcopy(attrs) if 'env_spec' not in copied_attrs: copied_attrs['env_spec'] = self.default_env_spec_name command_types = [] for attr in ALL_COMMAND_TYPES: if attr not in copied_attrs: continue # be sure we add this even if the command is broken, since it's # confusing to say "does not have a command line in it" below # if the issue is that the command line is broken. command_types.append(attr) if not is_string(copied_attrs[attr]): problems.append("%s: command '%s' attribute '%s' should be a string not '%r'" % (project_file.filename, name, attr, copied_attrs[attr])) failed = True if len(command_types) == 0: problems.append("%s: command '%s' does not have a command line in it" % (project_file.filename, name)) failed = True if ('notebook' in copied_attrs or 'bokeh_app' in copied_attrs) and (len(command_types) > 1): label = 'bokeh_app' if 'bokeh_app' in copied_attrs else 'notebook' others = copy(command_types) others.remove(label) others = [("'%s'" % other) for other in others] problems.append("%s: command '%s' has multiple commands in it, '%s' can't go with %s" % (project_file.filename, name, label, ", ".join(others))) failed = True # note that once one command fails, we don't add any more if not failed: commands[name] = ProjectCommand(name=name, attributes=copied_attrs) self._verify_notebook_commands(commands, problems, requirements, project_file) if failed: self.commands = dict() self.default_command_name = None else: self.commands = commands if 'default' in self.commands: self.default_command_name = 'default' else: # 'default' is always mapped to the first-listed if none is named 'default' # note: this may be None self.default_command_name = first_command_name
def _update_env_specs(self, problems, project_file): def _parse_string_list_with_special(parent_dict, key, what, special_filter): items = parent_dict.get(key, []) if not isinstance(items, (list, tuple)): problems.append("%s: %s: value should be a list of %ss, not '%r'" % (project_file.filename, key, what, items)) return ([], []) cleaned = [] special = [] for item in items: if is_string(item): cleaned.append(item.strip()) elif special_filter(item): special.append(item) else: problems.append("%s: %s: value should be a %s (as a string) not '%r'" % (project_file.filename, key, what, item)) return (cleaned, special) def _parse_string_list(parent_dict, key, what): return _parse_string_list_with_special(parent_dict, key, what, special_filter=lambda x: False)[0] def _parse_channels(parent_dict): return _parse_string_list(parent_dict, 'channels', 'channel name') def _parse_packages(parent_dict): (deps, pip_dicts) = _parse_string_list_with_special(parent_dict, 'packages', 'package name', lambda x: isinstance(x, dict) and ('pip' in x)) for dep in deps: parsed = conda_api.parse_spec(dep) if parsed is None: problems.append("%s: invalid package specification: %s" % (project_file.filename, dep)) # note that multiple "pip:" dicts are allowed pip_deps = [] for pip_dict in pip_dicts: pip_list = _parse_string_list(pip_dict, 'pip', 'pip package name') pip_deps.extend(pip_list) for dep in pip_deps: parsed = pip_api.parse_spec(dep) if parsed is None: problems.append("%s: invalid pip package specifier: %s" % (project_file.filename, dep)) return (deps, pip_deps) (shared_deps, shared_pip_deps) = _parse_packages(project_file.root) shared_channels = _parse_channels(project_file.root) env_specs = project_file.get_value('env_specs', default={}) first_env_spec_name = None env_specs_is_empty_or_missing = False # this should be iff it's an empty dict or absent entirely # this one isn't in the env_specs dict self.global_base_env_spec = EnvSpec(name=None, conda_packages=shared_deps, pip_packages=shared_pip_deps, channels=shared_channels, description="Global packages and channels", inherit_from_names=(), inherit_from=()) env_spec_attrs = dict() if isinstance(env_specs, dict): if len(env_specs) == 0: env_specs_is_empty_or_missing = True for (name, attrs) in env_specs.items(): if name.strip() == '': problems.append("Environment spec name cannot be empty string, found: '{}' as name".format(name)) continue description = attrs.get('description', None) if description is not None and not is_string(description): problems.append("{}: 'description' field of environment {} must be a string".format( project_file.filename, name)) continue problem_count = len(problems) inherit_from_names = attrs.get('inherit_from', None) if inherit_from_names is None: inherit_from_names = [] elif is_string(inherit_from_names): inherit_from_names = [inherit_from_names.strip()] else: inherit_from_names = _parse_string_list(attrs, 'inherit_from', 'env spec name') if len(problems) > problem_count: # we got a new problem from the bad inherit_from continue (deps, pip_deps) = _parse_packages(attrs) channels = _parse_channels(attrs) env_spec_attrs[name] = dict(name=name, conda_packages=deps, pip_packages=pip_deps, channels=channels, description=description, inherit_from_names=tuple(inherit_from_names), inherit_from=()) if first_env_spec_name is None: first_env_spec_name = name else: problems.append( "%s: env_specs should be a dictionary from environment name to environment attributes, not %r" % (project_file.filename, env_specs)) self.env_specs = dict() def make_env_spec(name, trail): assert name in env_spec_attrs if name not in self.env_specs: was_cycle = False if name in trail: problems.append( "{}: 'inherit_from' fields create circular inheritance among these env specs: {}".format( project_file.filename, ", ".join(sorted(trail)))) was_cycle = True trail.append(name) attrs = env_spec_attrs[name] if not was_cycle: inherit_from_names = attrs['inherit_from_names'] for parent in inherit_from_names: if parent not in env_spec_attrs: problems.append(("{}: name '{}' in 'inherit_from' field of env spec {} does not match " + "the name of another env spec").format(project_file.filename, parent, attrs['name'])) else: inherit_from = make_env_spec(parent, trail) attrs['inherit_from'] = attrs['inherit_from'] + (inherit_from, ) # All parent-less env specs get the global base spec as parent, # which means the global base spec is in everyone's ancestry if attrs['inherit_from'] == (): attrs['inherit_from'] = (self.global_base_env_spec, ) self.env_specs[name] = EnvSpec(**attrs) return self.env_specs[name] for name in env_spec_attrs.keys(): make_env_spec(name, []) assert name in self.env_specs # it's important to create all the env specs when possible # even if they are broken (e.g. bad inherit_from), so they # can be edited in order to fix them (importable_spec, importable_filename) = _find_out_of_sync_importable_spec(self.env_specs.values(), self.directory_path) if importable_spec is not None: skip_spec_import = project_file.get_value(['skip_imports', 'environment']) if skip_spec_import == importable_spec.channels_and_packages_hash: importable_spec = None if importable_spec is not None: old = self.env_specs.get(importable_spec.name) # this is a pretty bad hack, but if we injected "notebook" # or "bokeh" deps to make a notebook/bokeh command work, # we will end up out-of-sync for that reason # alone. environment.yml seems to typically not have # "notebook" in it because the environment.yml is used for # the kernel but not Jupyter itself. # We then end up in a problem loop where we complain about # missing notebook dep, add it, then complain about environment.yml # out of sync, and `conda-kapsel init` in a directory with a .ipynb # and an environment.yml doesn't result in a valid project. if importable_spec is not None and old is not None and \ importable_spec.diff_only_removes_notebook_or_bokeh(old): importable_spec = None if importable_spec is not None: if old is None: text = "Environment spec '%s' from %s is not in %s." % (importable_spec.name, importable_filename, os.path.basename(project_file.filename)) prompt = "Add env spec %s to %s?" % (importable_spec.name, os.path.basename(project_file.filename)) else: text = "Environment spec '%s' from %s is out of sync with %s. Diff:\n%s" % ( importable_spec.name, importable_filename, os.path.basename(project_file.filename), importable_spec.diff_from(old)) prompt = "Overwrite env spec %s with the changes from %s?" % (importable_spec.name, importable_filename) def overwrite_env_spec_from_importable(project): project.project_file.set_value(['env_specs', importable_spec.name], importable_spec.to_json()) def remember_no_import_importable(project): project.project_file.set_value(['skip_imports', 'environment'], importable_spec.channels_and_packages_hash) problems.append(ProjectProblem(text=text, fix_prompt=prompt, fix_function=overwrite_env_spec_from_importable, no_fix_function=remember_no_import_importable)) elif env_specs_is_empty_or_missing: # we do NOT want to add this problem if we merely # failed to parse individual env specs; it must be # safe to overwrite the env_specs key, so it has to # be empty or missing entirely. Also, we do NOT want # to add this if we are going to ask about environment.yml # import, above. def add_default_env_spec(project): default_spec = _anaconda_default_env_spec(self.global_base_env_spec) project.project_file.set_value(['env_specs', default_spec.name], default_spec.to_json()) problems.append(ProjectProblem(text=("%s has an empty env_specs section." % project_file.filename), fix_prompt=("Add an environment spec to %s?" % os.path.basename( project_file.filename)), fix_function=add_default_env_spec)) # this is only used for commands that don't specify anything # (when/if we require all commands to specify, then remove this.) if 'default' in self.env_specs: self.default_env_spec_name = 'default' else: self.default_env_spec_name = first_env_spec_name