def _update_icon(self, problems, project_file, conda_meta_file):
icon = project_file.get_value('icon', None)
if icon is not None and not is_string(icon):
problems.append(
"%s: icon: field should have a string value not %r" %
(project_file.filename, icon))
icon = None
if icon is None:
icon = conda_meta_file.icon
if icon is not None and not is_string(icon):
problems.append(
"%s: app: icon: field should have a string value not %r" %
(conda_meta_file.filename, icon))
icon = None
if icon is not None:
# relative to conda.recipe
icon = os.path.join(META_DIRECTORY, icon)
if icon is not None:
icon = os.path.join(self.directory_path, icon)
if not os.path.isfile(icon):
problems.append("Icon file %s does not exist." % icon)
icon = None
self.icon = icon
def _update_name(self, problems, project_file, conda_meta_file):
name = project_file.get_value('name', None)
if name is not None:
if not is_string(name):
problems.append(
"%s: name: field should have a string value not %r" %
(project_file.filename, name))
name = None
elif len(name.strip()) == 0:
problems.append(
"%s: name: field is an empty or all-whitespace string." %
(project_file.filename))
name = None
if name is None:
name = conda_meta_file.name
if name is not None and not is_string(name):
problems.append(
"%s: package: name: field should have a string value not %r"
% (conda_meta_file.filename, name))
name = None
if name is None:
name = os.path.basename(self.directory_path)
self.name = name
def _load_environment_yml(filename):
"""Load an environment.yml as an EnvSpec, or None if not loaded."""
try:
with codecs.open(filename, 'r', 'utf-8') as file:
contents = file.read()
yaml = _load_string(contents)
except (IOError, _YAMLError):
return None
name = None
if 'name' in yaml:
name = yaml['name']
if not name:
if 'prefix' in yaml and yaml['prefix']:
name = os.path.basename(yaml['prefix'])
if not name:
name = os.path.basename(filename)
# We don't do too much validation here because we end up doing it
# later if we import this into the project, and then load it from
# the project file. We will do the import such that we don't end up
# keeping the new project file if it's messed up.
#
# However we do try to avoid crashing on None or type errors here.
raw_dependencies = yaml.get('dependencies', [])
if not isinstance(raw_dependencies, list):
raw_dependencies = []
raw_channels = yaml.get('channels', [])
if not isinstance(raw_channels, list):
raw_channels = []
conda_packages = []
pip_packages = []
for dep in raw_dependencies:
if is_string(dep):
conda_packages.append(dep)
elif isinstance(dep, dict) and 'pip' in dep and isinstance(dep['pip'], list):
for pip_dep in dep['pip']:
if is_string(pip_dep):
pip_packages.append(pip_dep)
channels = []
for channel in raw_channels:
if is_string(channel):
channels.append(channel)
return EnvSpec(name=name, conda_packages=conda_packages, channels=channels, pip_packages=pip_packages)
def _parse_default(self, options, env_var, problems):
assert (isinstance(options, dict))
raw_default = options.get('default', None)
if raw_default is None:
good_default = True
elif isinstance(raw_default, bool):
# we have to check bool since it's considered an int apparently
good_default = False
elif is_string(raw_default) or isinstance(raw_default, (int, float)):
good_default = True
else:
good_default = False
if 'default' in options and raw_default is None:
# convert null to be the same as simply missing
del options['default']
if good_default:
return True
else:
problems.append(
"default value for variable {env_var} must be null, a string, or a number, not {value}."
.format(env_var=env_var, value=raw_default))
return False
def _parse(cls, registry, varname, item, problems, requirements):
"""Parse an item from the services: section."""
service_type = None
if is_string(item):
service_type = item
options = dict(type=service_type)
elif isinstance(item, dict):
service_type = item.get('type', None)
if service_type is None:
problems.append(
"Service {} doesn't contain a 'type' field.".format(
varname))
return
options = deepcopy(item)
else:
problems.append(
"Service {} should have a service type string or a dictionary as its value."
.format(varname))
return
if not EnvVarRequirement._parse_default(options, varname, problems):
return
requirement = registry.find_requirement_by_service_type(
service_type=service_type, env_var=varname, options=options)
if requirement is None:
problems.append("Service {} has an unknown type '{}'.".format(
varname, service_type))
else:
assert isinstance(requirement, ServiceRequirement)
assert 'type' in requirement.options
requirements.append(requirement)
def _parse_default(self, options, env_var, problems):
assert (isinstance(options, dict))
raw_default = options.get('default', None)
if raw_default is None:
good_default = True
elif isinstance(raw_default, bool):
# we have to check bool since it's considered an int apparently
good_default = False
elif is_string(raw_default) or isinstance(raw_default, (int, float)):
good_default = True
else:
good_default = False
if 'default' in options and raw_default is None:
# convert null to be the same as simply missing
del options['default']
if good_default:
return True
else:
problems.append(
"default value for variable {env_var} must be null, a string, or a number, not {value}.".format(
env_var=env_var,
value=raw_default))
return False
def _parse(cls, registry, varname, item, problems, requirements):
"""Parse an item from the services: section."""
service_type = None
if is_string(item):
service_type = item
options = dict(type=service_type)
elif isinstance(item, dict):
service_type = item.get('type', None)
if service_type is None:
problems.append("Service {} doesn't contain a 'type' field.".format(varname))
return
options = deepcopy(item)
else:
problems.append("Service {} should have a service type string or a dictionary as its value.".format(
varname))
return
if not EnvVarRequirement._parse_default(options, varname, problems):
return
requirement = registry.find_requirement_by_service_type(service_type=service_type,
env_var=varname,
options=options)
if requirement is None:
problems.append("Service {} has an unknown type '{}'.".format(varname, service_type))
else:
assert isinstance(requirement, ServiceRequirement)
assert 'type' in requirement.options
requirements.append(requirement)
def _path(cls, path):
if is_string(path):
return (path, )
else:
try:
return list(element for element in path)
except TypeError:
raise ValueError("YAML file path must be a string or an iterable of strings")
def status_for(self, env_var_or_class):
"""Get status for the given env var or class, or None if unknown."""
for status in self.statuses:
if is_string(env_var_or_class):
if isinstance(status.requirement, EnvVarRequirement) and \
status.requirement.env_var == env_var_or_class:
return status
elif isinstance(status.requirement, env_var_or_class):
return status
return None
def status_for(self, env_var_or_class):
"""Get status for the given env var or class, or None if unknown."""
for status in self.statuses:
if is_string(env_var_or_class):
if isinstance(status.requirement, EnvVarRequirement) and \
status.requirement.env_var == env_var_or_class:
return status
elif isinstance(status.requirement, env_var_or_class):
return status
return None
def _path(cls, path):
if is_string(path):
return (path, )
else:
try:
return list(element for element in path)
except TypeError:
raise ValueError(
"YAML file path must be a string or an iterable of strings"
)
def _update_description(self, problems, project_file):
desc = project_file.get_value('description', None)
if desc is not None and not is_string(desc):
problems.append("%s: description: field should have a string value not %r" % (project_file.filename, desc))
desc = None
if desc is None:
desc = ''
self.description = desc
def _update_variables(self, requirements, problems, project_file):
variables = project_file.get_value("variables")
def check_conda_reserved(key):
if key in ('CONDA_DEFAULT_ENV', 'CONDA_ENV_PATH', 'CONDA_PREFIX'):
problems.append(("Environment variable %s is reserved for Conda's use, " +
"so it can't appear in the variables section.") % key)
return True
else:
return False
# variables: section can contain a list of var names or a dict from
# var names to options OR default values. it can also be missing
# entirely which is the same as empty.
if variables is None:
pass
elif isinstance(variables, dict):
for key in variables.keys():
if check_conda_reserved(key):
continue
if key.strip() == '':
problems.append("Variable name cannot be empty string, found: '{}' as name".format(key))
continue
raw_options = variables[key]
if raw_options is None:
options = {}
elif isinstance(raw_options, dict):
options = deepcopy(raw_options) # so we can modify it below
else:
options = dict(default=raw_options)
assert (isinstance(options, dict))
if EnvVarRequirement._parse_default(options, key, problems):
requirement = self.registry.find_requirement_by_env_var(key, options)
requirements.append(requirement)
elif isinstance(variables, list):
for item in variables:
if is_string(item):
if item.strip() == '':
problems.append("Variable name cannot be empty string, found: '{}' as name".format(item))
continue
if check_conda_reserved(item):
continue
requirement = self.registry.find_requirement_by_env_var(item, options=dict())
requirements.append(requirement)
else:
problems.append(
"variables section should contain environment variable names, {item} is not a string".format(
item=item))
else:
problems.append(
"variables section contains wrong value type {value}, should be dict or list of requirements".format(
value=variables))
def _update_description(self, problems, project_file):
desc = project_file.get_value('description', None)
if desc is not None and not is_string(desc):
problems.append(
"%s: description: field should have a string value not %r" %
(project_file.filename, desc))
desc = None
if desc is None:
desc = ''
self.description = desc
def _update_name(self, problems, project_file, conda_meta_file):
name = project_file.get_value('name', None)
if name is not None:
if not is_string(name):
problems.append("%s: name: field should have a string value not %r" % (project_file.filename, name))
name = None
elif len(name.strip()) == 0:
problems.append("%s: name: field is an empty or all-whitespace string." % (project_file.filename))
name = None
if name is None:
name = conda_meta_file.name
if name is not None and not is_string(name):
problems.append("%s: package: name: field should have a string value not %r" %
(conda_meta_file.filename, name))
name = None
if name is None:
name = os.path.basename(self.directory_path)
self.name = name
def _in_provide_whitelist(provide_whitelist, requirement):
if provide_whitelist is None:
# whitelist of None means "everything"
return True
for env_var_or_class in provide_whitelist:
if is_string(env_var_or_class):
if isinstance(requirement, EnvVarRequirement) and requirement.env_var == env_var_or_class:
return True
else:
if isinstance(requirement, env_var_or_class):
return True
return False
def _in_provide_whitelist(provide_whitelist, requirement):
if provide_whitelist is None:
# whitelist of None means "everything"
return True
for env_var_or_class in provide_whitelist:
if is_string(env_var_or_class):
if isinstance(requirement, EnvVarRequirement
) and requirement.env_var == env_var_or_class:
return True
else:
if isinstance(requirement, env_var_or_class):
return True
return False
def _update_icon(self, problems, project_file, conda_meta_file):
icon = project_file.get_value('icon', None)
if icon is not None and not is_string(icon):
problems.append("%s: icon: field should have a string value not %r" % (project_file.filename, icon))
icon = None
if icon is None:
icon = conda_meta_file.icon
if icon is not None and not is_string(icon):
problems.append("%s: app: icon: field should have a string value not %r" %
(conda_meta_file.filename, icon))
icon = None
if icon is not None:
# relative to conda.recipe
icon = os.path.join(META_DIRECTORY, icon)
if icon is not None:
icon = os.path.join(self.directory_path, icon)
if not os.path.isfile(icon):
problems.append("Icon file %s does not exist." % icon)
icon = None
self.icon = icon
def _parse_string_list_with_special(parent_dict, key, what, special_filter):
items = parent_dict.get(key, [])
if not isinstance(items, (list, tuple)):
problems.append("%s: %s: value should be a list of %ss, not '%r'" %
(project_file.filename, key, what, items))
return ([], [])
cleaned = []
special = []
for item in items:
if is_string(item):
cleaned.append(item.strip())
elif special_filter(item):
special.append(item)
else:
problems.append("%s: %s: value should be a %s (as a string) not '%r'" %
(project_file.filename, key, what, item))
return (cleaned, special)
def __init__(self, registry, options):
"""Construct a Requirement.
Args:
registry (PluginRegistry): the plugin registry we came from
options (dict): dict of requirement options from the project config
"""
self.registry = registry
if options is None:
self.options = dict()
else:
self.options = deepcopy(options)
# always convert the default to a string (it's allowed to be a number
# in the config file, but env vars have to be strings), unless
# it's a dict because we use a dict for encrypted defaults
if 'default' in self.options and not (is_string(self.options['default']) or isinstance(
self.options['default'], dict)):
self.options['default'] = str(self.options['default'])
def __init__(self, registry, options):
"""Construct a Requirement.
Args:
registry (PluginRegistry): the plugin registry we came from
options (dict): dict of requirement options from the project config
"""
self.registry = registry
if options is None:
self.options = dict()
else:
self.options = deepcopy(options)
# always convert the default to a string (it's allowed to be a number
# in the config file, but env vars have to be strings), unless
# it's a dict because we use a dict for encrypted defaults
if 'default' in self.options and not (
is_string(self.options['default'])
or isinstance(self.options['default'], dict)):
self.options['default'] = str(self.options['default'])
def _parse_string_list_with_special(parent_dict, key, what,
special_filter):
items = parent_dict.get(key, [])
if not isinstance(items, (list, tuple)):
problems.append(
"%s: %s: value should be a list of %ss, not '%r'" %
(project_file.filename, key, what, items))
return ([], [])
cleaned = []
special = []
for item in items:
if is_string(item):
cleaned.append(item.strip())
elif special_filter(item):
special.append(item)
else:
problems.append(
"%s: %s: value should be a %s (as a string) not '%r'" %
(project_file.filename, key, what, item))
return (cleaned, special)
def _update_env_specs(self, problems, project_file):
def _parse_string_list_with_special(parent_dict, key, what,
special_filter):
items = parent_dict.get(key, [])
if not isinstance(items, (list, tuple)):
problems.append(
"%s: %s: value should be a list of %ss, not '%r'" %
(project_file.filename, key, what, items))
return ([], [])
cleaned = []
special = []
for item in items:
if is_string(item):
cleaned.append(item.strip())
elif special_filter(item):
special.append(item)
else:
problems.append(
"%s: %s: value should be a %s (as a string) not '%r'" %
(project_file.filename, key, what, item))
return (cleaned, special)
def _parse_string_list(parent_dict, key, what):
return _parse_string_list_with_special(
parent_dict, key, what, special_filter=lambda x: False)[0]
def _parse_channels(parent_dict):
return _parse_string_list(parent_dict, 'channels', 'channel name')
def _parse_packages(parent_dict):
(deps, pip_dicts) = _parse_string_list_with_special(
parent_dict, 'packages', 'package name',
lambda x: isinstance(x, dict) and ('pip' in x))
for dep in deps:
parsed = conda_api.parse_spec(dep)
if parsed is None:
problems.append("%s: invalid package specification: %s" %
(project_file.filename, dep))
# note that multiple "pip:" dicts are allowed
pip_deps = []
for pip_dict in pip_dicts:
pip_list = _parse_string_list(pip_dict, 'pip',
'pip package name')
pip_deps.extend(pip_list)
for dep in pip_deps:
parsed = pip_api.parse_spec(dep)
if parsed is None:
problems.append("%s: invalid pip package specifier: %s" %
(project_file.filename, dep))
return (deps, pip_deps)
self.env_specs = dict()
(shared_deps, shared_pip_deps) = _parse_packages(project_file.root)
shared_channels = _parse_channels(project_file.root)
env_specs = project_file.get_value('env_specs', default={})
first_env_spec_name = None
env_specs_is_empty_or_missing = False # this should be iff it's an empty dict or absent entirely
if isinstance(env_specs, dict):
if len(env_specs) == 0:
env_specs_is_empty_or_missing = True
for (name, attrs) in env_specs.items():
if name.strip() == '':
problems.append(
"Environment spec name cannot be empty string, found: '{}' as name"
.format(name))
continue
description = attrs.get('description', None)
if description is not None and not is_string(description):
problems.append(
"{}: 'description' field of environment {} must be a string"
.format(project_file.filename, name))
continue
(deps, pip_deps) = _parse_packages(attrs)
channels = _parse_channels(attrs)
# ideally we would merge same-name packages here, choosing the
# highest of the two versions or something. maybe conda will
# do that for us anyway?
all_deps = shared_deps + deps
all_pip_deps = shared_pip_deps + pip_deps
all_channels = shared_channels + channels
self.env_specs[name] = EnvSpec(name=name,
conda_packages=all_deps,
pip_packages=all_pip_deps,
channels=all_channels,
description=description)
if first_env_spec_name is None:
first_env_spec_name = name
else:
problems.append(
"%s: env_specs should be a dictionary from environment name to environment attributes, not %r"
% (project_file.filename, env_specs))
if env_specs_is_empty_or_missing:
# we do NOT want to add this problem if we merely
# failed to parse individual env specs; it must be
# safe to overwrite the env_specs key, so it has to
# be empty or missing entirely.
def add_default_env_spec(project):
project.project_file.set_value(['env_specs', 'default'],
dict(packages=[], channels=[]))
project.project_file.save()
problems.append(
ProjectProblem(
text=("%s has an empty env_specs section." %
project_file.filename),
fix_prompt=("Add an environment spec to %s?" %
os.path.basename(project_file.filename)),
fix_function=add_default_env_spec))
# this is only used for commands that don't specify anything
# (when/if we require all commands to specify, then remove this.)
if 'default' in self.env_specs:
self.default_env_spec_name = 'default'
else:
self.default_env_spec_name = first_env_spec_name
def _update_commands(self, problems, project_file, conda_meta_file,
requirements):
failed = False
app_entry_from_meta_yaml = conda_meta_file.app_entry
if app_entry_from_meta_yaml is not None:
if not is_string(app_entry_from_meta_yaml):
problems.append(
"%s: app: entry: should be a string not '%r'" %
(conda_meta_file.filename, app_entry_from_meta_yaml))
app_entry_from_meta_yaml = None
failed = True
first_command_name = None
commands = dict()
commands_section = project_file.get_value('commands', None)
if commands_section is not None and not isinstance(
commands_section, dict):
problems.append(
"%s: 'commands:' section should be a dictionary from command names to attributes, not %r"
% (project_file.filename, commands_section))
failed = True
elif commands_section is not None:
for (name, attrs) in commands_section.items():
if name.strip() == '':
problems.append(
"Command variable name cannot be empty string, found: '{}' as name"
.format(name))
failed = True
continue
if first_command_name is None:
first_command_name = name
if not isinstance(attrs, dict):
problems.append(
"%s: command name '%s' should be followed by a dictionary of attributes not %r"
% (project_file.filename, name, attrs))
failed = True
continue
if 'description' in attrs and not is_string(
attrs['description']):
problems.append(
"{}: 'description' field of command {} must be a string"
.format(project_file.filename, name))
failed = True
if 'env_spec' in attrs:
if not is_string(attrs['env_spec']):
problems.append(
"{}: 'env_spec' field of command {} must be a string (an environment spec name)"
.format(project_file.filename, name))
failed = True
elif attrs['env_spec'] not in self.env_specs:
problems.append(
"%s: env_spec '%s' for command '%s' does not appear in the env_specs section"
% (project_file.filename, attrs['env_spec'], name))
failed = True
copied_attrs = deepcopy(attrs)
if 'env_spec' not in copied_attrs:
copied_attrs['env_spec'] = self.default_env_spec_name
command_types = []
for attr in ALL_COMMAND_TYPES:
if attr not in copied_attrs:
continue
# be sure we add this even if the command is broken, since it's
# confusing to say "does not have a command line in it" below
# if the issue is that the command line is broken.
command_types.append(attr)
if not is_string(copied_attrs[attr]):
problems.append(
"%s: command '%s' attribute '%s' should be a string not '%r'"
% (project_file.filename, name, attr,
copied_attrs[attr]))
failed = True
if len(command_types) == 0:
problems.append(
"%s: command '%s' does not have a command line in it" %
(project_file.filename, name))
failed = True
if ('notebook' in copied_attrs or 'bokeh_app'
in copied_attrs) and (len(command_types) > 1):
label = 'bokeh_app' if 'bokeh_app' in copied_attrs else 'notebook'
others = copy(command_types)
others.remove(label)
others = [("'%s'" % other) for other in others]
problems.append(
"%s: command '%s' has multiple commands in it, '%s' can't go with %s"
% (project_file.filename, name, label,
", ".join(others)))
failed = True
# note that once one command fails, we don't add any more
if not failed:
commands[name] = ProjectCommand(name=name,
attributes=copied_attrs)
self._add_notebook_commands(commands, problems, requirements)
if failed:
self.commands = dict()
self.default_command_name = None
else:
# if no commands and we have a meta.yaml app entry, use the meta.yaml
if app_entry_from_meta_yaml is not None and len(commands) == 0:
commands['default'] = ProjectCommand(
name='default',
attributes=dict(conda_app_entry=app_entry_from_meta_yaml,
auto_generated=True,
env_spec=self.default_env_spec_name))
self.commands = commands
if first_command_name is None and len(commands) > 0:
# this happens if we created a command automatically
# from a notebook file or conda meta.yaml
first_command_name = sorted(commands.keys())[0]
if 'default' in self.commands:
self.default_command_name = 'default'
else:
# 'default' is always mapped to the first-listed if none is named 'default'
# note: this may be None
self.default_command_name = first_command_name
def _parse(cls, registry, varname, item, problems, requirements):
"""Parse an item from the downloads: section."""
url = None
filename = None
hash_algorithm = None
hash_value = None
unzip = None
description = None
if is_string(item):
url = item
elif isinstance(item, dict):
url = item.get('url', None)
if url is None:
problems.append("Download item {} doesn't contain a 'url' field.".format(varname))
return
description = item.get('description', None)
if description is not None and not is_string(description):
problems.append("'description' field for download item {} is not a string".format(varname))
return
for method in _hash_algorithms:
if method not in item:
continue
if hash_algorithm is not None:
problems.append("Multiple checksums for download {}: {} and {}.".format(varname, hash_algorithm,
method))
return
else:
hash_value = item[method]
if is_string(hash_value):
hash_algorithm = method
else:
problems.append("Checksum value for {} should be a string not {}.".format(varname, hash_value))
return
filename = item.get('filename', None)
unzip = item.get('unzip', None)
if unzip is not None and not isinstance(unzip, bool):
problems.append("Value of 'unzip' for download item {} should be a boolean, not {}.".format(varname,
unzip))
return
if url is None or not is_string(url):
problems.append(("Download name {} should be followed by a URL string or a dictionary " +
"describing the download.").format(varname))
return
if url == '':
problems.append("Download item {} has an empty 'url' field.".format(varname))
return
# urlsplit doesn't seem to ever throw an exception, but it can
# return pretty nonsensical stuff on invalid urls, in particular
# an empty path is very possible
url_path = os.path.basename(urlparse.urlsplit(url).path)
url_path_is_zip = url_path.lower().endswith(".zip")
if filename is None:
if url_path != '':
filename = url_path
if url_path_is_zip:
if unzip is None:
# url is a zip and neither filename nor unzip specified, assume unzip
unzip = True
if unzip:
# unzip specified True, or we guessed True, and url ends in zip;
# take the .zip off the filename we invented based on the url.
filename = filename[:-4]
elif url_path_is_zip and unzip is None and not filename.lower().endswith(".zip"):
# URL is a zip, filename is not a zip, unzip was not specified, so assume
# we want to unzip
unzip = True
if filename is None:
filename = varname
if unzip is None:
unzip = False
requirements.append(DownloadRequirement(registry,
env_var=varname,
url=url,
filename=filename,
hash_algorithm=hash_algorithm,
hash_value=hash_value,
unzip=unzip,
description=description))
def _update_variables(self, requirements, problems, project_file):
variables = project_file.get_value("variables")
def check_conda_reserved(key):
if key in ('CONDA_DEFAULT_ENV', 'CONDA_ENV_PATH', 'CONDA_PREFIX'):
problems.append(
("Environment variable %s is reserved for Conda's use, " +
"so it can't appear in the variables section.") % key)
return True
else:
return False
# variables: section can contain a list of var names or a dict from
# var names to options OR default values. it can also be missing
# entirely which is the same as empty.
if variables is None:
pass
elif isinstance(variables, dict):
for key in variables.keys():
if check_conda_reserved(key):
continue
if key.strip() == '':
problems.append(
"Variable name cannot be empty string, found: '{}' as name"
.format(key))
continue
raw_options = variables[key]
if raw_options is None:
options = {}
elif isinstance(raw_options, dict):
options = deepcopy(
raw_options) # so we can modify it below
else:
options = dict(default=raw_options)
assert (isinstance(options, dict))
if EnvVarRequirement._parse_default(options, key, problems):
requirement = self.registry.find_requirement_by_env_var(
key, options)
requirements.append(requirement)
elif isinstance(variables, list):
for item in variables:
if is_string(item):
if item.strip() == '':
problems.append(
"Variable name cannot be empty string, found: '{}' as name"
.format(item))
continue
if check_conda_reserved(item):
continue
requirement = self.registry.find_requirement_by_env_var(
item, options=dict())
requirements.append(requirement)
else:
problems.append(
"variables section should contain environment variable names, {item} is not a string"
.format(item=item))
else:
problems.append(
"variables section contains wrong value type {value}, should be dict or list of requirements"
.format(value=variables))
def _update_commands(self, problems, project_file, conda_meta_file, requirements):
failed = False
first_command_name = None
commands = dict()
commands_section = project_file.get_value('commands', None)
if commands_section is not None and not isinstance(commands_section, dict):
problems.append("%s: 'commands:' section should be a dictionary from command names to attributes, not %r" %
(project_file.filename, commands_section))
failed = True
elif commands_section is not None:
for (name, attrs) in commands_section.items():
if name.strip() == '':
problems.append("Command variable name cannot be empty string, found: '{}' as name".format(name))
failed = True
continue
if first_command_name is None:
first_command_name = name
if not isinstance(attrs, dict):
problems.append("%s: command name '%s' should be followed by a dictionary of attributes not %r" %
(project_file.filename, name, attrs))
failed = True
continue
if 'description' in attrs and not is_string(attrs['description']):
problems.append("{}: 'description' field of command {} must be a string".format(
project_file.filename, name))
failed = True
if 'supports_http_options' in attrs and not isinstance(attrs['supports_http_options'], bool):
problems.append("{}: 'supports_http_options' field of command {} must be a boolean".format(
project_file.filename, name))
failed = True
if 'env_spec' in attrs:
if not is_string(attrs['env_spec']):
problems.append(
"{}: 'env_spec' field of command {} must be a string (an environment spec name)".format(
project_file.filename, name))
failed = True
elif attrs['env_spec'] not in self.env_specs:
problems.append("%s: env_spec '%s' for command '%s' does not appear in the env_specs section" %
(project_file.filename, attrs['env_spec'], name))
failed = True
copied_attrs = deepcopy(attrs)
if 'env_spec' not in copied_attrs:
copied_attrs['env_spec'] = self.default_env_spec_name
command_types = []
for attr in ALL_COMMAND_TYPES:
if attr not in copied_attrs:
continue
# be sure we add this even if the command is broken, since it's
# confusing to say "does not have a command line in it" below
# if the issue is that the command line is broken.
command_types.append(attr)
if not is_string(copied_attrs[attr]):
problems.append("%s: command '%s' attribute '%s' should be a string not '%r'" %
(project_file.filename, name, attr, copied_attrs[attr]))
failed = True
if len(command_types) == 0:
problems.append("%s: command '%s' does not have a command line in it" %
(project_file.filename, name))
failed = True
if ('notebook' in copied_attrs or 'bokeh_app' in copied_attrs) and (len(command_types) > 1):
label = 'bokeh_app' if 'bokeh_app' in copied_attrs else 'notebook'
others = copy(command_types)
others.remove(label)
others = [("'%s'" % other) for other in others]
problems.append("%s: command '%s' has multiple commands in it, '%s' can't go with %s" %
(project_file.filename, name, label, ", ".join(others)))
failed = True
# note that once one command fails, we don't add any more
if not failed:
commands[name] = ProjectCommand(name=name, attributes=copied_attrs)
self._verify_notebook_commands(commands, problems, requirements, project_file)
if failed:
self.commands = dict()
self.default_command_name = None
else:
self.commands = commands
if 'default' in self.commands:
self.default_command_name = 'default'
else:
# 'default' is always mapped to the first-listed if none is named 'default'
# note: this may be None
self.default_command_name = first_command_name
def _update_env_specs(self, problems, project_file):
def _parse_string_list_with_special(parent_dict, key, what, special_filter):
items = parent_dict.get(key, [])
if not isinstance(items, (list, tuple)):
problems.append("%s: %s: value should be a list of %ss, not '%r'" %
(project_file.filename, key, what, items))
return ([], [])
cleaned = []
special = []
for item in items:
if is_string(item):
cleaned.append(item.strip())
elif special_filter(item):
special.append(item)
else:
problems.append("%s: %s: value should be a %s (as a string) not '%r'" %
(project_file.filename, key, what, item))
return (cleaned, special)
def _parse_string_list(parent_dict, key, what):
return _parse_string_list_with_special(parent_dict, key, what, special_filter=lambda x: False)[0]
def _parse_channels(parent_dict):
return _parse_string_list(parent_dict, 'channels', 'channel name')
def _parse_packages(parent_dict):
(deps, pip_dicts) = _parse_string_list_with_special(parent_dict, 'packages', 'package name',
lambda x: isinstance(x, dict) and ('pip' in x))
for dep in deps:
parsed = conda_api.parse_spec(dep)
if parsed is None:
problems.append("%s: invalid package specification: %s" % (project_file.filename, dep))
# note that multiple "pip:" dicts are allowed
pip_deps = []
for pip_dict in pip_dicts:
pip_list = _parse_string_list(pip_dict, 'pip', 'pip package name')
pip_deps.extend(pip_list)
for dep in pip_deps:
parsed = pip_api.parse_spec(dep)
if parsed is None:
problems.append("%s: invalid pip package specifier: %s" % (project_file.filename, dep))
return (deps, pip_deps)
(shared_deps, shared_pip_deps) = _parse_packages(project_file.root)
shared_channels = _parse_channels(project_file.root)
env_specs = project_file.get_value('env_specs', default={})
first_env_spec_name = None
env_specs_is_empty_or_missing = False # this should be iff it's an empty dict or absent entirely
# this one isn't in the env_specs dict
self.global_base_env_spec = EnvSpec(name=None,
conda_packages=shared_deps,
pip_packages=shared_pip_deps,
channels=shared_channels,
description="Global packages and channels",
inherit_from_names=(),
inherit_from=())
env_spec_attrs = dict()
if isinstance(env_specs, dict):
if len(env_specs) == 0:
env_specs_is_empty_or_missing = True
for (name, attrs) in env_specs.items():
if name.strip() == '':
problems.append("Environment spec name cannot be empty string, found: '{}' as name".format(name))
continue
description = attrs.get('description', None)
if description is not None and not is_string(description):
problems.append("{}: 'description' field of environment {} must be a string".format(
project_file.filename, name))
continue
problem_count = len(problems)
inherit_from_names = attrs.get('inherit_from', None)
if inherit_from_names is None:
inherit_from_names = []
elif is_string(inherit_from_names):
inherit_from_names = [inherit_from_names.strip()]
else:
inherit_from_names = _parse_string_list(attrs, 'inherit_from', 'env spec name')
if len(problems) > problem_count:
# we got a new problem from the bad inherit_from
continue
(deps, pip_deps) = _parse_packages(attrs)
channels = _parse_channels(attrs)
env_spec_attrs[name] = dict(name=name,
conda_packages=deps,
pip_packages=pip_deps,
channels=channels,
description=description,
inherit_from_names=tuple(inherit_from_names),
inherit_from=())
if first_env_spec_name is None:
first_env_spec_name = name
else:
problems.append(
"%s: env_specs should be a dictionary from environment name to environment attributes, not %r" %
(project_file.filename, env_specs))
self.env_specs = dict()
def make_env_spec(name, trail):
assert name in env_spec_attrs
if name not in self.env_specs:
was_cycle = False
if name in trail:
problems.append(
"{}: 'inherit_from' fields create circular inheritance among these env specs: {}".format(
project_file.filename, ", ".join(sorted(trail))))
was_cycle = True
trail.append(name)
attrs = env_spec_attrs[name]
if not was_cycle:
inherit_from_names = attrs['inherit_from_names']
for parent in inherit_from_names:
if parent not in env_spec_attrs:
problems.append(("{}: name '{}' in 'inherit_from' field of env spec {} does not match " +
"the name of another env spec").format(project_file.filename, parent,
attrs['name']))
else:
inherit_from = make_env_spec(parent, trail)
attrs['inherit_from'] = attrs['inherit_from'] + (inherit_from, )
# All parent-less env specs get the global base spec as parent,
# which means the global base spec is in everyone's ancestry
if attrs['inherit_from'] == ():
attrs['inherit_from'] = (self.global_base_env_spec, )
self.env_specs[name] = EnvSpec(**attrs)
return self.env_specs[name]
for name in env_spec_attrs.keys():
make_env_spec(name, [])
assert name in self.env_specs
# it's important to create all the env specs when possible
# even if they are broken (e.g. bad inherit_from), so they
# can be edited in order to fix them
(importable_spec, importable_filename) = _find_out_of_sync_importable_spec(self.env_specs.values(),
self.directory_path)
if importable_spec is not None:
skip_spec_import = project_file.get_value(['skip_imports', 'environment'])
if skip_spec_import == importable_spec.channels_and_packages_hash:
importable_spec = None
if importable_spec is not None:
old = self.env_specs.get(importable_spec.name)
# this is a pretty bad hack, but if we injected "notebook"
# or "bokeh" deps to make a notebook/bokeh command work,
# we will end up out-of-sync for that reason
# alone. environment.yml seems to typically not have
# "notebook" in it because the environment.yml is used for
# the kernel but not Jupyter itself.
# We then end up in a problem loop where we complain about
# missing notebook dep, add it, then complain about environment.yml
# out of sync, and `conda-kapsel init` in a directory with a .ipynb
# and an environment.yml doesn't result in a valid project.
if importable_spec is not None and old is not None and \
importable_spec.diff_only_removes_notebook_or_bokeh(old):
importable_spec = None
if importable_spec is not None:
if old is None:
text = "Environment spec '%s' from %s is not in %s." % (importable_spec.name, importable_filename,
os.path.basename(project_file.filename))
prompt = "Add env spec %s to %s?" % (importable_spec.name, os.path.basename(project_file.filename))
else:
text = "Environment spec '%s' from %s is out of sync with %s. Diff:\n%s" % (
importable_spec.name, importable_filename, os.path.basename(project_file.filename),
importable_spec.diff_from(old))
prompt = "Overwrite env spec %s with the changes from %s?" % (importable_spec.name, importable_filename)
def overwrite_env_spec_from_importable(project):
project.project_file.set_value(['env_specs', importable_spec.name], importable_spec.to_json())
def remember_no_import_importable(project):
project.project_file.set_value(['skip_imports', 'environment'],
importable_spec.channels_and_packages_hash)
problems.append(ProjectProblem(text=text,
fix_prompt=prompt,
fix_function=overwrite_env_spec_from_importable,
no_fix_function=remember_no_import_importable))
elif env_specs_is_empty_or_missing:
# we do NOT want to add this problem if we merely
# failed to parse individual env specs; it must be
# safe to overwrite the env_specs key, so it has to
# be empty or missing entirely. Also, we do NOT want
# to add this if we are going to ask about environment.yml
# import, above.
def add_default_env_spec(project):
default_spec = _anaconda_default_env_spec(self.global_base_env_spec)
project.project_file.set_value(['env_specs', default_spec.name], default_spec.to_json())
problems.append(ProjectProblem(text=("%s has an empty env_specs section." % project_file.filename),
fix_prompt=("Add an environment spec to %s?" % os.path.basename(
project_file.filename)),
fix_function=add_default_env_spec))
# this is only used for commands that don't specify anything
# (when/if we require all commands to specify, then remove this.)
if 'default' in self.env_specs:
self.default_env_spec_name = 'default'
else:
self.default_env_spec_name = first_env_spec_name