def check(dirname):
from anaconda_project.requirements_registry.requirements.conda_env import CondaEnvRequirement
_monkeypatch_download_file(monkeypatch, dirname, filename="nope")
no_foo = [('missing requirement to run this project: A downloaded file which is ' + 'referenced by FOO.'),
' Environment variable FOO is not set.']
# whitelist only the env req by class
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, provide_whitelist=(CondaEnvRequirement, ), environ=environ)
assert result.errors == no_foo
# whitelist by instance
env_req = [req for req in project.requirements(None) if isinstance(req, CondaEnvRequirement)][0]
result = prepare_without_interaction(project, provide_whitelist=(env_req, ), environ=environ)
assert result.errors == no_foo
# whitelist by variable name
result = prepare_without_interaction(project, provide_whitelist=(env_req.env_var, ), environ=environ)
assert result.errors == no_foo
# whitelist the download
result = prepare_without_interaction(project,
provide_whitelist=(env_req, project.download_requirements(None)[0]),
environ=environ)
assert result.errors == []
assert 'FOO' in result.environ
def provide_download(dirname):
project = project_no_dedicated_env(dirname)
prepare_without_interaction(
project, environ=minimal_environ(PROJECT_DIR=dirname))
assert (
"%s: 'downloads:' section should be a dictionary, found ['http://localhost/data.csv']"
% DEFAULT_PROJECT_FILENAME) in project.problems
def check(dirname):
env_var = conda_api.conda_prefix_variable()
try:
_push_fake_env_creator()
project = Project(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project,
environ=environ,
env_spec_name='foo')
expected_path = project.env_specs['foo'].path(
project.directory_path)
assert result.environ[env_var] == expected_path
environ = minimal_environ()
result = prepare_without_interaction(project,
environ=environ,
env_spec_name='bar')
assert result.errors == []
assert result
expected_path = project.env_specs['bar'].path(
project.directory_path)
assert result.environ[env_var] == expected_path
finally:
_pop_fake_env_creator()
def check(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ, command_name='foo')
assert result.errors == []
assert result
assert os.path.join(project.directory_path, 'foo.py') in result.command_exec_info.args
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ, command_name='bar')
assert result.errors == []
assert result
assert os.path.join(project.directory_path, 'bar.py') in result.command_exec_info.args
def prepare_project_scoped_env(dirname):
project = Project(dirname)
fake_old_path = "foo" + os.pathsep + "bar"
environ = dict(PROJECT_DIR=dirname, PATH=fake_old_path)
result = prepare_without_interaction(project, environ=environ)
assert result
expected_env = os.path.join(dirname, "envs", "bootstrap-env")
if platform.system() == 'Windows':
expected_new_path = expected_env + os.pathsep + os.path.join(
expected_env, script_dir) + os.pathsep + os.path.join(
expected_env, "Library",
"bin") + os.pathsep + "foo" + os.pathsep + "bar"
else:
expected_new_path = os.path.join(
expected_env,
script_dir) + os.pathsep + "foo" + os.pathsep + "bar"
expected = dict(PROJECT_DIR=project.directory_path,
PATH=expected_new_path,
BOOTSTRAP_ENV_PREFIX=expected_env)
conda_api.environ_set_prefix(expected, expected_env)
expected == result.environ
assert os.path.exists(os.path.join(expected_env, "conda-meta"))
conda_meta_mtime = os.path.getmtime(
os.path.join(expected_env, "conda-meta"))
# bare minimum bootstrap-env env shouldn't include these
# (contrast with the test later where we list them in
# requirements)
installed = conda_api.installed(expected_env)
assert 'ipython' not in installed
assert 'numpy' not in installed
# Prepare it again should no-op (use the already-existing environment)
environ = dict(PROJECT_DIR=dirname, PATH=fake_old_path)
result = prepare_without_interaction(project, environ=environ)
assert result
expected = dict(PROJECT_DIR=project.directory_path,
PATH=expected_new_path)
conda_api.environ_set_prefix(expected, expected_env)
assert conda_meta_mtime == os.path.getmtime(
os.path.join(expected_env, "conda-meta"))
# Now unprepare
status = unprepare(project, result)
assert status
# todo: this differs from standard CondaEnvProvider
assert status.status_description == 'Success.'
assert status.errors == []
assert not os.path.exists(expected_env)
def prepare_system_environ(dirname):
project = project_no_dedicated_env(dirname)
os_environ_copy = deepcopy(os.environ)
result = prepare_without_interaction(project)
assert result
assert result.errors == []
assert project.directory_path == strip_environ(
result.environ)['PROJECT_DIR']
# os.environ wasn't modified
assert os_environ_copy == os.environ
# result.environ inherits everything in os.environ
for key, original in os_environ_copy.items():
updated = result.environ.get(key)
if updated != original:
if original in ('root', 'base') and updated in ('root',
'base'):
print("we have a root/base environment name issue here")
continue
if key == 'PATH' and platform.system() == 'Windows':
print(
"prepare changed PATH on Windows and ideally it would not."
)
continue
updated = updated.split(os.pathsep)
original = original.split(os.pathsep)
print("ORIGINAL {}: {}".format(key, repr(original)))
print("UPDATED {}: {}".format(key, repr(updated)))
assert updated == original
def provide_download(dirname):
@gen.coroutine
def mock_downloader_run(self, loop):
class Res:
pass
res = Res()
res.code = 200
with open(os.path.join(dirname, 'data.csv'), 'w') as out:
out.write('data')
self._hash = '12345abcdef'
raise gen.Return(res)
monkeypatch.setattr(
"anaconda_project.internal.http_client.FileDownloader.run",
mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(
project, environ=minimal_environ(PROJECT_DIR=dirname))
assert hasattr(result, 'environ')
assert 'DATAFILE' in result.environ
filename = os.path.join(dirname, 'data.csv')
assert os.path.exists(filename)
project.frontend.reset()
status = unprepare(project, result)
assert project.frontend.logs == [
"Removed downloaded file %s." % filename,
("Current environment is not in %s, no need to delete it." %
dirname)
]
assert status.status_description == 'Success.'
assert status
assert not os.path.exists(filename)
def provide_download_of_zip(dirname):
with codecs.open(os.path.join(dirname, DEFAULT_PROJECT_FILENAME), 'w',
'utf-8') as f:
f.write(complete_project_file_content(ZIPPED_DATAFILE_CONTENT))
@gen.coroutine
def mock_downloader_run(self, loop):
class Res:
pass
res = Res()
res.code = 200
assert self._url.endswith(".zip")
assert self._filename.endswith(".zip")
with codecs.open(self._filename, 'w', 'utf-8') as f:
f.write("This is not a zip file.")
raise gen.Return(res)
monkeypatch.setattr(
"anaconda_project.internal.http_client.FileDownloader.run",
mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(
project, environ=minimal_environ(PROJECT_DIR=dirname))
assert not result
assert [(
"Failed to unzip %s: File is not a zip file" %
os.path.join(dirname, "data.zip")
), "missing requirement to run this project: A downloaded file which is referenced by DATAFILE.",
" Environment variable DATAFILE is not set."] == result.errors
def provide_download_of_zip(zipname, dirname):
with codecs.open(os.path.join(dirname, DEFAULT_PROJECT_FILENAME), 'w',
'utf-8') as f:
f.write(
complete_project_file_content(
ZIPPED_DATAFILE_CONTENT_NO_ZIP_SUFFIX))
@gen.coroutine
def mock_downloader_run(self, loop):
class Res:
pass
res = Res()
res.code = 200
# we add .zip to the download filename, even though it wasn't in the URL
assert not self._url.endswith(".zip")
assert self._filename.endswith(".zip")
shutil.copyfile(zipname, self._filename)
raise gen.Return(res)
monkeypatch.setattr(
"anaconda_project.internal.http_client.FileDownloader.run",
mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(
project, environ=minimal_environ(PROJECT_DIR=dirname))
assert hasattr(result, 'environ')
assert 'DATAFILE' in result.environ
assert os.path.isdir(os.path.join(dirname, 'data'))
assert os.path.isfile(os.path.join(dirname, 'data', 'foo'))
assert codecs.open(os.path.join(dirname, 'data',
'foo')).read() == 'hello\n'
def provide_download_of_zip_no_unzip(zipname, dirname):
with codecs.open(os.path.join(dirname, DEFAULT_PROJECT_FILENAME), 'w',
'utf-8') as f:
f.write(
complete_project_file_content(
ZIPPED_DATAFILE_CONTENT_NO_UNZIP))
@gen.coroutine
def mock_downloader_run(self, loop):
class Res:
pass
res = Res()
res.code = 200
assert self._url.endswith(".zip")
# we aren't going to unzip so we should be downloading straignt to
# the specified filename 'data' without the .zip on it
assert not self._filename.endswith(".zip")
shutil.copyfile(zipname, self._filename)
raise gen.Return(res)
monkeypatch.setattr(
"anaconda_project.internal.http_client.FileDownloader.run",
mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(
project, environ=minimal_environ(PROJECT_DIR=dirname))
assert hasattr(result, 'environ')
assert 'DATAFILE' in result.environ
assert os.path.isfile(os.path.join(dirname, 'data'))
with zipfile.ZipFile(os.path.join(dirname, 'data')) as zf:
assert zf.namelist() == ['foo']
def provide_download(dirname):
@gen.coroutine
def mock_downloader_run(self, loop):
raise Exception('error')
monkeypatch.setattr(
"anaconda_project.internal.http_client.FileDownloader.run",
mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(
project, environ=minimal_environ(PROJECT_DIR=dirname))
assert not result
assert (
'missing requirement to run this project: A downloaded file which is referenced by DATAFILE.'
) in result.errors
project.frontend.reset()
status = unprepare(project, result)
filename = os.path.join(dirname, 'data.csv')
assert project.frontend.logs == [
"No need to remove %s which wasn't downloaded." % filename,
("Current environment is not in %s, no need to delete it." %
dirname)
]
assert status.status_description == 'Success.'
def prepare_system_environ(dirname):
project = project_no_dedicated_env(dirname)
os_environ_copy = deepcopy(os.environ)
result = prepare_without_interaction(project)
assert project.directory_path == strip_environ(
result.environ)['PROJECT_DIR']
# os.environ wasn't modified
assert os_environ_copy == os.environ
# result.environ inherits everything in os.environ
for key in os_environ_copy:
if key == 'PATH' and platform.system(
) == 'Windows' and result.environ[key] != os.environ[key]:
print(
"prepare changed PATH on Windows and ideally it would not."
)
else:
if key == 'PATH' and result.environ[key] != os.environ[key]:
original = os.environ[key].split(os.pathsep)
updated = result.environ[key].split(os.pathsep)
print("ORIGINAL PATH: " + repr(original))
print("UPDATED PATH: " + repr(updated))
assert original == updated
assert result.errors == []
assert result
assert result.environ.get(key) == os.environ.get(key)
def provide_download_of_zip(zipname, dirname):
with codecs.open(os.path.join(dirname, DEFAULT_PROJECT_FILENAME), 'w', 'utf-8') as f:
f.write(complete_project_file_content(ZIPPED_DATAFILE_CONTENT_CHECKSUM))
@gen.coroutine
def mock_downloader_run(self, loop):
class Res:
pass
res = Res()
res.code = 200
assert self._url.endswith(".zip")
assert self._filename.endswith(".zip")
shutil.copyfile(zipname, self._filename)
self._hash = '12345abcdef'
raise gen.Return(res)
monkeypatch.setattr("anaconda_project.internal.http_client.FileDownloader.run", mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(PROJECT_DIR=dirname))
assert hasattr(result, 'environ')
assert 'DATAFILE' in result.environ
assert os.path.isdir(os.path.join(dirname, 'data'))
assert os.path.isfile(os.path.join(dirname, 'data', 'foo'))
assert codecs.open(os.path.join(dirname, 'data', 'foo')).read() == 'hello\n'
project.frontend.reset()
status = unprepare(project, result)
filename = os.path.join(dirname, 'data')
assert project.frontend.logs == ["Removed downloaded file %s." % filename,
("Current environment is not in %s, no need to delete it." % dirname)]
assert status.status_description == "Success."
def provide_download(dirname):
@gen.coroutine
def mock_downloader_run(self, loop):
class Res:
pass
res = Res()
res.code = 200
with open(os.path.join(dirname, 'data.csv'), 'w') as out:
out.write('data')
self._hash = '12345abcdef'
raise gen.Return(res)
monkeypatch.setattr("anaconda_project.internal.http_client.FileDownloader.run", mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(PROJECT_DIR=dirname))
assert hasattr(result, 'environ')
assert 'DATAFILE' in result.environ
filename = os.path.join(dirname, 'data.csv')
assert os.path.exists(filename)
def mock_remove(path):
raise IOError("Not gonna remove this")
monkeypatch.setattr("os.remove", mock_remove)
project.frontend.reset()
status = unprepare(project, result)
assert project.frontend.logs == []
assert status.status_description == ('Failed to remove %s: Not gonna remove this.' % filename)
assert status.errors == []
assert not status
assert os.path.exists(filename)
monkeypatch.undo() # so os.remove isn't broken during directory cleanup
def check(dirname):
project = Project(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert result
# now mimmick an unpacked project
packed_file = os.path.join(dirname, 'envs', 'default', 'conda-meta',
'.packed')
with open(packed_file, 'wt') as f:
f.write(conda_api.current_platform())
# without a functional conda-unpack script it will rebuild the env
result = prepare_without_interaction(project, environ=environ)
assert result
assert not os.path.exists(packed_file)
def prepare_project_scoped_env_not_attempted(dirname):
project = Project(dirname)
environ = minimal_environ(PROJECT_DIR=dirname)
result = prepare_without_interaction(project,
environ=environ,
mode=provide.PROVIDE_MODE_CHECK)
assert not result
# expected_env_path = os.path.join(dirname, "envs", "default")
bootstrap_env_path = os.path.join(dirname, "envs", "bootstrap-env")
for err in [
('missing requirement to run this project: ' +
'The project needs a Conda bootstrap environment containing all required packages.'
),
" '%s' doesn't look like it contains a Conda environment yet."
% bootstrap_env_path,
]:
assert err in result.errors
# unprepare should not have anything to do
status = unprepare(project, result)
assert status
assert status.errors == []
# todo: would be good to understand the different message got with a "normal" env
assert status.status_description == ("Success.")
def check(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ(BAR='bar')
result = prepare_without_interaction(project, environ=environ, command_name="blah")
assert not result
assert result.errors
assert "Command name 'blah' is not in" in result.errors[0]
def _commit_requirement_if_it_works(project,
env_var_or_class,
env_spec_name=None):
project.project_file.use_changes_without_saving()
# See if we can perform the download
result = prepare.prepare_without_interaction(project,
provide_whitelist=(
CondaEnvRequirement,
env_var_or_class,
),
env_spec_name=env_spec_name)
status = result.status_for(env_var_or_class)
if status is None:
# I _think_ this is currently impossible, but if it were possible,
# we'd need to below code and it's hard to prove it's impossible.
status = project.problems_status(
) # pragma: no cover # no way to cause right now?
# caller was supposed to expect env_var_or_class to still exist,
# unless project file got mangled
assert status is not None # pragma: no cover
if not status:
# reload from disk, discarding our changes because they did not work
project.project_file.load()
else:
# yay!
project.project_file.save()
return status
def prepare_some_env_var(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ(BAR='bar')
result = prepare_without_interaction(project, environ=environ)
assert not result
assert result.env_prefix is not None
assert dict(BAR='bar') == strip_environ(environ)
def prepare_project_locally(self,
project,
environ,
env_spec_name=None,
command_name=None,
command=None,
extra_command_args=None):
"""Prepare a project to run one of its commands.
"Locally" means a machine where development will go on,
contrasted with say a production deployment.
This method takes any needed actions such as creating
environments or starting services, without asking the user
for permission.
This method returns a result object. The result object has
a ``failed`` property. If the result is failed, the
``errors`` property has the errors. If the result is not
failed, the ``command_exec_info`` property has the stuff
you need to run the project's default command, and the
``environ`` property has the updated environment. The
passed-in ``environ`` is not modified in-place.
You can update your original environment with
``result.update_environ()`` if you like, but it's probably
a bad idea to modify ``os.environ`` in that way because
the calling app won't want to have the project
environment.
The ``environ`` should usually be kept between
preparations, starting out as ``os.environ`` but then
being modified by the user.
If the project has a non-empty ``problems`` attribute,
this function returns the project problems inside a failed
result. So ``project.problems`` does not need to be checked in
advance.
Args:
project (Project): from the ``load_project`` method
environ (dict): os.environ or the previously-prepared environ; not modified in-place
env_spec_name (str): the package set name to require, or None for default
command_name (str): which named command to choose from the project, None for default
command (ProjectCommand): a command object (alternative to command_name)
extra_command_args (list): extra args to include in the returned command argv
Returns:
a ``PrepareResult`` instance, which has a ``failed`` flag
"""
return prepare.prepare_without_interaction(
project=project,
environ=environ,
mode=provide.PROVIDE_MODE_DEVELOPMENT,
env_spec_name=env_spec_name,
command_name=command_name,
command=command,
extra_command_args=extra_command_args)
def prepare_some_env_var(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ(FOO='bar')
result = prepare_without_interaction(project, environ=environ)
assert result.errors == []
assert result
assert dict(FOO='bar', PROJECT_DIR=project.directory_path) == strip_environ(result.environ)
assert dict(FOO='bar') == strip_environ(environ)
def unprepare_empty(dirname):
project = Project(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert result.errors == []
assert result
status = unprepare(project, result)
assert status.errors == []
assert status
def prepare_empty(dirname):
project = Project(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert result.errors == []
assert result
assert dict(PROJECT_DIR=project.directory_path) == strip_environ(result.environ)
assert dict() == strip_environ(environ)
assert result.command_exec_info is None
def check_env_var_provider_prepare(dirname):
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(FOO='bar'))
assert result
status = unprepare(project, result)
assert status
assert status.status_description == 'Success.'
assert project.frontend.logs == ["Nothing to clean up for FOO.",
("Current environment is not in %s, no need to delete it." % dirname)]
def _prepare_printing_errors(project, environ=None, mode=provide.PROVIDE_MODE_DEVELOPMENT):
result = prepare_without_interaction(project, environ=environ, mode=mode)
for message in project.frontend.logs:
print(message)
for error in project.frontend.errors:
print(error, file=sys.stderr)
if not result:
assert result.errors == project.frontend.errors
return result
def prepare_project_scoped_env_with_packages(dirname):
project = Project(dirname)
environ = minimal_environ(PROJECT_DIR=dirname)
result = prepare_without_interaction(project,
environ=environ,
env_spec_name='bootstrap-env')
assert result
envs_dir = os.path.join(dirname, "envs")
env_name = 'bootstrap-env'
prefix = os.path.join(envs_dir, env_name)
installed = conda_api.installed(prefix)
assert 'bokeh' not in installed
deps = ['ipython', 'numpy', 'pip']
for pkg in deps:
assert pkg in installed
deps += ['bokeh']
# Preparing it again with new packages added should add those
project.project_file.set_value('packages', deps)
project.project_file.save()
environ = minimal_environ(PROJECT_DIR=dirname)
result = prepare_without_interaction(project, environ=environ)
assert result
prefix = result.environ[conda_env_var]
installed = conda_api.installed(prefix)
for pkg in deps:
assert pkg in installed
installed_pip = pip_api.installed(prefix)
assert 'flake8' in installed_pip
# Preparing it again with a bogus package should fail
deps = project.project_file.get_value('packages')
project.project_file.set_value(['packages'], deps + ['boguspackage'])
project.project_file.save()
environ = minimal_environ(PROJECT_DIR=dirname)
result = prepare_without_interaction(project, environ=environ)
assert not result
def unprepare_nothing(dirname):
project = Project(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert result.errors == []
assert result
status = unprepare(project, result, whitelist=[])
assert status.errors == []
assert status
assert status.status_description == 'Nothing to clean up.'
def unprepare_problems(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert not result
status = unprepare(project, result)
assert not status
assert status.status_description == 'Unable to load the project.'
assert status.errors == [('%s: variables section contains wrong value type 42, ' +
'should be dict or list of requirements') % project.project_file.basename]
def prepare_then_update_environ(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ(FOO='bar')
result = prepare_without_interaction(project, environ=environ)
assert result.errors == []
assert result
other = minimal_environ(BAR='baz')
result.update_environ(other)
assert dict(FOO='bar', BAR='baz', PROJECT_DIR=dirname) == strip_environ(other)
def check(dirname):
# create a command that isn't in the Project
project = project_no_dedicated_env(dirname)
command = ProjectCommand(
name="foo", attributes=dict(bokeh_app="foo.py", env_spec=project.default_env_spec_name))
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ, command=command)
assert result.errors == []
assert result
assert os.path.join(project.directory_path, 'foo.py') in result.command_exec_info.args
