def remove_download(project, prepare_result, env_var):
"""Remove file or directory referenced by ``env_var`` from file system and the project.
The returned ``Status`` will be an instance of ``SimpleStatus``. A False
status will have an ``errors`` property with a list of error
strings.
Args:
project (Project): the project
prepare_result (PrepareResult): result of a previous prepare
env_var (str): env var to store the local filename
Returns:
``Status`` instance
"""
failed = project.problems_status()
if failed is not None:
return failed
# Modify the project file _in memory only_, do not save
requirement = project.find_requirements(env_var, klass=DownloadRequirement)
if not requirement:
return SimpleStatus(
success=False,
description="Download requirement: {} not found.".format(env_var))
assert len(requirement) == 1 # duplicate env vars aren't allowed
requirement = requirement[0]
status = prepare.unprepare(project, prepare_result, whitelist=[env_var])
if status:
project.project_file.unset_value(['downloads', env_var])
project.project_file.use_changes_without_saving()
assert project.problems == []
project.project_file.save()
return status
def provide_download(dirname):
@gen.coroutine
def mock_downloader_run(self, loop):
class Res:
pass
res = Res()
res.code = 200
with open(os.path.join(dirname, 'data.csv'), 'w') as out:
out.write('data')
self._hash = '12345abcdef'
raise gen.Return(res)
monkeypatch.setattr(
"anaconda_project.internal.http_client.FileDownloader.run",
mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(
project, environ=minimal_environ(PROJECT_DIR=dirname))
assert hasattr(result, 'environ')
assert 'DATAFILE' in result.environ
filename = os.path.join(dirname, 'data.csv')
assert os.path.exists(filename)
project.frontend.reset()
status = unprepare(project, result)
assert project.frontend.logs == [
"Removed downloaded file %s." % filename,
("Current environment is not in %s, no need to delete it." %
dirname)
]
assert status.status_description == 'Success.'
assert status
assert not os.path.exists(filename)
def provide_download(dirname):
@gen.coroutine
def mock_downloader_run(self, loop):
raise Exception('error')
monkeypatch.setattr(
"anaconda_project.internal.http_client.FileDownloader.run",
mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(
project, environ=minimal_environ(PROJECT_DIR=dirname))
assert not result
assert (
'missing requirement to run this project: A downloaded file which is referenced by DATAFILE.'
) in result.errors
project.frontend.reset()
status = unprepare(project, result)
filename = os.path.join(dirname, 'data.csv')
assert project.frontend.logs == [
"No need to remove %s which wasn't downloaded." % filename,
("Current environment is not in %s, no need to delete it." %
dirname)
]
assert status.status_description == 'Success.'
def provide_download_of_zip(zipname, dirname):
with codecs.open(os.path.join(dirname, DEFAULT_PROJECT_FILENAME), 'w', 'utf-8') as f:
f.write(complete_project_file_content(ZIPPED_DATAFILE_CONTENT_CHECKSUM))
@gen.coroutine
def mock_downloader_run(self, loop):
class Res:
pass
res = Res()
res.code = 200
assert self._url.endswith(".zip")
assert self._filename.endswith(".zip")
shutil.copyfile(zipname, self._filename)
self._hash = '12345abcdef'
raise gen.Return(res)
monkeypatch.setattr("anaconda_project.internal.http_client.FileDownloader.run", mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(PROJECT_DIR=dirname))
assert hasattr(result, 'environ')
assert 'DATAFILE' in result.environ
assert os.path.isdir(os.path.join(dirname, 'data'))
assert os.path.isfile(os.path.join(dirname, 'data', 'foo'))
assert codecs.open(os.path.join(dirname, 'data', 'foo')).read() == 'hello\n'
project.frontend.reset()
status = unprepare(project, result)
filename = os.path.join(dirname, 'data')
assert project.frontend.logs == ["Removed downloaded file %s." % filename,
("Current environment is not in %s, no need to delete it." % dirname)]
assert status.status_description == "Success."
def provide_download(dirname):
@gen.coroutine
def mock_downloader_run(self, loop):
class Res:
pass
res = Res()
res.code = 200
with open(os.path.join(dirname, 'data.csv'), 'w') as out:
out.write('data')
self._hash = '12345abcdef'
raise gen.Return(res)
monkeypatch.setattr("anaconda_project.internal.http_client.FileDownloader.run", mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(PROJECT_DIR=dirname))
assert hasattr(result, 'environ')
assert 'DATAFILE' in result.environ
filename = os.path.join(dirname, 'data.csv')
assert os.path.exists(filename)
def mock_remove(path):
raise IOError("Not gonna remove this")
monkeypatch.setattr("os.remove", mock_remove)
project.frontend.reset()
status = unprepare(project, result)
assert project.frontend.logs == []
assert status.status_description == ('Failed to remove %s: Not gonna remove this.' % filename)
assert status.errors == []
assert not status
assert os.path.exists(filename)
monkeypatch.undo() # so os.remove isn't broken during directory cleanup
def prepare_project_scoped_env_not_attempted(dirname):
project = Project(dirname)
environ = minimal_environ(PROJECT_DIR=dirname)
result = prepare_without_interaction(project,
environ=environ,
mode=provide.PROVIDE_MODE_CHECK)
assert not result
# expected_env_path = os.path.join(dirname, "envs", "default")
bootstrap_env_path = os.path.join(dirname, "envs", "bootstrap-env")
for err in [
('missing requirement to run this project: ' +
'The project needs a Conda bootstrap environment containing all required packages.'
),
" '%s' doesn't look like it contains a Conda environment yet."
% bootstrap_env_path,
]:
assert err in result.errors
# unprepare should not have anything to do
status = unprepare(project, result)
assert status
assert status.errors == []
# todo: would be good to understand the different message got with a "normal" env
assert status.status_description == ("Success.")
def unprepare_empty(dirname):
project = Project(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert result.errors == []
assert result
status = unprepare(project, result)
assert status.errors == []
assert status
def check_env_var_provider_prepare(dirname):
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(FOO='bar'))
assert result
status = unprepare(project, result)
assert status
assert status.status_description == 'Success.'
assert project.frontend.logs == ["Nothing to clean up for FOO.",
("Current environment is not in %s, no need to delete it." % dirname)]
def unprepare_problems(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert not result
status = unprepare(project, result)
assert not status
assert status.status_description == 'Unable to load the project.'
assert status.errors == [('%s: variables section contains wrong value type 42, ' +
'should be dict or list of requirements') % project.project_file.basename]
def unprepare_nothing(dirname):
project = Project(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert result.errors == []
assert result
status = unprepare(project, result, whitelist=[])
assert status.errors == []
assert status
assert status.status_description == 'Nothing to clean up.'
def start_local_redis(dirname):
project = project_no_dedicated_env(dirname)
result = _prepare_printing_errors(project, environ=minimal_environ())
assert result
assert 'REDIS_URL' in result.environ
local_state_file = LocalStateFile.load_for_directory(dirname)
state = local_state_file.get_service_run_state("REDIS_URL")
assert 'port' in state
port = state['port']
assert dict(REDIS_URL=("redis://localhost:" + str(port)),
PROJECT_DIR=project.directory_path) == strip_environ(
result.environ)
assert len(can_connect_args_list) >= 2
pidfile = os.path.join(dirname, "services/REDIS_URL/redis.pid")
logfile = os.path.join(dirname, "services/REDIS_URL/redis.log")
assert os.path.exists(pidfile)
assert os.path.exists(logfile)
assert real_can_connect_to_socket(host='localhost', port=port)
# be sure we generate the config html that would use the old one
requirement = _redis_requirement()
status = requirement.check_status(result.environ, local_state_file,
'default', UserConfigOverrides())
# now try again, and we should re-use the exact same server
pidfile_mtime = os.path.getmtime(pidfile)
with codecs.open(pidfile, 'r', 'utf-8') as file:
pidfile_content = file.read()
result2 = _prepare_printing_errors(project, environ=minimal_environ())
assert result2
# port should be the same, and set in the environment
assert dict(REDIS_URL=("redis://localhost:" + str(port)),
PROJECT_DIR=project.directory_path) == strip_environ(
result2.environ)
# no new pid file
assert pidfile_mtime == os.path.getmtime(pidfile)
with codecs.open(pidfile, 'r', 'utf-8') as file:
pidfile_content2 = file.read()
assert pidfile_content == pidfile_content2
# now clean it up
status = unprepare(project, result2)
assert status
assert not os.path.exists(pidfile)
assert not real_can_connect_to_socket(host='localhost', port=port)
local_state_file.load()
assert dict() == local_state_file.get_service_run_state("REDIS_URL")
def start_local_redis(dirname):
project = project_no_dedicated_env(dirname)
result = _prepare_printing_errors(project, environ=minimal_environ())
assert result
# now clean it up, but arrange for that to fail
local_state_file = LocalStateFile.load_for_directory(dirname)
local_state_file.set_service_run_state('REDIS_URL', {'shutdown_commands': [['false']]})
local_state_file.save()
status = unprepare(project, result)
assert not status
assert status.status_description == 'Shutdown commands failed for REDIS_URL.'
def remove_service(project, prepare_result, variable_name):
"""Remove a service to anaconda-project.yml.
Returns a ``Status`` instance which evaluates to True on
success and has an ``errors`` property (with a list of error
strings) on failure.
Args:
project (Project): the project
prepare_result (PrepareResult): result of a previous prepare
variable_name (str): environment variable name for the service requirement
Returns:
``Status`` instance
"""
failed = project.problems_status()
if failed is not None:
return failed
requirements = [
req for req in project.find_requirements(klass=ServiceRequirement)
if req.service_type == variable_name or req.env_var == variable_name
]
if not requirements:
return SimpleStatus(
success=False,
description="Service '{}' not found in the project file.".format(
variable_name))
if len(requirements) > 1:
return SimpleStatus(
success=False,
description=(
"Conflicting results, found {} matches, use list-services"
" to identify which service you want to remove").format(
len(requirements)))
env_var = requirements[0].env_var
status = prepare.unprepare(project, prepare_result, whitelist=[env_var])
if not status:
return status
project.project_file.unset_value(['services', env_var])
project.project_file.use_changes_without_saving()
assert project.problems == []
project.project_file.save()
return SimpleStatus(
success=True,
description="Removed service '{}' from the project file.".format(
variable_name))
def prepare_project_scoped_env(dirname):
project = Project(dirname)
fake_old_path = "foo" + os.pathsep + "bar"
environ = dict(PROJECT_DIR=dirname, PATH=fake_old_path)
result = prepare_without_interaction(project, environ=environ)
assert result
expected_env = os.path.join(dirname, "envs", "bootstrap-env")
if platform.system() == 'Windows':
expected_new_path = expected_env + os.pathsep + os.path.join(
expected_env, script_dir) + os.pathsep + os.path.join(
expected_env, "Library",
"bin") + os.pathsep + "foo" + os.pathsep + "bar"
else:
expected_new_path = os.path.join(
expected_env,
script_dir) + os.pathsep + "foo" + os.pathsep + "bar"
expected = dict(PROJECT_DIR=project.directory_path,
PATH=expected_new_path,
BOOTSTRAP_ENV_PREFIX=expected_env)
conda_api.environ_set_prefix(expected, expected_env)
expected == result.environ
assert os.path.exists(os.path.join(expected_env, "conda-meta"))
conda_meta_mtime = os.path.getmtime(
os.path.join(expected_env, "conda-meta"))
# bare minimum bootstrap-env env shouldn't include these
# (contrast with the test later where we list them in
# requirements)
installed = conda_api.installed(expected_env)
assert 'ipython' not in installed
assert 'numpy' not in installed
# Prepare it again should no-op (use the already-existing environment)
environ = dict(PROJECT_DIR=dirname, PATH=fake_old_path)
result = prepare_without_interaction(project, environ=environ)
assert result
expected = dict(PROJECT_DIR=project.directory_path,
PATH=expected_new_path)
conda_api.environ_set_prefix(expected, expected_env)
assert conda_meta_mtime == os.path.getmtime(
os.path.join(expected_env, "conda-meta"))
# Now unprepare
status = unprepare(project, result)
assert status
# todo: this differs from standard CondaEnvProvider
assert status.status_description == 'Success.'
assert status.errors == []
assert not os.path.exists(expected_env)
def clean(project, prepare_result):
"""Blow away auto-provided state for the project.
This should not remove any potential "user data" such as
anaconda-project-local.yml.
This includes a call to ``anaconda_project.prepare.unprepare``
but also removes the entire services/ and envs/ directories
even if they contain leftovers that we didn't prepare in the
most recent prepare() call.
Args:
project (Project): the project instance
prepare_result (PrepareResult): result of a previous prepare
Returns:
a ``Status`` instance
"""
status = prepare.unprepare(project, prepare_result)
logs = status.logs
errors = status.errors
if status:
logs = logs + [status.status_description]
else:
errors = errors + [status.status_description]
# we also nuke any "debris" from non-current choices, like old
# environments or services
def cleanup_dir(dirname):
if os.path.isdir(dirname):
logs.append("Removing %s." % dirname)
try:
shutil.rmtree(dirname)
except Exception as e:
errors.append("Error removing %s: %s." % (dirname, str(e)))
cleanup_dir(os.path.join(project.directory_path, "services"))
cleanup_dir(os.path.join(project.directory_path, "envs"))
if status and len(errors) == 0:
return SimpleStatus(success=True,
description="Cleaned.",
logs=logs,
errors=errors)
else:
return SimpleStatus(success=False,
description="Failed to clean everything up.",
logs=logs,
errors=errors)
def prepare_project_scoped_env_fails(dirname):
project = Project(dirname)
environ = minimal_environ(PROJECT_DIR=dirname)
result = prepare_without_interaction(project, environ=environ)
assert not result
assert 'CONDA_DEFAULT_ENV' not in result.environ
assert 'CONDA_ENV_PATH' not in result.environ
# unprepare should not have anything to do
status = unprepare(project, result)
assert status
assert status.errors == []
assert status.status_description == "Nothing to clean up for environment 'default'."
def unprepare(self, project, prepare_result, whitelist=None):
"""Attempt to clean up project-scoped resources allocated by prepare().
This will retain any user configuration choices about how to
provide requirements, but it stops project-scoped services.
Global system services or other services potentially shared
among projects will not be stopped.
To stop a single service, use ``whitelist=["SERVICE_VARIABLE"]``.
Args:
project (Project): the project
prepare_result (PrepareResult): result from the previous prepare
whitelist (iterable of str or type): ONLY call shutdown commands for the listed env vars' requirements
"""
return prepare.unprepare(project=project, prepare_result=prepare_result, whitelist=whitelist)
def prepare_project_scoped_env_not_attempted(dirname):
project = Project(dirname)
environ = minimal_environ(PROJECT_DIR=dirname)
result = prepare_without_interaction(project,
environ=environ,
mode=provide.PROVIDE_MODE_CHECK)
assert not result
expected_env_path = os.path.join(dirname, "envs", "default")
assert [(
'missing requirement to run this project: ' +
'The project needs a Conda environment containing all required packages.'
),
" '%s' doesn't look like it contains a Conda environment yet."
% expected_env_path] == result.errors
# unprepare should not have anything to do
status = unprepare(project, result)
assert status
assert status.errors == []
assert status.status_description == (
"Nothing to clean up for environment 'default'.")
def start_local_redis(dirname):
project = project_no_dedicated_env(dirname)
result = _prepare_printing_errors(project, environ=minimal_environ())
assert result
local_state_file = LocalStateFile.load_for_directory(dirname)
state = local_state_file.get_service_run_state('REDIS_URL')
assert 'port' in state
port = state['port']
assert dict(REDIS_URL=("redis://localhost:" + str(port)),
PROJECT_DIR=project.directory_path) == strip_environ(
result.environ)
assert len(can_connect_args_list) >= 2
servicedir = os.path.join(dirname, "services")
redisdir = os.path.join(servicedir, "REDIS_URL")
pidfile = os.path.join(redisdir, "redis.pid")
logfile = os.path.join(redisdir, "redis.log")
assert os.path.exists(pidfile)
assert os.path.exists(logfile)
assert real_can_connect_to_socket(host='localhost', port=port)
# now clean it up
status = unprepare(project, result)
assert status
assert not os.path.exists(pidfile)
assert not os.path.exists(logfile)
assert not os.path.exists(redisdir)
assert not os.path.exists(servicedir)
assert not real_can_connect_to_socket(host='localhost', port=port)
local_state_file.load()
assert dict() == local_state_file.get_service_run_state("REDIS_URL")
def prepare_project_scoped_env(dirname):
project = Project(dirname)
environ = minimal_environ(PROJECT_DIR=dirname)
result = prepare_without_interaction(project, environ=environ)
assert result
expected_env = os.path.join(dirname, "envs", "default")
# Now unprepare
def mock_rmtree(path):
raise IOError("I will never rm the tree!")
monkeypatch.setattr('shutil.rmtree', mock_rmtree)
status = unprepare(project, result)
assert status.status_description == (
'Failed to remove environment files in %s: I will never rm the tree!.'
% (expected_env))
assert not status
assert os.path.exists(expected_env)
# so we can rmtree our tmp directory
monkeypatch.undo()
