def provide_download_of_zip(zipname, dirname):
with codecs.open(os.path.join(dirname, DEFAULT_PROJECT_FILENAME), 'w', 'utf-8') as f:
f.write(complete_project_file_content(ZIPPED_DATAFILE_CONTENT_CHECKSUM))
@gen.coroutine
def mock_downloader_run(self, loop):
class Res:
pass
res = Res()
res.code = 200
assert self._url.endswith(".zip")
assert self._filename.endswith(".zip")
shutil.copyfile(zipname, self._filename)
self._hash = '12345abcdef'
raise gen.Return(res)
monkeypatch.setattr("conda_kapsel.internal.http_client.FileDownloader.run", mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(PROJECT_DIR=dirname))
assert hasattr(result, 'environ')
assert 'DATAFILE' in result.environ
assert os.path.isdir(os.path.join(dirname, 'data'))
assert os.path.isfile(os.path.join(dirname, 'data', 'foo'))
assert codecs.open(os.path.join(dirname, 'data', 'foo')).read() == 'hello\n'
status = unprepare(project, result)
filename = os.path.join(dirname, 'data')
assert status.logs == ["Removed downloaded file %s." % filename,
("Current environment is not in %s, no need to delete it." % dirname)]
assert status.status_description == "Success."
def unprepare_empty(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert result
status = unprepare(project, result)
assert status
def remove_download(project, prepare_result, env_var):
"""Remove file or directory referenced by ``env_var`` from file system and the project.
The returned ``Status`` will be an instance of ``SimpleStatus``. A False
status will have an ``errors`` property with a list of error
strings.
Args:
project (Project): the project
prepare_result (PrepareResult): result of a previous prepare
env_var (str): env var to store the local filename
Returns:
``Status`` instance
"""
failed = project.problems_status()
if failed is not None:
return failed
# Modify the project file _in memory only_, do not save
requirement = project.find_requirements(env_var, klass=DownloadRequirement)
if not requirement:
return SimpleStatus(
success=False,
description="Download requirement: {} not found.".format(env_var))
assert len(requirement) == 1 # duplicate env vars aren't allowed
requirement = requirement[0]
status = prepare.unprepare(project, prepare_result, whitelist=[env_var])
if status:
project.project_file.unset_value(['downloads', env_var])
project.project_file.use_changes_without_saving()
assert project.problems == []
project.project_file.save()
return status
def unprepare_empty(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert result
status = unprepare(project, result)
assert status
def provide_download(dirname):
@gen.coroutine
def mock_downloader_run(self, loop):
class Res:
pass
res = Res()
res.code = 200
with open(os.path.join(dirname, 'data.csv'), 'w') as out:
out.write('data')
self._hash = '12345abcdef'
raise gen.Return(res)
monkeypatch.setattr("conda_kapsel.internal.http_client.FileDownloader.run", mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(PROJECT_DIR=dirname))
assert hasattr(result, 'environ')
assert 'DATAFILE' in result.environ
filename = os.path.join(dirname, 'data.csv')
assert os.path.exists(filename)
def mock_remove(path):
raise IOError("Not gonna remove this")
monkeypatch.setattr("os.remove", mock_remove)
status = unprepare(project, result)
assert status.logs == []
assert status.status_description == ('Failed to remove %s: Not gonna remove this.' % filename)
assert status.errors == []
assert not status
assert os.path.exists(filename)
monkeypatch.undo() # so os.remove isn't broken during directory cleanup
def remove_download(project, prepare_result, env_var):
"""Remove file or directory referenced by ``env_var`` from file system and the project.
The returned ``Status`` will be an instance of ``SimpleStatus``. A False
status will have an ``errors`` property with a list of error
strings.
Args:
project (Project): the project
prepare_result (PrepareResult): result of a previous prepare
env_var (str): env var to store the local filename
Returns:
``Status`` instance
"""
failed = project.problems_status()
if failed is not None:
return failed
# Modify the project file _in memory only_, do not save
requirement = project.find_requirements(env_var, klass=DownloadRequirement)
if not requirement:
return SimpleStatus(success=False, description="Download requirement: {} not found.".format(env_var))
assert len(requirement) == 1 # duplicate env vars aren't allowed
requirement = requirement[0]
status = prepare.unprepare(project, prepare_result, whitelist=[env_var])
if status:
project.project_file.unset_value(['downloads', env_var])
project.project_file.use_changes_without_saving()
assert project.problems == []
project.project_file.save()
return status
def provide_download(dirname):
@gen.coroutine
def mock_downloader_run(self, loop):
class Res:
pass
res = Res()
res.code = 200
with open(os.path.join(dirname, 'data.csv'), 'w') as out:
out.write('data')
self._hash = '12345abcdef'
raise gen.Return(res)
monkeypatch.setattr("conda_kapsel.internal.http_client.FileDownloader.run", mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(PROJECT_DIR=dirname))
assert hasattr(result, 'environ')
assert 'DATAFILE' in result.environ
filename = os.path.join(dirname, 'data.csv')
assert os.path.exists(filename)
status = unprepare(project, result)
assert status.logs == ["Removed downloaded file %s." % filename,
("Current environment is not in %s, no need to delete it." % dirname)]
assert status.status_description == 'Success.'
assert status
assert not os.path.exists(filename)
def start_local_redis(dirname):
project = project_no_dedicated_env(dirname)
result = _prepare_printing_errors(project, environ=minimal_environ())
assert result
assert 'REDIS_URL' in result.environ
local_state_file = LocalStateFile.load_for_directory(dirname)
state = local_state_file.get_service_run_state("REDIS_URL")
assert 'port' in state
port = state['port']
assert dict(REDIS_URL=("redis://localhost:" + str(port)),
PROJECT_DIR=project.directory_path) == strip_environ(
result.environ)
assert len(can_connect_args_list) >= 2
pidfile = os.path.join(dirname, "services/REDIS_URL/redis.pid")
logfile = os.path.join(dirname, "services/REDIS_URL/redis.log")
assert os.path.exists(pidfile)
assert os.path.exists(logfile)
assert real_can_connect_to_socket(host='localhost', port=port)
# be sure we generate the config html that would use the old one
requirement = _redis_requirement()
status = requirement.check_status(result.environ, local_state_file,
'default', UserConfigOverrides())
html = RedisProvider().config_html(requirement, result.environ,
local_state_file,
UserConfigOverrides(), status)
assert 'Use the redis-server we started earlier' in html
# now try again, and we should re-use the exact same server
pidfile_mtime = os.path.getmtime(pidfile)
with codecs.open(pidfile, 'r', 'utf-8') as file:
pidfile_content = file.read()
result2 = _prepare_printing_errors(project, environ=minimal_environ())
assert result2
# port should be the same, and set in the environment
assert dict(REDIS_URL=("redis://localhost:" + str(port)),
PROJECT_DIR=project.directory_path) == strip_environ(
result2.environ)
# no new pid file
assert pidfile_mtime == os.path.getmtime(pidfile)
with codecs.open(pidfile, 'r', 'utf-8') as file:
pidfile_content2 = file.read()
assert pidfile_content == pidfile_content2
# now clean it up
status = unprepare(project, result2)
assert status
assert not os.path.exists(pidfile)
assert not real_can_connect_to_socket(host='localhost', port=port)
local_state_file.load()
assert dict() == local_state_file.get_service_run_state("REDIS_URL")
def unprepare_nothing(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert result
status = unprepare(project, result, whitelist=[])
assert status
assert status.status_description == 'Nothing to clean up.'
def unprepare_nothing(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert result
status = unprepare(project, result, whitelist=[])
assert status
assert status.status_description == 'Nothing to clean up.'
def check_env_var_provider_prepare(dirname):
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(FOO='bar'))
assert result
status = unprepare(project, result)
assert status
assert status.status_description == 'Success.'
assert status.logs == ["Nothing to clean up for FOO.",
("Current environment is not in %s, no need to delete it." % dirname)]
def unprepare_problems(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert not result
status = unprepare(project, result)
assert not status
assert status.status_description == 'Unable to load the project.'
assert status.errors == ['variables section contains wrong value type 42, ' +
'should be dict or list of requirements']
def unprepare_problems(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert not result
status = unprepare(project, result)
assert not status
assert status.status_description == 'Unable to load the project.'
assert status.errors == ['variables section contains wrong value type 42, ' +
'should be dict or list of requirements']
def start_local_redis(dirname):
project = project_no_dedicated_env(dirname)
result = _prepare_printing_errors(project, environ=minimal_environ())
assert result
assert 'REDIS_URL' in result.environ
local_state_file = LocalStateFile.load_for_directory(dirname)
state = local_state_file.get_service_run_state("REDIS_URL")
assert 'port' in state
port = state['port']
assert dict(REDIS_URL=("redis://localhost:" + str(port)),
PROJECT_DIR=project.directory_path) == strip_environ(result.environ)
assert len(can_connect_args_list) >= 2
pidfile = os.path.join(dirname, "services/REDIS_URL/redis.pid")
logfile = os.path.join(dirname, "services/REDIS_URL/redis.log")
assert os.path.exists(pidfile)
assert os.path.exists(logfile)
assert real_can_connect_to_socket(host='localhost', port=port)
# be sure we generate the config html that would use the old one
requirement = _redis_requirement()
status = requirement.check_status(result.environ, local_state_file, 'default', UserConfigOverrides())
html = RedisProvider().config_html(requirement, result.environ, local_state_file, UserConfigOverrides(), status)
assert 'Use the redis-server we started earlier' in html
# now try again, and we should re-use the exact same server
pidfile_mtime = os.path.getmtime(pidfile)
with codecs.open(pidfile, 'r', 'utf-8') as file:
pidfile_content = file.read()
result2 = _prepare_printing_errors(project, environ=minimal_environ())
assert result2
# port should be the same, and set in the environment
assert dict(REDIS_URL=("redis://localhost:" + str(port)),
PROJECT_DIR=project.directory_path) == strip_environ(result2.environ)
# no new pid file
assert pidfile_mtime == os.path.getmtime(pidfile)
with codecs.open(pidfile, 'r', 'utf-8') as file:
pidfile_content2 = file.read()
assert pidfile_content == pidfile_content2
# now clean it up
status = unprepare(project, result2)
assert status
assert not os.path.exists(pidfile)
assert not real_can_connect_to_socket(host='localhost', port=port)
local_state_file.load()
assert dict() == local_state_file.get_service_run_state("REDIS_URL")
def start_local_redis(dirname):
project = project_no_dedicated_env(dirname)
result = _prepare_printing_errors(project, environ=minimal_environ())
assert result
# now clean it up, but arrange for that to fail
local_state_file = LocalStateFile.load_for_directory(dirname)
local_state_file.set_service_run_state('REDIS_URL', {'shutdown_commands': [['false']]})
local_state_file.save()
status = unprepare(project, result)
assert not status
assert status.status_description == 'Shutdown commands failed for REDIS_URL.'
def remove_service(project, prepare_result, variable_name):
"""Remove a service to kapsel.yml.
Returns a ``Status`` instance which evaluates to True on
success and has an ``errors`` property (with a list of error
strings) on failure.
Args:
project (Project): the project
prepare_result (PrepareResult): result of a previous prepare
variable_name (str): environment variable name for the service requirement
Returns:
``Status`` instance
"""
failed = project.problems_status()
if failed is not None:
return failed
requirements = [
req for req in project.find_requirements(klass=ServiceRequirement)
if req.service_type == variable_name or req.env_var == variable_name
]
if not requirements:
return SimpleStatus(
success=False,
description="Service '{}' not found in the project file.".format(
variable_name))
if len(requirements) > 1:
return SimpleStatus(
success=False,
description=(
"Conflicting results, found {} matches, use list-services"
" to identify which service you want to remove").format(
len(requirements)))
env_var = requirements[0].env_var
status = prepare.unprepare(project, prepare_result, whitelist=[env_var])
if not status:
return status
project.project_file.unset_value(['services', env_var])
project.project_file.use_changes_without_saving()
assert project.problems == []
project.project_file.save()
return SimpleStatus(
success=True,
description="Removed service '{}' from the project file.".format(
variable_name))
def start_local_redis(dirname):
project = project_no_dedicated_env(dirname)
result = _prepare_printing_errors(project, environ=minimal_environ())
assert result
# now clean it up, but arrange for that to fail
local_state_file = LocalStateFile.load_for_directory(dirname)
local_state_file.set_service_run_state(
'REDIS_URL', {'shutdown_commands': [['false']]})
local_state_file.save()
status = unprepare(project, result)
assert not status
assert status.status_description == 'Shutdown commands failed for REDIS_URL.'
def prepare_project_scoped_env_fails(dirname):
project = Project(dirname)
environ = minimal_environ(PROJECT_DIR=dirname)
result = prepare_without_interaction(project, environ=environ)
assert not result
assert 'CONDA_DEFAULT_ENV' not in result.environ
assert 'CONDA_ENV_PATH' not in result.environ
# unprepare should not have anything to do
status = unprepare(project, result)
assert status
assert status.errors == []
assert status.status_description == "Nothing to clean up for environment 'default'."
def clean(project, prepare_result):
"""Blow away auto-provided state for the project.
This should not remove any potential "user data" such as
kapsel-local.yml.
This includes a call to ``conda_kapsel.prepare.unprepare``
but also removes the entire services/ and envs/ directories
even if they contain leftovers that we didn't prepare in the
most recent prepare() call.
Args:
project (Project): the project instance
prepare_result (PrepareResult): result of a previous prepare
Returns:
a ``Status`` instance
"""
status = prepare.unprepare(project, prepare_result)
logs = status.logs
errors = status.errors
if status:
logs = logs + [status.status_description]
else:
errors = errors + [status.status_description]
# we also nuke any "debris" from non-current choices, like old
# environments or services
def cleanup_dir(dirname):
if os.path.isdir(dirname):
logs.append("Removing %s." % dirname)
try:
shutil.rmtree(dirname)
except Exception as e:
errors.append("Error removing %s: %s." % (dirname, str(e)))
cleanup_dir(os.path.join(project.directory_path, "services"))
cleanup_dir(os.path.join(project.directory_path, "envs"))
if status and len(errors) == 0:
return SimpleStatus(success=True,
description="Cleaned.",
logs=logs,
errors=errors)
else:
return SimpleStatus(success=False,
description="Failed to clean everything up.",
logs=logs,
errors=errors)
def prepare_project_scoped_env_not_attempted(dirname):
project = Project(dirname)
environ = minimal_environ(PROJECT_DIR=dirname)
result = prepare_without_interaction(project, environ=environ, mode=provide.PROVIDE_MODE_CHECK)
assert not result
expected_env_path = os.path.join(dirname, "envs", "default")
assert [
('missing requirement to run this project: ' +
'The project needs a Conda environment containing all required packages.'),
" '%s' doesn't look like it contains a Conda environment yet." % expected_env_path
] == result.errors
# unprepare should not have anything to do
status = unprepare(project, result)
assert status
assert status.errors == []
assert status.status_description == ("Nothing to clean up for environment 'default'.")
def provide_download(dirname):
@gen.coroutine
def mock_downloader_run(self, loop):
raise Exception('error')
monkeypatch.setattr("conda_kapsel.internal.http_client.FileDownloader.run", mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(PROJECT_DIR=dirname))
assert not result
assert ('missing requirement to run this project: A downloaded file which is referenced by DATAFILE.'
) in result.errors
status = unprepare(project, result)
filename = os.path.join(dirname, 'data.csv')
assert status.logs == ["No need to remove %s which wasn't downloaded." % filename,
("Current environment is not in %s, no need to delete it." % dirname)]
assert status.status_description == 'Success.'
def clean(project, prepare_result):
"""Blow away auto-provided state for the project.
This should not remove any potential "user data" such as
kapsel-local.yml.
This includes a call to ``conda_kapsel.prepare.unprepare``
but also removes the entire services/ and envs/ directories
even if they contain leftovers that we didn't prepare in the
most recent prepare() call.
Args:
project (Project): the project instance
prepare_result (PrepareResult): result of a previous prepare
Returns:
a ``Status`` instance
"""
status = prepare.unprepare(project, prepare_result)
logs = status.logs
errors = status.errors
if status:
logs = logs + [status.status_description]
else:
errors = errors + [status.status_description]
# we also nuke any "debris" from non-current choices, like old
# environments or services
def cleanup_dir(dirname):
if os.path.isdir(dirname):
logs.append("Removing %s." % dirname)
try:
shutil.rmtree(dirname)
except Exception as e:
errors.append("Error removing %s: %s." % (dirname, str(e)))
cleanup_dir(os.path.join(project.directory_path, "services"))
cleanup_dir(os.path.join(project.directory_path, "envs"))
if status and len(errors) == 0:
return SimpleStatus(success=True, description="Cleaned.", logs=logs, errors=errors)
else:
return SimpleStatus(success=False, description="Failed to clean everything up.", logs=logs, errors=errors)
def remove_service(project, prepare_result, variable_name):
"""Remove a service to kapsel.yml.
Returns a ``Status`` instance which evaluates to True on
success and has an ``errors`` property (with a list of error
strings) on failure.
Args:
project (Project): the project
prepare_result (PrepareResult): result of a previous prepare
variable_name (str): environment variable name for the service requirement
Returns:
``Status`` instance
"""
failed = project.problems_status()
if failed is not None:
return failed
requirements = [req
for req in project.find_requirements(klass=ServiceRequirement)
if req.service_type == variable_name or req.env_var == variable_name]
if not requirements:
return SimpleStatus(success=False,
description="Service '{}' not found in the project file.".format(variable_name))
if len(requirements) > 1:
return SimpleStatus(success=False,
description=("Conflicting results, found {} matches, use list-services"
" to identify which service you want to remove").format(len(requirements)))
env_var = requirements[0].env_var
status = prepare.unprepare(project, prepare_result, whitelist=[env_var])
if not status:
return status
project.project_file.unset_value(['services', env_var])
project.project_file.use_changes_without_saving()
assert project.problems == []
project.project_file.save()
return SimpleStatus(success=True, description="Removed service '{}' from the project file.".format(variable_name))
def unprepare(self, project, prepare_result, whitelist=None):
"""Attempt to clean up project-scoped resources allocated by prepare().
This will retain any user configuration choices about how to
provide requirements, but it stops project-scoped services.
Global system services or other services potentially shared
among projects will not be stopped.
To stop a single service, use ``whitelist=["SERVICE_VARIABLE"]``.
Args:
project (Project): the project
prepare_result (PrepareResult): result from the previous prepare
whitelist (iterable of str or type): ONLY call shutdown commands for the listed env vars' requirements
"""
return prepare.unprepare(project=project,
prepare_result=prepare_result,
whitelist=whitelist)
def prepare_project_scoped_env(dirname):
project = Project(dirname)
fake_old_path = "foo" + os.pathsep + "bar"
environ = dict(PROJECT_DIR=dirname, PATH=fake_old_path)
result = prepare_without_interaction(project, environ=environ)
assert result
expected_env = os.path.join(dirname, "envs", "default")
if platform.system() == 'Windows':
expected_new_path = expected_env + os.pathsep + os.path.join(
expected_env, script_dir) + os.pathsep + os.path.join(expected_env, "Library",
"bin") + os.pathsep + "foo" + os.pathsep + "bar"
else:
expected_new_path = os.path.join(expected_env, script_dir) + os.pathsep + "foo" + os.pathsep + "bar"
expected = dict(PROJECT_DIR=project.directory_path, PATH=expected_new_path)
conda_api.environ_set_prefix(expected, expected_env)
expected == result.environ
assert os.path.exists(os.path.join(expected_env, "conda-meta"))
conda_meta_mtime = os.path.getmtime(os.path.join(expected_env, "conda-meta"))
# bare minimum default env shouldn't include these
# (contrast with the test later where we list them in
# requirements)
installed = conda_api.installed(expected_env)
assert 'ipython' not in installed
assert 'numpy' not in installed
# Prepare it again should no-op (use the already-existing environment)
environ = dict(PROJECT_DIR=dirname, PATH=fake_old_path)
result = prepare_without_interaction(project, environ=environ)
assert result
expected = dict(PROJECT_DIR=project.directory_path, PATH=expected_new_path)
conda_api.environ_set_prefix(expected, expected_env)
assert expected == result.environ
assert conda_meta_mtime == os.path.getmtime(os.path.join(expected_env, "conda-meta"))
# Now unprepare
status = unprepare(project, result)
assert status
assert status.status_description == ('Deleted environment files in %s.' % (expected_env))
assert status.errors == []
assert not os.path.exists(expected_env)
def prepare_project_scoped_env(dirname):
project = Project(dirname)
environ = minimal_environ(PROJECT_DIR=dirname)
result = prepare_without_interaction(project, environ=environ)
assert result
expected_env = os.path.join(dirname, "envs", "default")
# Now unprepare
def mock_rmtree(path):
raise IOError("I will never rm the tree!")
monkeypatch.setattr('shutil.rmtree', mock_rmtree)
status = unprepare(project, result)
assert status.status_description == ('Failed to remove environment files in %s: I will never rm the tree!.' %
(expected_env))
assert not status
assert os.path.exists(expected_env)
# so we can rmtree our tmp directory
monkeypatch.undo()
def start_local_redis(dirname):
project = project_no_dedicated_env(dirname)
result = _prepare_printing_errors(project, environ=minimal_environ())
assert result
local_state_file = LocalStateFile.load_for_directory(dirname)
state = local_state_file.get_service_run_state('REDIS_URL')
assert 'port' in state
port = state['port']
assert dict(REDIS_URL=("redis://localhost:" + str(port)),
PROJECT_DIR=project.directory_path) == strip_environ(
result.environ)
assert len(can_connect_args_list) >= 2
servicedir = os.path.join(dirname, "services")
redisdir = os.path.join(servicedir, "REDIS_URL")
pidfile = os.path.join(redisdir, "redis.pid")
logfile = os.path.join(redisdir, "redis.log")
assert os.path.exists(pidfile)
assert os.path.exists(logfile)
assert real_can_connect_to_socket(host='localhost', port=port)
# now clean it up
status = unprepare(project, result)
assert status
assert not os.path.exists(pidfile)
assert not os.path.exists(logfile)
assert not os.path.exists(redisdir)
assert not os.path.exists(servicedir)
assert not real_can_connect_to_socket(host='localhost', port=port)
local_state_file.load()
assert dict() == local_state_file.get_service_run_state("REDIS_URL")
def start_local_redis(dirname):
project = project_no_dedicated_env(dirname)
result = _prepare_printing_errors(project, environ=minimal_environ())
assert result
local_state_file = LocalStateFile.load_for_directory(dirname)
state = local_state_file.get_service_run_state('REDIS_URL')
assert 'port' in state
port = state['port']
assert dict(REDIS_URL=("redis://localhost:" + str(port)),
PROJECT_DIR=project.directory_path) == strip_environ(result.environ)
assert len(can_connect_args_list) >= 2
servicedir = os.path.join(dirname, "services")
redisdir = os.path.join(servicedir, "REDIS_URL")
pidfile = os.path.join(redisdir, "redis.pid")
logfile = os.path.join(redisdir, "redis.log")
assert os.path.exists(pidfile)
assert os.path.exists(logfile)
assert real_can_connect_to_socket(host='localhost', port=port)
# now clean it up
status = unprepare(project, result)
assert status
assert not os.path.exists(pidfile)
assert not os.path.exists(logfile)
assert not os.path.exists(redisdir)
assert not os.path.exists(servicedir)
assert not real_can_connect_to_socket(host='localhost', port=port)
local_state_file.load()
assert dict() == local_state_file.get_service_run_state("REDIS_URL")
