def prepare_with_browser(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
result = prepare_with_browser_ui(project, environ=environ, keep_going_until_success=False, io_loop=io_loop)
assert result
assert dict(FOO_PASSWORD='******', PROJECT_DIR=project.directory_path) == strip_environ(result.environ)
assert dict() == strip_environ(environ)
# wait for the results of the POST to come back,
# awesome hack-tacular
while 'post_fill_in_password' not in http_results:
io_loop.call_later(0.01, lambda: io_loop.stop())
io_loop.start()
assert 'get_click_submit' in http_results
assert 'post_click_submit' in http_results
assert 'post_fill_in_password' in http_results
assert 200 == http_results['get_click_submit'].code
assert 200 == http_results['post_click_submit'].code
assert 200 == http_results['post_fill_in_password'].code
final_done_html = str(http_results['post_fill_in_password'].body)
assert "Done!" in final_done_html
assert "Environment variable FOO_PASSWORD is set." in final_done_html
local_state_file = LocalStateFile.load_for_directory(project.directory_path)
assert local_state_file.get_value(['variables', 'FOO_PASSWORD']) is None
def provide_download_of_zip(dirname):
with codecs.open(os.path.join(dirname, DEFAULT_PROJECT_FILENAME), 'w', 'utf-8') as f:
f.write(complete_project_file_content(ZIPPED_DATAFILE_CONTENT))
@gen.coroutine
def mock_downloader_run(self, loop):
class Res:
pass
res = Res()
res.code = 200
assert self._url.endswith(".zip")
assert self._filename.endswith(".zip")
with codecs.open(self._filename, 'w', 'utf-8') as f:
f.write("This is not a zip file.")
raise gen.Return(res)
monkeypatch.setattr("conda_kapsel.internal.http_client.FileDownloader.run", mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(PROJECT_DIR=dirname))
assert not result
assert [("Failed to unzip %s: File is not a zip file" % os.path.join(dirname, "data.zip")),
"missing requirement to run this project: A downloaded file which is referenced by DATAFILE.",
" Environment variable DATAFILE is not set."] == result.errors
def unprepare_empty(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert result
status = unprepare(project, result)
assert status
def prepare_some_env_var(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ(FOO='bar')
result = prepare_without_interaction(project, environ=environ)
assert result
assert dict(FOO='bar', PROJECT_DIR=project.directory_path) == strip_environ(result.environ)
assert dict(FOO='bar') == strip_environ(environ)
def prepare_with_browser(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
result = prepare_with_browser_ui(project, environ=environ, keep_going_until_success=False, io_loop=io_loop)
assert result
assert dict(FOO_PASSWORD='******', PROJECT_DIR=project.directory_path) == strip_environ(result.environ)
assert dict() == strip_environ(environ)
# wait for the results of the POST to come back,
# awesome hack-tacular
while 'post_fill_in_password' not in http_results:
io_loop.call_later(0.01, lambda: io_loop.stop())
io_loop.start()
assert 'get_click_submit' in http_results
assert 'post_click_submit' in http_results
assert 'post_fill_in_password' in http_results
assert 200 == http_results['get_click_submit'].code
assert 200 == http_results['post_click_submit'].code
assert 200 == http_results['post_fill_in_password'].code
final_done_html = str(http_results['post_fill_in_password'].body)
assert "Done!" in final_done_html
assert "Environment variable FOO_PASSWORD is set." in final_done_html
local_state_file = LocalStateFile.load_for_directory(project.directory_path)
assert local_state_file.get_value(['variables', 'FOO_PASSWORD']) is None
def provide_download(dirname):
@gen.coroutine
def mock_downloader_run(self, loop):
class Res:
pass
res = Res()
res.code = 200
with open(os.path.join(dirname, 'data.csv'), 'w') as out:
out.write('data')
self._hash = '12345abcdef'
raise gen.Return(res)
monkeypatch.setattr("conda_kapsel.internal.http_client.FileDownloader.run", mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(PROJECT_DIR=dirname))
assert hasattr(result, 'environ')
assert 'DATAFILE' in result.environ
filename = os.path.join(dirname, 'data.csv')
assert os.path.exists(filename)
def mock_remove(path):
raise IOError("Not gonna remove this")
monkeypatch.setattr("os.remove", mock_remove)
status = unprepare(project, result)
assert status.logs == []
assert status.status_description == ('Failed to remove %s: Not gonna remove this.' % filename)
assert status.errors == []
assert not status
assert os.path.exists(filename)
monkeypatch.undo() # so os.remove isn't broken during directory cleanup
def start_local_redis(dirname):
project = project_no_dedicated_env(dirname)
result = test_redis_provider._prepare_printing_errors(project, environ=minimal_environ())
assert result
local_state_file = LocalStateFile.load_for_directory(dirname)
state = local_state_file.get_service_run_state('REDIS_URL')
assert 'port' in state
port = state['port']
assert dict(REDIS_URL=("redis://localhost:" + str(port)),
PROJECT_DIR=project.directory_path) == strip_environ(result.environ)
assert len(can_connect_args_list) >= 2
pidfile = os.path.join(dirname, "services/REDIS_URL/redis.pid")
logfile = os.path.join(dirname, "services/REDIS_URL/redis.log")
assert os.path.exists(pidfile)
assert os.path.exists(logfile)
assert real_can_connect_to_socket(host='localhost', port=port)
# now clean it up
code = _parse_args_and_run_subcommand(['conda-kapsel', 'remove-service', 'REDIS_URL', '--directory', dirname])
assert code == 0
assert not os.path.exists(pidfile)
assert not os.path.exists(os.path.join(dirname, "services"))
assert not real_can_connect_to_socket(host='localhost', port=port)
local_state_file.load()
assert dict() == local_state_file.get_service_run_state("REDIS_URL")
def provide_download_of_zip_no_unzip(zipname, dirname):
with codecs.open(os.path.join(dirname, DEFAULT_PROJECT_FILENAME), 'w', 'utf-8') as f:
f.write(complete_project_file_content(ZIPPED_DATAFILE_CONTENT_NO_UNZIP))
@gen.coroutine
def mock_downloader_run(self, loop):
class Res:
pass
res = Res()
res.code = 200
assert self._url.endswith(".zip")
# we aren't going to unzip so we should be downloading straignt to
# the specified filename 'data' without the .zip on it
assert not self._filename.endswith(".zip")
shutil.copyfile(zipname, self._filename)
raise gen.Return(res)
monkeypatch.setattr("conda_kapsel.internal.http_client.FileDownloader.run", mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(PROJECT_DIR=dirname))
assert hasattr(result, 'environ')
assert 'DATAFILE' in result.environ
assert os.path.isfile(os.path.join(dirname, 'data'))
with zipfile.ZipFile(os.path.join(dirname, 'data')) as zf:
assert zf.namelist() == ['foo']
def provide_download_of_zip(zipname, dirname):
with codecs.open(os.path.join(dirname, DEFAULT_PROJECT_FILENAME), 'w', 'utf-8') as f:
f.write(complete_project_file_content(ZIPPED_DATAFILE_CONTENT_CHECKSUM))
@gen.coroutine
def mock_downloader_run(self, loop):
class Res:
pass
res = Res()
res.code = 200
assert self._url.endswith(".zip")
assert self._filename.endswith(".zip")
shutil.copyfile(zipname, self._filename)
self._hash = '12345abcdef'
raise gen.Return(res)
monkeypatch.setattr("conda_kapsel.internal.http_client.FileDownloader.run", mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(PROJECT_DIR=dirname))
assert hasattr(result, 'environ')
assert 'DATAFILE' in result.environ
assert os.path.isdir(os.path.join(dirname, 'data'))
assert os.path.isfile(os.path.join(dirname, 'data', 'foo'))
assert codecs.open(os.path.join(dirname, 'data', 'foo')).read() == 'hello\n'
status = unprepare(project, result)
filename = os.path.join(dirname, 'data')
assert status.logs == ["Removed downloaded file %s." % filename,
("Current environment is not in %s, no need to delete it." % dirname)]
assert status.status_description == "Success."
def prepare_redis_url(dirname):
project = project_no_dedicated_env(dirname)
result = _prepare_printing_errors(project, environ=minimal_environ())
assert result
assert dict(REDIS_URL="redis://localhost:6379",
PROJECT_DIR=project.directory_path) == strip_environ(result.environ)
assert dict(host='localhost', port=6379, timeout_seconds=0.5) == can_connect_args
def provide_download_of_zip(zipname, dirname):
with codecs.open(os.path.join(dirname, DEFAULT_PROJECT_FILENAME), 'w', 'utf-8') as f:
f.write(complete_project_file_content(ZIPPED_DATAFILE_CONTENT_NO_ZIP_SUFFIX))
@gen.coroutine
def mock_downloader_run(self, loop):
class Res:
pass
res = Res()
res.code = 200
# we add .zip to the download filename, even though it wasn't in the URL
assert not self._url.endswith(".zip")
assert self._filename.endswith(".zip")
shutil.copyfile(zipname, self._filename)
raise gen.Return(res)
monkeypatch.setattr("conda_kapsel.internal.http_client.FileDownloader.run", mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(PROJECT_DIR=dirname))
assert hasattr(result, 'environ')
assert 'DATAFILE' in result.environ
assert os.path.isdir(os.path.join(dirname, 'data'))
assert os.path.isfile(os.path.join(dirname, 'data', 'foo'))
assert codecs.open(os.path.join(dirname, 'data', 'foo')).read() == 'hello\n'
def provide_download(dirname):
@gen.coroutine
def mock_downloader_run(self, loop):
class Res:
pass
res = Res()
res.code = 200
with open(os.path.join(dirname, 'data.csv'), 'w') as out:
out.write('data')
self._hash = '12345abcdef'
raise gen.Return(res)
monkeypatch.setattr("conda_kapsel.internal.http_client.FileDownloader.run", mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(PROJECT_DIR=dirname))
assert hasattr(result, 'environ')
assert 'DATAFILE' in result.environ
filename = os.path.join(dirname, 'data.csv')
assert os.path.exists(filename)
status = unprepare(project, result)
assert status.logs == ["Removed downloaded file %s." % filename,
("Current environment is not in %s, no need to delete it." % dirname)]
assert status.status_description == 'Success.'
assert status
assert not os.path.exists(filename)
def check(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ(BAR='bar')
result = prepare_without_interaction(project, environ=environ, command_name="blah")
assert not result
assert result.errors
assert "Command name 'blah' is not in" in result.errors[0]
def prepare_some_env_var(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ(FOO='bar')
result = prepare_without_interaction(project, environ=environ)
assert result
assert dict(FOO='bar', PROJECT_DIR=project.directory_path) == strip_environ(result.environ)
assert dict(FOO='bar') == strip_environ(environ)
def check(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ(BAR='bar')
result = prepare_without_interaction(project, environ=environ, command_name="blah")
assert not result
assert result.errors
assert "Command name 'blah' is not in" in result.errors[0]
def no_start_local_redis(dirname):
project = project_no_dedicated_env(dirname)
result = _prepare_printing_errors(project,
environ=minimal_environ(),
mode=provide.PROVIDE_MODE_CHECK)
assert not result
assert 3 == len(can_connect_args_list)
def unprepare_empty(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert result
status = unprepare(project, result)
assert status
def start_local_redis(dirname):
project = project_no_dedicated_env(dirname)
result = _prepare_printing_errors(project, environ=minimal_environ())
assert result
assert 'REDIS_URL' in result.environ
local_state_file = LocalStateFile.load_for_directory(dirname)
state = local_state_file.get_service_run_state("REDIS_URL")
assert 'port' in state
port = state['port']
assert dict(REDIS_URL=("redis://localhost:" + str(port)),
PROJECT_DIR=project.directory_path) == strip_environ(
result.environ)
assert len(can_connect_args_list) >= 2
pidfile = os.path.join(dirname, "services/REDIS_URL/redis.pid")
logfile = os.path.join(dirname, "services/REDIS_URL/redis.log")
assert os.path.exists(pidfile)
assert os.path.exists(logfile)
assert real_can_connect_to_socket(host='localhost', port=port)
# be sure we generate the config html that would use the old one
requirement = _redis_requirement()
status = requirement.check_status(result.environ, local_state_file,
'default', UserConfigOverrides())
html = RedisProvider().config_html(requirement, result.environ,
local_state_file,
UserConfigOverrides(), status)
assert 'Use the redis-server we started earlier' in html
# now try again, and we should re-use the exact same server
pidfile_mtime = os.path.getmtime(pidfile)
with codecs.open(pidfile, 'r', 'utf-8') as file:
pidfile_content = file.read()
result2 = _prepare_printing_errors(project, environ=minimal_environ())
assert result2
# port should be the same, and set in the environment
assert dict(REDIS_URL=("redis://localhost:" + str(port)),
PROJECT_DIR=project.directory_path) == strip_environ(
result2.environ)
# no new pid file
assert pidfile_mtime == os.path.getmtime(pidfile)
with codecs.open(pidfile, 'r', 'utf-8') as file:
pidfile_content2 = file.read()
assert pidfile_content == pidfile_content2
# now clean it up
status = unprepare(project, result2)
assert status
assert not os.path.exists(pidfile)
assert not real_can_connect_to_socket(host='localhost', port=port)
local_state_file.load()
assert dict() == local_state_file.get_service_run_state("REDIS_URL")
def unprepare_nothing(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert result
status = unprepare(project, result, whitelist=[])
assert status
assert status.status_description == 'Nothing to clean up.'
def prepare_empty(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert result
assert dict(PROJECT_DIR=project.directory_path) == strip_environ(result.environ)
assert dict() == strip_environ(environ)
assert result.command_exec_info is None
def unprepare_nothing(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert result
status = unprepare(project, result, whitelist=[])
assert status
assert status.status_description == 'Nothing to clean up.'
def prepare_empty(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert result
assert dict(PROJECT_DIR=project.directory_path) == strip_environ(result.environ)
assert dict() == strip_environ(environ)
assert result.command_exec_info is None
def check(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ(BAR='bar')
result = prepare_with_browser_ui(project, environ=environ, keep_going_until_success=False)
assert not result
assert dict(BAR='bar') == strip_environ(environ)
assert [('Icon file %s does not exist.' % os.path.join(dirname, 'foo.png')), 'Unable to load the project.'
] == result.errors
def prepare_then_update_environ(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ(FOO='bar')
result = prepare_without_interaction(project, environ=environ)
assert result
other = minimal_environ(BAR='baz')
result.update_environ(other)
assert dict(FOO='bar', BAR='baz', PROJECT_DIR=dirname) == strip_environ(other)
def check_env_var_provider_prepare(dirname):
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(FOO='bar'))
assert result
status = unprepare(project, result)
assert status
assert status.status_description == 'Success.'
assert status.logs == ["Nothing to clean up for FOO.",
("Current environment is not in %s, no need to delete it." % dirname)]
def prepare_then_update_environ(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ(FOO='bar')
result = prepare_without_interaction(project, environ=environ)
assert result
other = minimal_environ(BAR='baz')
result.update_environ(other)
assert dict(FOO='bar', BAR='baz', PROJECT_DIR=dirname) == strip_environ(other)
def check(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ(BAR='bar')
result = prepare_with_browser_ui(project, environ=environ, keep_going_until_success=False)
assert not result
assert dict(BAR='bar') == strip_environ(environ)
assert [('Icon file %s does not exist.' % os.path.join(dirname, 'foo.png')), 'Unable to load the project.'
] == result.errors
def prepare_redis_url(dirname):
project = project_no_dedicated_env(dirname)
result = _prepare_printing_errors(project, environ=minimal_environ())
assert result
assert dict(REDIS_URL="redis://localhost:6379",
PROJECT_DIR=project.directory_path) == strip_environ(
result.environ)
assert dict(host='localhost', port=6379,
timeout_seconds=0.5) == can_connect_args
def unprepare_problems(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert not result
status = unprepare(project, result)
assert not status
assert status.status_description == 'Unable to load the project.'
assert status.errors == ['variables section contains wrong value type 42, ' +
'should be dict or list of requirements']
def check(dirname):
# create a command that isn't in the Project
project = project_no_dedicated_env(dirname)
command = ProjectCommand(name="foo",
attributes=dict(bokeh_app="foo.py",
env_spec=project.default_env_spec_name))
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ, command=command)
assert result
assert result.command_exec_info.bokeh_app == 'foo.py'
def unprepare_problems(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert not result
status = unprepare(project, result)
assert not status
assert status.status_description == 'Unable to load the project.'
assert status.errors == ['variables section contains wrong value type 42, ' +
'should be dict or list of requirements']
def check(dirname):
# create a command that isn't in the Project
project = project_no_dedicated_env(dirname)
command = ProjectCommand(name="foo",
attributes=dict(bokeh_app="foo.py",
env_spec=project.default_env_spec_name))
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ, command=command)
assert result
assert os.path.join(project.directory_path, 'foo.py') in result.command_exec_info.args
def check(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ, command_name='foo')
assert result
assert result.command_exec_info.bokeh_app == 'foo.py'
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ, command_name='bar')
assert result
assert result.command_exec_info.bokeh_app == 'bar.py'
def start_local_redis(dirname):
project = project_no_dedicated_env(dirname)
result = _prepare_printing_errors(project, environ=minimal_environ())
assert result
assert 'REDIS_URL' in result.environ
local_state_file = LocalStateFile.load_for_directory(dirname)
state = local_state_file.get_service_run_state("REDIS_URL")
assert 'port' in state
port = state['port']
assert dict(REDIS_URL=("redis://localhost:" + str(port)),
PROJECT_DIR=project.directory_path) == strip_environ(result.environ)
assert len(can_connect_args_list) >= 2
pidfile = os.path.join(dirname, "services/REDIS_URL/redis.pid")
logfile = os.path.join(dirname, "services/REDIS_URL/redis.log")
assert os.path.exists(pidfile)
assert os.path.exists(logfile)
assert real_can_connect_to_socket(host='localhost', port=port)
# be sure we generate the config html that would use the old one
requirement = _redis_requirement()
status = requirement.check_status(result.environ, local_state_file, 'default', UserConfigOverrides())
html = RedisProvider().config_html(requirement, result.environ, local_state_file, UserConfigOverrides(), status)
assert 'Use the redis-server we started earlier' in html
# now try again, and we should re-use the exact same server
pidfile_mtime = os.path.getmtime(pidfile)
with codecs.open(pidfile, 'r', 'utf-8') as file:
pidfile_content = file.read()
result2 = _prepare_printing_errors(project, environ=minimal_environ())
assert result2
# port should be the same, and set in the environment
assert dict(REDIS_URL=("redis://localhost:" + str(port)),
PROJECT_DIR=project.directory_path) == strip_environ(result2.environ)
# no new pid file
assert pidfile_mtime == os.path.getmtime(pidfile)
with codecs.open(pidfile, 'r', 'utf-8') as file:
pidfile_content2 = file.read()
assert pidfile_content == pidfile_content2
# now clean it up
status = unprepare(project, result2)
assert status
assert not os.path.exists(pidfile)
assert not real_can_connect_to_socket(host='localhost', port=port)
local_state_file.load()
assert dict() == local_state_file.get_service_run_state("REDIS_URL")
def check(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ, command_name='foo')
assert result
assert os.path.join(project.directory_path, 'foo.py') in result.command_exec_info.args
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ, command_name='bar')
assert result
assert os.path.join(project.directory_path, 'bar.py') in result.command_exec_info.args
def provide_download(dirname):
@gen.coroutine
def mock_downloader_run(self, loop):
raise Exception("should not have tried to download in check mode")
monkeypatch.setattr("conda_kapsel.internal.http_client.FileDownloader.run", mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project,
environ=minimal_environ(PROJECT_DIR=dirname),
mode=provide.PROVIDE_MODE_CHECK)
assert not result
def start_local_redis(dirname):
project = project_no_dedicated_env(dirname)
result = _prepare_printing_errors(project, environ=minimal_environ())
assert result
# now clean it up, but arrange for that to fail
local_state_file = LocalStateFile.load_for_directory(dirname)
local_state_file.set_service_run_state('REDIS_URL', {'shutdown_commands': [['false']]})
local_state_file.save()
status = unprepare(project, result)
assert not status
assert status.status_description == 'Shutdown commands failed for REDIS_URL.'
def provide_download(dirname):
FILENAME = os.path.join(dirname, 'data.csv')
requirement = _download_requirement()
local_state_file = LocalStateFile.load_for_directory(dirname)
local_state_file.set_service_run_state(requirement.env_var, {'filename': FILENAME})
local_state_file.save()
with open(FILENAME, 'w') as out:
out.write('data')
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(PROJECT_DIR=dirname))
assert hasattr(result, 'environ')
assert 'DATAFILE' in result.environ
def start_local_redis(dirname):
project = project_no_dedicated_env(dirname)
result = _prepare_printing_errors(project, environ=minimal_environ())
assert result
# now clean it up, but arrange for that to fail
local_state_file = LocalStateFile.load_for_directory(dirname)
local_state_file.set_service_run_state(
'REDIS_URL', {'shutdown_commands': [['false']]})
local_state_file.save()
status = unprepare(project, result)
assert not status
assert status.status_description == 'Shutdown commands failed for REDIS_URL.'
def provide_download(dirname):
@gen.coroutine
def mock_downloader_run(self, loop):
# if we don't even get an HTTP response, the errors are handled this way,
# e.g. if the URL is bad.
self._errors = ['This went horribly wrong']
raise gen.Return(None)
monkeypatch.setattr("conda_kapsel.internal.http_client.FileDownloader.run", mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(PROJECT_DIR=dirname))
assert not result
assert ('missing requirement to run this project: A downloaded file which is referenced by DATAFILE.'
) in result.errors
def provide_download(dirname):
@gen.coroutine
def mock_downloader_run(self, loop):
class Res:
pass
res = Res()
res.code = 400
raise gen.Return(res)
monkeypatch.setattr("conda_kapsel.internal.http_client.FileDownloader.run", mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(PROJECT_DIR=dirname))
assert not result
assert ('missing requirement to run this project: A downloaded file which is referenced by DATAFILE.'
) in result.errors
def start_local_redis(dirname):
from subprocess import Popen as real_Popen
def mock_Popen(*args, **kwargs):
if 'args' not in kwargs:
# `pip list` goes through this codepath while redis launch
# happens to specify args= as a kwarg
assert 'pip' in args[0][0]
return real_Popen(*args, **kwargs)
kwargs['args'] = ['this-is-not-on-the-path']
return real_Popen(*args, **kwargs)
monkeypatch.setattr("subprocess.Popen", mock_Popen)
project = project_no_dedicated_env(dirname)
result = _prepare_printing_errors(project, environ=minimal_environ())
assert not result
def provide_download(dirname):
@gen.coroutine
def mock_downloader_run(self, loop):
raise Exception('error')
monkeypatch.setattr("conda_kapsel.internal.http_client.FileDownloader.run", mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(PROJECT_DIR=dirname))
assert not result
assert ('missing requirement to run this project: A downloaded file which is referenced by DATAFILE.'
) in result.errors
status = unprepare(project, result)
filename = os.path.join(dirname, 'data.csv')
assert status.logs == ["No need to remove %s which wasn't downloaded." % filename,
("Current environment is not in %s, no need to delete it." % dirname)]
assert status.status_description == 'Success.'
def start_local_redis(dirname):
from subprocess import Popen as real_Popen
def mock_Popen(*args, **kwargs):
if 'args' not in kwargs:
# `pip list` goes through this codepath while redis launch
# happens to specify args= as a kwarg
assert 'pip' in args[0][0]
return real_Popen(*args, **kwargs)
kwargs['args'] = ['this-is-not-on-the-path']
return real_Popen(*args, **kwargs)
monkeypatch.setattr("subprocess.Popen", mock_Popen)
project = project_no_dedicated_env(dirname)
result = _prepare_printing_errors(project, environ=minimal_environ())
assert not result
def provide_download(dirname):
@gen.coroutine
def mock_downloader_run(self, loop):
class Res:
pass
res = Res()
res.code = 200
with open(os.path.join(dirname, 'data.csv'), 'w') as out:
out.write('data')
raise gen.Return(res)
monkeypatch.setattr("conda_kapsel.internal.http_client.FileDownloader.run", mock_downloader_run)
project = project_no_dedicated_env(dirname)
result = prepare_without_interaction(project, environ=minimal_environ(PROJECT_DIR=dirname))
assert hasattr(result, 'environ')
assert 'DATAFILE' in result.environ
def start_local_redis_and_time_out(dirname):
project = project_no_dedicated_env(dirname)
from time import sleep as real_sleep
killed = {}
def mock_sleep_kills_redis(seconds):
# first time the Redis provider sleeps to wait for the
# server to appear, we kill the server; after that
# we make sleep into a no-op so we rapidly time out.
if 'done' in killed:
return
pidfile = os.path.join(dirname, "services", "REDIS_URL",
"redis.pid")
count = 0
while count < 15:
if os.path.exists(pidfile):
break
real_sleep(0.1)
count = count + 1
assert os.path.exists(pidfile)
with codecs.open(pidfile, 'r', 'utf-8') as f:
for line in f.readlines():
try:
import signal
os.kill(int(line.strip()), signal.SIGKILL)
except Exception:
pass
# be sure it's gone
real_sleep(0.1)
killed['done'] = True
monkeypatch.setattr('time.sleep', mock_sleep_kills_redis)
result = _prepare_printing_errors(project, environ=minimal_environ())
assert not result
out, err = capsys.readouterr()
assert "redis-server started successfully, but we timed out trying to connect to it on port" in out
assert "redis-server process failed or timed out, exited with code 0" in err
def prepare_with_browser(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ(BAR='bar')
result = prepare_with_browser_ui(project, environ=environ, keep_going_until_success=False, io_loop=io_loop)
assert not result
assert dict(BAR='bar') == strip_environ(environ)
# wait for the results of the POST to come back,
# awesome hack-tacular
while 'post' not in http_results:
io_loop.call_later(0.01, lambda: io_loop.stop())
io_loop.start()
assert 'get' in http_results
assert 'post' in http_results
assert 200 == http_results['get'].code
assert 200 == http_results['post'].code
def prepare_some_env_var_keep_going(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ(BAR='bar')
stage = prepare_in_stages(project, environ=environ, keep_going_until_success=True)
# there's an initial stage to set the conda env
next_stage = stage.execute()
assert not stage.failed
assert stage.environ['PROJECT_DIR'] == dirname
stage = next_stage
for i in range(1, 10):
next_stage = stage.execute()
assert next_stage is not None
assert stage.failed
assert stage.environ['PROJECT_DIR'] == dirname
stage = next_stage
assert dict(BAR='bar') == strip_environ(environ)
def prepare_some_env_var_keep_going(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ(BAR='bar')
stage = prepare_in_stages(project, environ=environ, keep_going_until_success=True)
# there's an initial stage to set the conda env
next_stage = stage.execute()
assert not stage.failed
assert stage.environ['PROJECT_DIR'] == dirname
stage = next_stage
for i in range(1, 10):
next_stage = stage.execute()
assert next_stage is not None
assert stage.failed
assert stage.environ['PROJECT_DIR'] == dirname
stage = next_stage
assert dict(BAR='bar') == strip_environ(environ)
def prepare_with_browser(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ(BAR='bar')
result = prepare_with_browser_ui(project, environ=environ, keep_going_until_success=False, io_loop=io_loop)
assert not result
assert dict(BAR='bar') == strip_environ(environ)
# wait for the results of the POST to come back,
# awesome hack-tacular
while 'post' not in http_results:
io_loop.call_later(0.01, lambda: io_loop.stop())
io_loop.start()
assert 'get' in http_results
assert 'post' in http_results
assert 200 == http_results['get'].code
assert 200 == http_results['post'].code
def prepare_system_environ(dirname):
project = project_no_dedicated_env(dirname)
os_environ_copy = deepcopy(os.environ)
result = prepare_without_interaction(project)
assert project.directory_path == strip_environ(result.environ)['PROJECT_DIR']
# os.environ wasn't modified
assert os_environ_copy == os.environ
# result.environ inherits everything in os.environ
for key in os_environ_copy:
if key == 'PATH' and platform.system() == 'Windows' and result.environ[key] != os.environ[key]:
print("prepare changed PATH on Windows and ideally it would not.")
else:
if key == 'PATH' and result.environ[key] != os.environ[key]:
original = os.environ[key].split(os.pathsep)
updated = result.environ[key].split(os.pathsep)
print("ORIGINAL PATH: " + repr(original))
print("UPDATED PATH: " + repr(updated))
assert original == updated
assert result.environ.get(key) == os.environ.get(key)
def prepare_with_app_entry(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ(FOO='bar')
env_path = conda_api.environ_get_prefix(environ)
result = prepare_without_interaction(project, environ=environ)
assert result
command = result.command_exec_info
assert 'FOO' in command.env
assert command.cwd == project.directory_path
if platform.system() == 'Windows':
commandpath = os.path.join(env_path, "python.exe")
else:
commandpath = os.path.join(env_path, "bin", "python")
assert command.args == [commandpath, 'echo.py', env_path, 'foo', 'bar']
p = command.popen(stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(out, err) = p.communicate()
# strip is to pull off the platform-specific newline
assert out.decode().strip() == ("['echo.py', '%s', 'foo', 'bar']" % (env_path.replace("\\", "\\\\")))
assert err.decode() == ""
def start_local_redis(dirname):
logfile = os.path.join(dirname, "services/REDIS_URL/redis.log")
from subprocess import Popen as real_Popen
failscript = os.path.join(dirname, "fail.py")
with codecs.open(failscript, 'w', 'utf-8') as file:
file.write("""
from __future__ import print_function
import codecs
import sys
import os
print('It did not work stdout')
print('It did not work stderr', file=sys.stderr)
logfile = sys.argv[1]
fail_mode = sys.argv[2]
if fail_mode == 'no_logfile':
pass
elif fail_mode == 'is_dir':
os.makedirs(logfile)
else:
with codecs.open(logfile, 'w', 'utf-8') as f:
f.write('This is in the logfile')
sys.exit(1)
""")
def mock_Popen(*args, **kwargs):
if 'args' not in kwargs:
# `pip list` goes through this codepath while redis launch
# happens to specify args= as a kwarg
assert 'pip' in args[0][0]
return real_Popen(*args, **kwargs)
kwargs['args'] = ['python', failscript, logfile, logfile_fail_mode]
return real_Popen(*args, **kwargs)
monkeypatch.setattr("subprocess.Popen", mock_Popen)
project = project_no_dedicated_env(dirname)
result = _prepare_printing_errors(project, environ=minimal_environ())
assert not result
def start_local_redis(dirname):
project = project_no_dedicated_env(dirname)
result = _prepare_printing_errors(project, environ=minimal_environ())
assert result
local_state_file = LocalStateFile.load_for_directory(dirname)
state = local_state_file.get_service_run_state('REDIS_URL')
assert 'port' in state
port = state['port']
assert dict(REDIS_URL=("redis://localhost:" + str(port)),
PROJECT_DIR=project.directory_path) == strip_environ(
result.environ)
assert len(can_connect_args_list) >= 2
servicedir = os.path.join(dirname, "services")
redisdir = os.path.join(servicedir, "REDIS_URL")
pidfile = os.path.join(redisdir, "redis.pid")
logfile = os.path.join(redisdir, "redis.log")
assert os.path.exists(pidfile)
assert os.path.exists(logfile)
assert real_can_connect_to_socket(host='localhost', port=port)
# now clean it up
status = unprepare(project, result)
assert status
assert not os.path.exists(pidfile)
assert not os.path.exists(logfile)
assert not os.path.exists(redisdir)
assert not os.path.exists(servicedir)
assert not real_can_connect_to_socket(host='localhost', port=port)
local_state_file.load()
assert dict() == local_state_file.get_service_run_state("REDIS_URL")
def start_local_redis(dirname):
project = project_no_dedicated_env(dirname)
result = test_redis_provider._prepare_printing_errors(
project, environ=minimal_environ())
assert result
local_state_file = LocalStateFile.load_for_directory(dirname)
state = local_state_file.get_service_run_state('REDIS_URL')
assert 'port' in state
port = state['port']
assert dict(REDIS_URL=("redis://localhost:" + str(port)),
PROJECT_DIR=project.directory_path) == strip_environ(
result.environ)
assert len(can_connect_args_list) >= 2
pidfile = os.path.join(dirname, "services/REDIS_URL/redis.pid")
logfile = os.path.join(dirname, "services/REDIS_URL/redis.log")
assert os.path.exists(pidfile)
assert os.path.exists(logfile)
assert real_can_connect_to_socket(host='localhost', port=port)
# now clean it up
code = _parse_args_and_run_subcommand([
'conda-kapsel', 'remove-service', 'REDIS_URL', '--directory',
dirname
])
assert code == 0
assert not os.path.exists(pidfile)
assert not os.path.exists(os.path.join(dirname, "services"))
assert not real_can_connect_to_socket(host='localhost', port=port)
local_state_file.load()
assert dict() == local_state_file.get_service_run_state("REDIS_URL")
def prepare_some_env_var(dirname):
project = project_no_dedicated_env(dirname)
environ = minimal_environ(BAR='bar')
result = prepare_without_interaction(project, environ=environ)
assert not result
assert dict(BAR='bar') == strip_environ(environ)
def early_result_grab(dirname):
project = project_no_dedicated_env(dirname)
first_stage = prepare_in_stages(project)
with pytest.raises(RuntimeError) as excinfo:
first_stage.result
assert "result property isn't available" in repr(excinfo.value)
def early_status_grab(dirname):
project = project_no_dedicated_env(dirname)
first_stage = prepare_in_stages(project)
with pytest.raises(RuntimeError) as excinfo:
first_stage.statuses_after_execute
assert "statuses_after_execute isn't available" in repr(excinfo.value)
def prepare_after_setting_scope(dirname):
local_state = LocalStateFile.load_for_directory(dirname)
requirement = _redis_requirement()
provider = RedisProvider()
environ = minimal_environ()
config = provider.read_config(requirement, environ, local_state,
'default', UserConfigOverrides())
assert config['source'] == 'find_all'
provider.set_config_values_as_strings(requirement, environ,
local_state, 'default',
UserConfigOverrides(),
dict(source='find_project'))
config = provider.read_config(requirement, environ, local_state,
'default', UserConfigOverrides())
assert config['source'] == 'find_project'
provider.set_config_values_as_strings(requirement, environ,
local_state, 'default',
UserConfigOverrides(),
dict(source='find_all'))
config = provider.read_config(requirement, environ, local_state,
'default', UserConfigOverrides())
assert config['source'] == 'find_all'
provider.set_config_values_as_strings(requirement, environ,
local_state, 'default',
UserConfigOverrides(),
dict(source='environ'))
config = provider.read_config(requirement, environ, local_state,
'default', UserConfigOverrides())
assert config['source'] == 'find_all' # default if no env var set
provider.set_config_values_as_strings(requirement, environ,
local_state, 'default',
UserConfigOverrides(),
dict(source='environ'))
environ_with_redis_url = environ.copy()
environ_with_redis_url['REDIS_URL'] = 'blah'
config = provider.read_config(requirement, environ_with_redis_url,
local_state, 'default',
UserConfigOverrides())
assert config['source'] == 'environ' # default when the env var IS set
# use local variable when env var not set
provider.set_config_values_as_strings(
requirement, environ, local_state, 'default',
UserConfigOverrides(), dict(source='variables', value='foo'))
config = provider.read_config(requirement, environ, local_state,
'default', UserConfigOverrides())
assert config['source'] == 'variables'
assert config['value'] == 'foo'
# use local variable when env var _is_ set
provider.set_config_values_as_strings(
requirement, environ_with_redis_url, local_state, 'default',
UserConfigOverrides(), dict(source='variables', value='foo'))
config = provider.read_config(requirement, environ, local_state,
'default', UserConfigOverrides())
assert config['source'] == 'variables'
assert config['value'] == 'foo'
# set to use system, which should override using the local state
provider.set_config_values_as_strings(requirement, environ,
local_state, 'default',
UserConfigOverrides(),
dict(source='find_system'))
config = provider.read_config(requirement, environ, local_state,
'default', UserConfigOverrides())
assert config['source'] == 'find_system'
project = project_no_dedicated_env(dirname)
result = _prepare_printing_errors(project, environ=minimal_environ())
assert result
assert dict(REDIS_URL="redis://localhost:6379",
PROJECT_DIR=project.directory_path) == strip_environ(
result.environ)
assert dict(host='localhost', port=6379,
timeout_seconds=0.5) == can_connect_args
def prepare_bad_provide_mode(dirname):
with pytest.raises(ValueError) as excinfo:
project = project_no_dedicated_env(dirname)
environ = minimal_environ()
prepare_in_stages(project, mode="BAD_PROVIDE_MODE", environ=environ)
assert "invalid provide mode" in repr(excinfo.value)
