def check(dirname): from anaconda_project.requirements_registry.requirements.conda_env import CondaEnvRequirement _monkeypatch_download_file(monkeypatch, dirname, filename="nope") no_foo = [('missing requirement to run this project: A downloaded file which is ' + 'referenced by FOO.'), ' Environment variable FOO is not set.'] # whitelist only the env req by class project = project_no_dedicated_env(dirname) environ = minimal_environ() result = prepare_without_interaction(project, provide_whitelist=(CondaEnvRequirement, ), environ=environ) assert result.errors == no_foo # whitelist by instance env_req = [req for req in project.requirements(None) if isinstance(req, CondaEnvRequirement)][0] result = prepare_without_interaction(project, provide_whitelist=(env_req, ), environ=environ) assert result.errors == no_foo # whitelist by variable name result = prepare_without_interaction(project, provide_whitelist=(env_req.env_var, ), environ=environ) assert result.errors == no_foo # whitelist the download result = prepare_without_interaction(project, provide_whitelist=(env_req, project.download_requirements(None)[0]), environ=environ) assert result.errors == [] assert 'FOO' in result.environ
def provide_download(dirname): project = project_no_dedicated_env(dirname) prepare_without_interaction( project, environ=minimal_environ(PROJECT_DIR=dirname)) assert ( "%s: 'downloads:' section should be a dictionary, found ['http://localhost/data.csv']" % DEFAULT_PROJECT_FILENAME) in project.problems
def check(dirname): env_var = conda_api.conda_prefix_variable() try: _push_fake_env_creator() project = Project(dirname) environ = minimal_environ() result = prepare_without_interaction(project, environ=environ, env_spec_name='foo') expected_path = project.env_specs['foo'].path( project.directory_path) assert result.environ[env_var] == expected_path environ = minimal_environ() result = prepare_without_interaction(project, environ=environ, env_spec_name='bar') assert result.errors == [] assert result expected_path = project.env_specs['bar'].path( project.directory_path) assert result.environ[env_var] == expected_path finally: _pop_fake_env_creator()
def check(dirname): project = project_no_dedicated_env(dirname) environ = minimal_environ() result = prepare_without_interaction(project, environ=environ, command_name='foo') assert result.errors == [] assert result assert os.path.join(project.directory_path, 'foo.py') in result.command_exec_info.args environ = minimal_environ() result = prepare_without_interaction(project, environ=environ, command_name='bar') assert result.errors == [] assert result assert os.path.join(project.directory_path, 'bar.py') in result.command_exec_info.args
def prepare_project_scoped_env(dirname): project = Project(dirname) fake_old_path = "foo" + os.pathsep + "bar" environ = dict(PROJECT_DIR=dirname, PATH=fake_old_path) result = prepare_without_interaction(project, environ=environ) assert result expected_env = os.path.join(dirname, "envs", "bootstrap-env") if platform.system() == 'Windows': expected_new_path = expected_env + os.pathsep + os.path.join( expected_env, script_dir) + os.pathsep + os.path.join( expected_env, "Library", "bin") + os.pathsep + "foo" + os.pathsep + "bar" else: expected_new_path = os.path.join( expected_env, script_dir) + os.pathsep + "foo" + os.pathsep + "bar" expected = dict(PROJECT_DIR=project.directory_path, PATH=expected_new_path, BOOTSTRAP_ENV_PREFIX=expected_env) conda_api.environ_set_prefix(expected, expected_env) expected == result.environ assert os.path.exists(os.path.join(expected_env, "conda-meta")) conda_meta_mtime = os.path.getmtime( os.path.join(expected_env, "conda-meta")) # bare minimum bootstrap-env env shouldn't include these # (contrast with the test later where we list them in # requirements) installed = conda_api.installed(expected_env) assert 'ipython' not in installed assert 'numpy' not in installed # Prepare it again should no-op (use the already-existing environment) environ = dict(PROJECT_DIR=dirname, PATH=fake_old_path) result = prepare_without_interaction(project, environ=environ) assert result expected = dict(PROJECT_DIR=project.directory_path, PATH=expected_new_path) conda_api.environ_set_prefix(expected, expected_env) assert conda_meta_mtime == os.path.getmtime( os.path.join(expected_env, "conda-meta")) # Now unprepare status = unprepare(project, result) assert status # todo: this differs from standard CondaEnvProvider assert status.status_description == 'Success.' assert status.errors == [] assert not os.path.exists(expected_env)
def prepare_system_environ(dirname): project = project_no_dedicated_env(dirname) os_environ_copy = deepcopy(os.environ) result = prepare_without_interaction(project) assert result assert result.errors == [] assert project.directory_path == strip_environ( result.environ)['PROJECT_DIR'] # os.environ wasn't modified assert os_environ_copy == os.environ # result.environ inherits everything in os.environ for key, original in os_environ_copy.items(): updated = result.environ.get(key) if updated != original: if original in ('root', 'base') and updated in ('root', 'base'): print("we have a root/base environment name issue here") continue if key == 'PATH' and platform.system() == 'Windows': print( "prepare changed PATH on Windows and ideally it would not." ) continue updated = updated.split(os.pathsep) original = original.split(os.pathsep) print("ORIGINAL {}: {}".format(key, repr(original))) print("UPDATED {}: {}".format(key, repr(updated))) assert updated == original
def provide_download(dirname): @gen.coroutine def mock_downloader_run(self, loop): class Res: pass res = Res() res.code = 200 with open(os.path.join(dirname, 'data.csv'), 'w') as out: out.write('data') self._hash = '12345abcdef' raise gen.Return(res) monkeypatch.setattr( "anaconda_project.internal.http_client.FileDownloader.run", mock_downloader_run) project = project_no_dedicated_env(dirname) result = prepare_without_interaction( project, environ=minimal_environ(PROJECT_DIR=dirname)) assert hasattr(result, 'environ') assert 'DATAFILE' in result.environ filename = os.path.join(dirname, 'data.csv') assert os.path.exists(filename) project.frontend.reset() status = unprepare(project, result) assert project.frontend.logs == [ "Removed downloaded file %s." % filename, ("Current environment is not in %s, no need to delete it." % dirname) ] assert status.status_description == 'Success.' assert status assert not os.path.exists(filename)
def provide_download_of_zip(dirname): with codecs.open(os.path.join(dirname, DEFAULT_PROJECT_FILENAME), 'w', 'utf-8') as f: f.write(complete_project_file_content(ZIPPED_DATAFILE_CONTENT)) @gen.coroutine def mock_downloader_run(self, loop): class Res: pass res = Res() res.code = 200 assert self._url.endswith(".zip") assert self._filename.endswith(".zip") with codecs.open(self._filename, 'w', 'utf-8') as f: f.write("This is not a zip file.") raise gen.Return(res) monkeypatch.setattr( "anaconda_project.internal.http_client.FileDownloader.run", mock_downloader_run) project = project_no_dedicated_env(dirname) result = prepare_without_interaction( project, environ=minimal_environ(PROJECT_DIR=dirname)) assert not result assert [( "Failed to unzip %s: File is not a zip file" % os.path.join(dirname, "data.zip") ), "missing requirement to run this project: A downloaded file which is referenced by DATAFILE.", " Environment variable DATAFILE is not set."] == result.errors
def provide_download_of_zip(zipname, dirname): with codecs.open(os.path.join(dirname, DEFAULT_PROJECT_FILENAME), 'w', 'utf-8') as f: f.write( complete_project_file_content( ZIPPED_DATAFILE_CONTENT_NO_ZIP_SUFFIX)) @gen.coroutine def mock_downloader_run(self, loop): class Res: pass res = Res() res.code = 200 # we add .zip to the download filename, even though it wasn't in the URL assert not self._url.endswith(".zip") assert self._filename.endswith(".zip") shutil.copyfile(zipname, self._filename) raise gen.Return(res) monkeypatch.setattr( "anaconda_project.internal.http_client.FileDownloader.run", mock_downloader_run) project = project_no_dedicated_env(dirname) result = prepare_without_interaction( project, environ=minimal_environ(PROJECT_DIR=dirname)) assert hasattr(result, 'environ') assert 'DATAFILE' in result.environ assert os.path.isdir(os.path.join(dirname, 'data')) assert os.path.isfile(os.path.join(dirname, 'data', 'foo')) assert codecs.open(os.path.join(dirname, 'data', 'foo')).read() == 'hello\n'
def provide_download_of_zip_no_unzip(zipname, dirname): with codecs.open(os.path.join(dirname, DEFAULT_PROJECT_FILENAME), 'w', 'utf-8') as f: f.write( complete_project_file_content( ZIPPED_DATAFILE_CONTENT_NO_UNZIP)) @gen.coroutine def mock_downloader_run(self, loop): class Res: pass res = Res() res.code = 200 assert self._url.endswith(".zip") # we aren't going to unzip so we should be downloading straignt to # the specified filename 'data' without the .zip on it assert not self._filename.endswith(".zip") shutil.copyfile(zipname, self._filename) raise gen.Return(res) monkeypatch.setattr( "anaconda_project.internal.http_client.FileDownloader.run", mock_downloader_run) project = project_no_dedicated_env(dirname) result = prepare_without_interaction( project, environ=minimal_environ(PROJECT_DIR=dirname)) assert hasattr(result, 'environ') assert 'DATAFILE' in result.environ assert os.path.isfile(os.path.join(dirname, 'data')) with zipfile.ZipFile(os.path.join(dirname, 'data')) as zf: assert zf.namelist() == ['foo']
def provide_download(dirname): @gen.coroutine def mock_downloader_run(self, loop): raise Exception('error') monkeypatch.setattr( "anaconda_project.internal.http_client.FileDownloader.run", mock_downloader_run) project = project_no_dedicated_env(dirname) result = prepare_without_interaction( project, environ=minimal_environ(PROJECT_DIR=dirname)) assert not result assert ( 'missing requirement to run this project: A downloaded file which is referenced by DATAFILE.' ) in result.errors project.frontend.reset() status = unprepare(project, result) filename = os.path.join(dirname, 'data.csv') assert project.frontend.logs == [ "No need to remove %s which wasn't downloaded." % filename, ("Current environment is not in %s, no need to delete it." % dirname) ] assert status.status_description == 'Success.'
def prepare_system_environ(dirname): project = project_no_dedicated_env(dirname) os_environ_copy = deepcopy(os.environ) result = prepare_without_interaction(project) assert project.directory_path == strip_environ( result.environ)['PROJECT_DIR'] # os.environ wasn't modified assert os_environ_copy == os.environ # result.environ inherits everything in os.environ for key in os_environ_copy: if key == 'PATH' and platform.system( ) == 'Windows' and result.environ[key] != os.environ[key]: print( "prepare changed PATH on Windows and ideally it would not." ) else: if key == 'PATH' and result.environ[key] != os.environ[key]: original = os.environ[key].split(os.pathsep) updated = result.environ[key].split(os.pathsep) print("ORIGINAL PATH: " + repr(original)) print("UPDATED PATH: " + repr(updated)) assert original == updated assert result.errors == [] assert result assert result.environ.get(key) == os.environ.get(key)
def provide_download_of_zip(zipname, dirname): with codecs.open(os.path.join(dirname, DEFAULT_PROJECT_FILENAME), 'w', 'utf-8') as f: f.write(complete_project_file_content(ZIPPED_DATAFILE_CONTENT_CHECKSUM)) @gen.coroutine def mock_downloader_run(self, loop): class Res: pass res = Res() res.code = 200 assert self._url.endswith(".zip") assert self._filename.endswith(".zip") shutil.copyfile(zipname, self._filename) self._hash = '12345abcdef' raise gen.Return(res) monkeypatch.setattr("anaconda_project.internal.http_client.FileDownloader.run", mock_downloader_run) project = project_no_dedicated_env(dirname) result = prepare_without_interaction(project, environ=minimal_environ(PROJECT_DIR=dirname)) assert hasattr(result, 'environ') assert 'DATAFILE' in result.environ assert os.path.isdir(os.path.join(dirname, 'data')) assert os.path.isfile(os.path.join(dirname, 'data', 'foo')) assert codecs.open(os.path.join(dirname, 'data', 'foo')).read() == 'hello\n' project.frontend.reset() status = unprepare(project, result) filename = os.path.join(dirname, 'data') assert project.frontend.logs == ["Removed downloaded file %s." % filename, ("Current environment is not in %s, no need to delete it." % dirname)] assert status.status_description == "Success."
def provide_download(dirname): @gen.coroutine def mock_downloader_run(self, loop): class Res: pass res = Res() res.code = 200 with open(os.path.join(dirname, 'data.csv'), 'w') as out: out.write('data') self._hash = '12345abcdef' raise gen.Return(res) monkeypatch.setattr("anaconda_project.internal.http_client.FileDownloader.run", mock_downloader_run) project = project_no_dedicated_env(dirname) result = prepare_without_interaction(project, environ=minimal_environ(PROJECT_DIR=dirname)) assert hasattr(result, 'environ') assert 'DATAFILE' in result.environ filename = os.path.join(dirname, 'data.csv') assert os.path.exists(filename) def mock_remove(path): raise IOError("Not gonna remove this") monkeypatch.setattr("os.remove", mock_remove) project.frontend.reset() status = unprepare(project, result) assert project.frontend.logs == [] assert status.status_description == ('Failed to remove %s: Not gonna remove this.' % filename) assert status.errors == [] assert not status assert os.path.exists(filename) monkeypatch.undo() # so os.remove isn't broken during directory cleanup
def check(dirname): project = Project(dirname) environ = minimal_environ() result = prepare_without_interaction(project, environ=environ) assert result # now mimmick an unpacked project packed_file = os.path.join(dirname, 'envs', 'default', 'conda-meta', '.packed') with open(packed_file, 'wt') as f: f.write(conda_api.current_platform()) # without a functional conda-unpack script it will rebuild the env result = prepare_without_interaction(project, environ=environ) assert result assert not os.path.exists(packed_file)
def prepare_project_scoped_env_not_attempted(dirname): project = Project(dirname) environ = minimal_environ(PROJECT_DIR=dirname) result = prepare_without_interaction(project, environ=environ, mode=provide.PROVIDE_MODE_CHECK) assert not result # expected_env_path = os.path.join(dirname, "envs", "default") bootstrap_env_path = os.path.join(dirname, "envs", "bootstrap-env") for err in [ ('missing requirement to run this project: ' + 'The project needs a Conda bootstrap environment containing all required packages.' ), " '%s' doesn't look like it contains a Conda environment yet." % bootstrap_env_path, ]: assert err in result.errors # unprepare should not have anything to do status = unprepare(project, result) assert status assert status.errors == [] # todo: would be good to understand the different message got with a "normal" env assert status.status_description == ("Success.")
def check(dirname): project = project_no_dedicated_env(dirname) environ = minimal_environ(BAR='bar') result = prepare_without_interaction(project, environ=environ, command_name="blah") assert not result assert result.errors assert "Command name 'blah' is not in" in result.errors[0]
def _commit_requirement_if_it_works(project, env_var_or_class, env_spec_name=None): project.project_file.use_changes_without_saving() # See if we can perform the download result = prepare.prepare_without_interaction(project, provide_whitelist=( CondaEnvRequirement, env_var_or_class, ), env_spec_name=env_spec_name) status = result.status_for(env_var_or_class) if status is None: # I _think_ this is currently impossible, but if it were possible, # we'd need to below code and it's hard to prove it's impossible. status = project.problems_status( ) # pragma: no cover # no way to cause right now? # caller was supposed to expect env_var_or_class to still exist, # unless project file got mangled assert status is not None # pragma: no cover if not status: # reload from disk, discarding our changes because they did not work project.project_file.load() else: # yay! project.project_file.save() return status
def prepare_some_env_var(dirname): project = project_no_dedicated_env(dirname) environ = minimal_environ(BAR='bar') result = prepare_without_interaction(project, environ=environ) assert not result assert result.env_prefix is not None assert dict(BAR='bar') == strip_environ(environ)
def prepare_project_locally(self, project, environ, env_spec_name=None, command_name=None, command=None, extra_command_args=None): """Prepare a project to run one of its commands. "Locally" means a machine where development will go on, contrasted with say a production deployment. This method takes any needed actions such as creating environments or starting services, without asking the user for permission. This method returns a result object. The result object has a ``failed`` property. If the result is failed, the ``errors`` property has the errors. If the result is not failed, the ``command_exec_info`` property has the stuff you need to run the project's default command, and the ``environ`` property has the updated environment. The passed-in ``environ`` is not modified in-place. You can update your original environment with ``result.update_environ()`` if you like, but it's probably a bad idea to modify ``os.environ`` in that way because the calling app won't want to have the project environment. The ``environ`` should usually be kept between preparations, starting out as ``os.environ`` but then being modified by the user. If the project has a non-empty ``problems`` attribute, this function returns the project problems inside a failed result. So ``project.problems`` does not need to be checked in advance. Args: project (Project): from the ``load_project`` method environ (dict): os.environ or the previously-prepared environ; not modified in-place env_spec_name (str): the package set name to require, or None for default command_name (str): which named command to choose from the project, None for default command (ProjectCommand): a command object (alternative to command_name) extra_command_args (list): extra args to include in the returned command argv Returns: a ``PrepareResult`` instance, which has a ``failed`` flag """ return prepare.prepare_without_interaction( project=project, environ=environ, mode=provide.PROVIDE_MODE_DEVELOPMENT, env_spec_name=env_spec_name, command_name=command_name, command=command, extra_command_args=extra_command_args)
def prepare_some_env_var(dirname): project = project_no_dedicated_env(dirname) environ = minimal_environ(FOO='bar') result = prepare_without_interaction(project, environ=environ) assert result.errors == [] assert result assert dict(FOO='bar', PROJECT_DIR=project.directory_path) == strip_environ(result.environ) assert dict(FOO='bar') == strip_environ(environ)
def unprepare_empty(dirname): project = Project(dirname) environ = minimal_environ() result = prepare_without_interaction(project, environ=environ) assert result.errors == [] assert result status = unprepare(project, result) assert status.errors == [] assert status
def prepare_empty(dirname): project = Project(dirname) environ = minimal_environ() result = prepare_without_interaction(project, environ=environ) assert result.errors == [] assert result assert dict(PROJECT_DIR=project.directory_path) == strip_environ(result.environ) assert dict() == strip_environ(environ) assert result.command_exec_info is None
def check_env_var_provider_prepare(dirname): project = project_no_dedicated_env(dirname) result = prepare_without_interaction(project, environ=minimal_environ(FOO='bar')) assert result status = unprepare(project, result) assert status assert status.status_description == 'Success.' assert project.frontend.logs == ["Nothing to clean up for FOO.", ("Current environment is not in %s, no need to delete it." % dirname)]
def _prepare_printing_errors(project, environ=None, mode=provide.PROVIDE_MODE_DEVELOPMENT): result = prepare_without_interaction(project, environ=environ, mode=mode) for message in project.frontend.logs: print(message) for error in project.frontend.errors: print(error, file=sys.stderr) if not result: assert result.errors == project.frontend.errors return result
def prepare_project_scoped_env_with_packages(dirname): project = Project(dirname) environ = minimal_environ(PROJECT_DIR=dirname) result = prepare_without_interaction(project, environ=environ, env_spec_name='bootstrap-env') assert result envs_dir = os.path.join(dirname, "envs") env_name = 'bootstrap-env' prefix = os.path.join(envs_dir, env_name) installed = conda_api.installed(prefix) assert 'bokeh' not in installed deps = ['ipython', 'numpy', 'pip'] for pkg in deps: assert pkg in installed deps += ['bokeh'] # Preparing it again with new packages added should add those project.project_file.set_value('packages', deps) project.project_file.save() environ = minimal_environ(PROJECT_DIR=dirname) result = prepare_without_interaction(project, environ=environ) assert result prefix = result.environ[conda_env_var] installed = conda_api.installed(prefix) for pkg in deps: assert pkg in installed installed_pip = pip_api.installed(prefix) assert 'flake8' in installed_pip # Preparing it again with a bogus package should fail deps = project.project_file.get_value('packages') project.project_file.set_value(['packages'], deps + ['boguspackage']) project.project_file.save() environ = minimal_environ(PROJECT_DIR=dirname) result = prepare_without_interaction(project, environ=environ) assert not result
def unprepare_nothing(dirname): project = Project(dirname) environ = minimal_environ() result = prepare_without_interaction(project, environ=environ) assert result.errors == [] assert result status = unprepare(project, result, whitelist=[]) assert status.errors == [] assert status assert status.status_description == 'Nothing to clean up.'
def unprepare_problems(dirname): project = project_no_dedicated_env(dirname) environ = minimal_environ() result = prepare_without_interaction(project, environ=environ) assert not result status = unprepare(project, result) assert not status assert status.status_description == 'Unable to load the project.' assert status.errors == [('%s: variables section contains wrong value type 42, ' + 'should be dict or list of requirements') % project.project_file.basename]
def prepare_then_update_environ(dirname): project = project_no_dedicated_env(dirname) environ = minimal_environ(FOO='bar') result = prepare_without_interaction(project, environ=environ) assert result.errors == [] assert result other = minimal_environ(BAR='baz') result.update_environ(other) assert dict(FOO='bar', BAR='baz', PROJECT_DIR=dirname) == strip_environ(other)
def check(dirname): # create a command that isn't in the Project project = project_no_dedicated_env(dirname) command = ProjectCommand( name="foo", attributes=dict(bokeh_app="foo.py", env_spec=project.default_env_spec_name)) environ = minimal_environ() result = prepare_without_interaction(project, environ=environ, command=command) assert result.errors == [] assert result assert os.path.join(project.directory_path, 'foo.py') in result.command_exec_info.args