def teardown_package(): import os if os.environ.get('DATALAD_TESTS_NOTEARDOWN'): return from datalad.ui import ui ui.set_backend(_test_states['ui_backend']) if _test_states['loglevel'] is not None: lgr.setLevel(_test_states['loglevel']) if _test_states['DATALAD_LOG_LEVEL'] is None: os.environ.pop('DATALAD_LOG_LEVEL') else: os.environ['DATALAD_LOG_LEVEL'] = _test_states['DATALAD_LOG_LEVEL'] from datalad.tests import _TEMP_PATHS_GENERATED from datalad.tests.utils import rmtemp if len(_TEMP_PATHS_GENERATED): msg = "Removing %d dirs/files: %s" % (len(_TEMP_PATHS_GENERATED), ', '.join(_TEMP_PATHS_GENERATED)) else: msg = "Nothing to remove" lgr.debug("Teardown tests. " + msg) for path in _TEMP_PATHS_GENERATED: rmtemp(path, ignore_errors=True) lgr.debug("Printing versioning information collected so far") from datalad.support.external_versions import external_versions as ev print(ev.dumps(query=True))
def teardown_package(): import os if os.environ.get('DATALAD_TESTS_NOTEARDOWN'): return from datalad.ui import ui ui.set_backend(_test_states['ui_backend']) if _test_states['loglevel'] is not None: lgr.setLevel(_test_states['loglevel']) if _test_states['DATALAD_LOG_LEVEL'] is None: os.environ.pop('DATALAD_LOG_LEVEL') else: os.environ['DATALAD_LOG_LEVEL'] = _test_states['DATALAD_LOG_LEVEL'] from datalad.tests import _TEMP_PATHS_GENERATED from datalad.tests.utils import rmtemp if len(_TEMP_PATHS_GENERATED): msg = "Removing %d dirs/files: %s" % (len(_TEMP_PATHS_GENERATED), ', '.join(_TEMP_PATHS_GENERATED)) else: msg = "Nothing to remove" lgr.debug("Teardown tests. " + msg) for path in _TEMP_PATHS_GENERATED: rmtemp(path, ignore_errors=True) lgr.debug("Printing versioning information collected so far") from datalad.support.external_versions import external_versions as ev # request versioning for few others which we do not check at runtime for m in ('git', 'system-ssh'): try: # Let's make sure to not blow up when we are almost done ev[m] except Exception: pass print(ev.dumps(query=True))
def setup_package(): import os # To overcome pybuild overriding HOME but us possibly wanting our # own HOME where we pre-setup git for testing (name, email) if 'GIT_HOME' in os.environ: os.environ['HOME'] = os.environ['GIT_HOME'] # To overcome pybuild by default defining http{,s}_proxy we would need # to define them to e.g. empty value so it wouldn't bother touching them. # But then haskell libraries do not digest empty value nicely, so we just # pop them out from the environment for ev in ('http_proxy', 'https_proxy'): if ev in os.environ and not (os.environ[ev]): lgr.debug("Removing %s from the environment since it is empty", ev) os.environ.pop(ev) DATALAD_LOG_LEVEL = os.environ.get('DATALAD_LOG_LEVEL', None) if DATALAD_LOG_LEVEL is None: # very very silent. Tests introspecting logs should use # swallow_logs(new_level=...) _test_states['loglevel'] = lgr.getEffectiveLevel() lgr.setLevel(100) # And we should also set it within environ so underlying commands also stay silent _test_states['DATALAD_LOG_LEVEL'] = DATALAD_LOG_LEVEL os.environ['DATALAD_LOG_LEVEL'] = '100' else: # We are not overriding them, since explicitly were asked to have some log level _test_states['loglevel'] = None # Set to non-interactive UI from datalad.ui import ui _test_states['ui_backend'] = ui.backend ui.set_backend('tests-noninteractive')
def teardown_package(): import os from datalad.tests.utils import rmtemp, OBSCURE_FILENAME lgr.debug("Printing versioning information collected so far") from datalad.support.external_versions import external_versions as ev print(ev.dumps(query=True)) try: print("Obscure filename: str=%s repr=%r" % (OBSCURE_FILENAME.encode('utf-8'), OBSCURE_FILENAME)) except UnicodeEncodeError as exc: from .dochelpers import exc_str print("Obscure filename failed to print: %s" % exc_str(exc)) def print_dict(d): return " ".join("%s=%r" % v for v in d.items()) print("Encodings: %s" % print_dict(get_encoding_info())) print("Environment: %s" % print_dict(get_envvars_info())) if os.environ.get('DATALAD_TESTS_NOTEARDOWN'): return from datalad.ui import ui from datalad import consts ui.set_backend(_test_states['ui_backend']) if _test_states['loglevel'] is not None: lgr.setLevel(_test_states['loglevel']) from datalad.tests import _TEMP_PATHS_GENERATED if len(_TEMP_PATHS_GENERATED): msg = "Removing %d dirs/files: %s" % (len(_TEMP_PATHS_GENERATED), ', '.join(_TEMP_PATHS_GENERATED)) else: msg = "Nothing to remove" lgr.debug("Teardown tests. " + msg) for path in _TEMP_PATHS_GENERATED: rmtemp(path, ignore_errors=True) # restore all the env variables for v, val in _test_states['env'].items(): if val is not None: os.environ[v] = val else: os.environ.pop(v) # Re-establish correct global config after changing $HOME. # Might be superfluous, since after teardown datalad.cfg shouldn't be # needed. However, maintaining a consistent state seems a good thing # either way. cfg.reload(force=True) consts.DATASETS_TOPURL = _test_states['DATASETS_TOPURL'] from datalad.support.cookies import cookies_db cookies_db.close() from datalad.support.annexrepo import AnnexRepo AnnexRepo._ALLOW_LOCAL_URLS = False # stay safe!
def teardown_package(): import os from datalad.tests.utils import rmtemp, OBSCURE_FILENAME lgr.debug("Printing versioning information collected so far") from datalad.support.external_versions import external_versions as ev print(ev.dumps(query=True)) try: print("Obscure filename: str=%s repr=%r" % (OBSCURE_FILENAME.encode('utf-8'), OBSCURE_FILENAME)) except UnicodeEncodeError as exc: from .dochelpers import exc_str print("Obscure filename failed to print: %s" % exc_str(exc)) def print_dict(d): return " ".join("%s=%r" % v for v in d.items()) print("Encodings: %s" % print_dict(get_encoding_info())) print("Environment: %s" % print_dict(get_envvars_info())) if os.environ.get('DATALAD_TESTS_NOTEARDOWN'): return from datalad.ui import ui from datalad import consts ui.set_backend(_test_states['ui_backend']) if _test_states['loglevel'] is not None: lgr.setLevel(_test_states['loglevel']) if _test_states['DATALAD_LOG_LEVEL'] is None: os.environ.pop('DATALAD_LOG_LEVEL') else: os.environ['DATALAD_LOG_LEVEL'] = _test_states['DATALAD_LOG_LEVEL'] from datalad.tests import _TEMP_PATHS_GENERATED if len(_TEMP_PATHS_GENERATED): msg = "Removing %d dirs/files: %s" % (len(_TEMP_PATHS_GENERATED), ', '.join(_TEMP_PATHS_GENERATED)) else: msg = "Nothing to remove" lgr.debug("Teardown tests. " + msg) for path in _TEMP_PATHS_GENERATED: rmtemp(path, ignore_errors=True) if _test_states['HOME'] is not None: os.environ['HOME'] = _test_states['HOME'] if _test_states['DATASETS_TOPURL_ENV']: os.environ['DATALAD_DATASETS_TOPURL'] = _test_states[ 'DATASETS_TOPURL_ENV'] consts.DATASETS_TOPURL = _test_states['DATASETS_TOPURL'] from datalad.support.cookies import cookies_db cookies_db.close() from datalad.support.annexrepo import AnnexRepo AnnexRepo._ALLOW_LOCAL_URLS = False # stay safe!
def newfunc(*args, **kwargs): from datalad.ui import ui old_backend = ui.backend try: ui.set_backend('tests' if interactive else 'tests-noninteractive') if responses: ui.add_responses(responses) ret = t(*args, **kwargs) if responses: responses_left = ui.get_responses() assert not len(responses_left), "Some responses were left not used: %s" % str(responses_left) return ret finally: ui.set_backend(old_backend)
def newfunc(*args, **kwargs): from datalad.ui import ui old_backend = ui.backend try: ui.set_backend('tests' if interactive else 'tests-noninteractive') if responses: ui.add_responses(responses) ret = t(*args, **kwargs) if responses: responses_left = ui.get_responses() assert not len(responses_left), "Some responses were left not used: %s" % str(responses_left) return ret finally: ui.set_backend(old_backend)
def teardown_package(): import os from datalad.tests.utils import rmtemp, OBSCURE_FILENAME lgr.debug("Printing versioning information collected so far") from datalad.support.external_versions import external_versions as ev print(ev.dumps(query=True)) try: print("Obscure filename: str=%s repr=%r" % (OBSCURE_FILENAME.encode('utf-8'), OBSCURE_FILENAME)) except UnicodeEncodeError as exc: from .dochelpers import exc_str print("Obscure filename failed to print: %s" % exc_str(exc)) def print_dict(d): return " ".join("%s=%r" % v for v in d.items()) print("Encodings: %s" % print_dict(get_encoding_info())) print("Environment: %s" % print_dict(get_envvars_info())) if os.environ.get('DATALAD_TESTS_NOTEARDOWN'): return from datalad.ui import ui from datalad import consts ui.set_backend(_test_states['ui_backend']) if _test_states['loglevel'] is not None: lgr.setLevel(_test_states['loglevel']) if _test_states['DATALAD_LOG_LEVEL'] is None: os.environ.pop('DATALAD_LOG_LEVEL') else: os.environ['DATALAD_LOG_LEVEL'] = _test_states['DATALAD_LOG_LEVEL'] from datalad.tests import _TEMP_PATHS_GENERATED if len(_TEMP_PATHS_GENERATED): msg = "Removing %d dirs/files: %s" % (len(_TEMP_PATHS_GENERATED), ', '.join(_TEMP_PATHS_GENERATED)) else: msg = "Nothing to remove" lgr.debug("Teardown tests. " + msg) for path in _TEMP_PATHS_GENERATED: rmtemp(path, ignore_errors=True) if _test_states['HOME'] is not None: os.environ['HOME'] = _test_states['HOME'] if _test_states['DATASETS_TOPURL_ENV']: os.environ['DATALAD_DATASETS_TOPURL'] = _test_states['DATASETS_TOPURL_ENV'] consts.DATASETS_TOPURL = _test_states['DATASETS_TOPURL'] from datalad.support.cookies import cookies_db cookies_db.close() from datalad.support.annexrepo import AnnexRepo AnnexRepo._ALLOW_LOCAL_URLS = False # stay safe!
def main(args=None, cls=None, remote_name=None, description=None): import sys parser = setup_parser(remote_name, description) # parse cmd args args = parser.parse_args(args) # stdin/stdout will be used for interactions with annex ui.set_backend('annex') if args.common_debug: # So we could see/stop clearly at the point of failure setup_exceptionhook() _main(args, cls) else: # Otherwise - guard and only log the summary. Postmortem is not # as convenient if being caught in this ultimate except try: _main(args, cls) except Exception as exc: lgr.error('%s (%s)', str(exc), exc.__class__.__name__) sys.exit(1)
def teardown_package(): import os if os.environ.get('DATALAD_TESTS_NOTEARDOWN'): return from datalad.ui import ui ui.set_backend(_test_states['ui_backend']) if _test_states['loglevel'] is not None: lgr.setLevel(_test_states['loglevel']) if _test_states['DATALAD_LOG_LEVEL'] is None: os.environ.pop('DATALAD_LOG_LEVEL') else: os.environ['DATALAD_LOG_LEVEL'] = _test_states['DATALAD_LOG_LEVEL'] from datalad.tests import _TEMP_PATHS_GENERATED from datalad.tests.utils import rmtemp if len(_TEMP_PATHS_GENERATED): msg = "Removing %d dirs/files: %s" % (len(_TEMP_PATHS_GENERATED), ', '.join(_TEMP_PATHS_GENERATED)) else: msg = "Nothing to remove" lgr.debug("Teardown tests. " + msg) for path in _TEMP_PATHS_GENERATED: rmtemp(path, ignore_errors=True)
def setup_package(): import os from datalad import consts _test_states['HOME'] = os.environ.get('HOME', None) _test_states['DATASETS_TOPURL_ENV'] = os.environ.get( 'DATALAD_DATASETS_TOPURL', None) _test_states['DATASETS_TOPURL'] = consts.DATASETS_TOPURL os.environ[ 'DATALAD_DATASETS_TOPURL'] = consts.DATASETS_TOPURL = 'http://datasets-tests.datalad.org/' # To overcome pybuild overriding HOME but us possibly wanting our # own HOME where we pre-setup git for testing (name, email) if 'GIT_HOME' in os.environ: os.environ['HOME'] = os.environ['GIT_HOME'] else: # we setup our own new HOME, the BEST and HUGE one from datalad.utils import make_tempfile from datalad.tests import _TEMP_PATHS_GENERATED # TODO: split into a function + context manager with make_tempfile(mkdir=True) as new_home: os.environ['HOME'] = new_home if not os.path.exists(new_home): os.makedirs(new_home) with open(os.path.join(new_home, '.gitconfig'), 'w') as f: f.write("""\ [user] name = DataLad Tester email = [email protected] """) _TEMP_PATHS_GENERATED.append(new_home) # For now we will just verify that it is ready to run the tests from datalad.support.gitrepo import check_git_configured check_git_configured() # To overcome pybuild by default defining http{,s}_proxy we would need # to define them to e.g. empty value so it wouldn't bother touching them. # But then haskell libraries do not digest empty value nicely, so we just # pop them out from the environment for ev in ('http_proxy', 'https_proxy'): if ev in os.environ and not (os.environ[ev]): lgr.debug("Removing %s from the environment since it is empty", ev) os.environ.pop(ev) # During tests we allow for "insecure" access to local file:// and # http://localhost URLs since all of them either generated as tests # fixtures or cloned from trusted sources from datalad.support.annexrepo import AnnexRepo AnnexRepo._ALLOW_LOCAL_URLS = True DATALAD_LOG_LEVEL = os.environ.get('DATALAD_LOG_LEVEL', None) if DATALAD_LOG_LEVEL is None: # very very silent. Tests introspecting logs should use # swallow_logs(new_level=...) _test_states['loglevel'] = lgr.getEffectiveLevel() lgr.setLevel(100) # And we should also set it within environ so underlying commands also stay silent _test_states['DATALAD_LOG_LEVEL'] = DATALAD_LOG_LEVEL os.environ['DATALAD_LOG_LEVEL'] = '100' else: # We are not overriding them, since explicitly were asked to have some log level _test_states['loglevel'] = None # Set to non-interactive UI from datalad.ui import ui _test_states['ui_backend'] = ui.backend # obtain() since that one consults for the default value ui.set_backend(cfg.obtain('datalad.tests.ui.backend'))
def setup_package(): import os from datalad.utils import on_osx from datalad.tests import _TEMP_PATHS_GENERATED if on_osx: # enforce honoring TMPDIR (see gh-5307) import tempfile tempfile.tempdir = os.environ.get('TMPDIR', tempfile.gettempdir()) from datalad import consts _test_states['env'] = {} def set_envvar(v, val): """Memoize and then set env var""" _test_states['env'][v] = os.environ.get(v, None) os.environ[v] = val _test_states['DATASETS_TOPURL'] = consts.DATASETS_TOPURL consts.DATASETS_TOPURL = 'http://datasets-tests.datalad.org/' set_envvar('DATALAD_DATASETS_TOPURL', consts.DATASETS_TOPURL) from datalad.tests.utils import ( DEFAULT_BRANCH, DEFAULT_REMOTE, ) set_envvar( "GIT_CONFIG_PARAMETERS", "'init.defaultBranch={}' 'clone.defaultRemoteName={}'".format( DEFAULT_BRANCH, DEFAULT_REMOTE)) # To overcome pybuild overriding HOME but us possibly wanting our # own HOME where we pre-setup git for testing (name, email) if 'GIT_HOME' in os.environ: set_envvar('HOME', os.environ['GIT_HOME']) set_envvar('DATALAD_LOG_EXC', "1") else: # we setup our own new HOME, the BEST and HUGE one from datalad.utils import make_tempfile # TODO: split into a function + context manager with make_tempfile(mkdir=True) as new_home: pass for v, val in get_home_envvars(new_home).items(): set_envvar(v, val) if not os.path.exists(new_home): os.makedirs(new_home) with open(os.path.join(new_home, '.gitconfig'), 'w') as f: f.write("""\ [user] name = DataLad Tester email = [email protected] [datalad "log"] exc = 1 """) _TEMP_PATHS_GENERATED.append(new_home) # Re-load ConfigManager, since otherwise it won't consider global config # from new $HOME (see gh-4153 cfg.reload(force=True) from datalad.interface.common_cfg import compute_cfg_defaults compute_cfg_defaults() # datalad.locations.sockets has likely changed. Discard any cached values. ssh_manager._socket_dir = None # To overcome pybuild by default defining http{,s}_proxy we would need # to define them to e.g. empty value so it wouldn't bother touching them. # But then haskell libraries do not digest empty value nicely, so we just # pop them out from the environment for ev in ('http_proxy', 'https_proxy'): if ev in os.environ and not (os.environ[ev]): lgr.debug("Removing %s from the environment since it is empty", ev) os.environ.pop(ev) # During tests we allow for "insecure" access to local file:// and # http://localhost URLs since all of them either generated as tests # fixtures or cloned from trusted sources from datalad.support.annexrepo import AnnexRepo AnnexRepo._ALLOW_LOCAL_URLS = True DATALAD_LOG_LEVEL = os.environ.get('DATALAD_LOG_LEVEL', None) if DATALAD_LOG_LEVEL is None: # very very silent. Tests introspecting logs should use # swallow_logs(new_level=...) _test_states['loglevel'] = lgr.getEffectiveLevel() lgr.setLevel(100) # And we should also set it within environ so underlying commands also stay silent set_envvar('DATALAD_LOG_LEVEL', '100') else: # We are not overriding them, since explicitly were asked to have some log level _test_states['loglevel'] = None # Set to non-interactive UI from datalad.ui import ui _test_states['ui_backend'] = ui.backend # obtain() since that one consults for the default value ui.set_backend(cfg.obtain('datalad.tests.ui.backend')) # Monkey patch nose so it does not ERROR out whenever code asks for fileno # of the output. See https://github.com/nose-devs/nose/issues/6 from io import StringIO as OrigStringIO class StringIO(OrigStringIO): fileno = lambda self: 1 encoding = None from nose.ext import dtcompat from nose.plugins import capture, multiprocess, plugintest dtcompat.StringIO = StringIO capture.StringIO = StringIO multiprocess.StringIO = StringIO plugintest.StringIO = StringIO # in order to avoid having to fiddle with rather uncommon # file:// URLs in the tests, have a standard HTTP server # that serves an 'httpserve' directory in the test HOME # the URL will be available from datalad.test_http_server.url from datalad.tests.utils import HTTPPath import tempfile global test_http_server serve_path = tempfile.mkdtemp( dir=cfg.get("datalad.tests.temp.dir"), prefix='httpserve', ) test_http_server = HTTPPath(serve_path) test_http_server.start() _TEMP_PATHS_GENERATED.append(serve_path) if cfg.obtain('datalad.tests.setup.testrepos'): lgr.debug("Pre-populating testrepos") from datalad.tests.utils import with_testrepos with_testrepos()(lambda repo: 1)()
def setup_package(): import os from datalad import consts _test_states['HOME'] = os.environ.get('HOME', None) _test_states['DATASETS_TOPURL_ENV'] = os.environ.get('DATALAD_DATASETS_TOPURL', None) _test_states['DATASETS_TOPURL'] = consts.DATASETS_TOPURL os.environ['DATALAD_DATASETS_TOPURL'] = consts.DATASETS_TOPURL = 'http://datasets-tests.datalad.org/' # To overcome pybuild overriding HOME but us possibly wanting our # own HOME where we pre-setup git for testing (name, email) if 'GIT_HOME' in os.environ: os.environ['HOME'] = os.environ['GIT_HOME'] else: # we setup our own new HOME, the BEST and HUGE one from datalad.utils import make_tempfile from datalad.tests import _TEMP_PATHS_GENERATED # TODO: split into a function + context manager with make_tempfile(mkdir=True) as new_home: os.environ['HOME'] = new_home if not os.path.exists(new_home): os.makedirs(new_home) with open(os.path.join(new_home, '.gitconfig'), 'w') as f: f.write("""\ [user] name = DataLad Tester email = [email protected] """) _TEMP_PATHS_GENERATED.append(new_home) # If there is a bundled git, make sure GitPython uses it too # (some parts of the test utilities still rely on GitPython) from datalad.cmd import GitRunner GitRunner._check_git_path() if GitRunner._GIT_PATH: import os os.environ['GIT_PYTHON_GIT_EXECUTABLE'] = \ os.path.join(GitRunner._GIT_PATH, 'git') # Re-load ConfigManager, since otherwise it won't consider global config # from new $HOME (see gh-4153 cfg.reload(force=True) # To overcome pybuild by default defining http{,s}_proxy we would need # to define them to e.g. empty value so it wouldn't bother touching them. # But then haskell libraries do not digest empty value nicely, so we just # pop them out from the environment for ev in ('http_proxy', 'https_proxy'): if ev in os.environ and not (os.environ[ev]): lgr.debug("Removing %s from the environment since it is empty", ev) os.environ.pop(ev) # During tests we allow for "insecure" access to local file:// and # http://localhost URLs since all of them either generated as tests # fixtures or cloned from trusted sources from datalad.support.annexrepo import AnnexRepo AnnexRepo._ALLOW_LOCAL_URLS = True DATALAD_LOG_LEVEL = os.environ.get('DATALAD_LOG_LEVEL', None) if DATALAD_LOG_LEVEL is None: # very very silent. Tests introspecting logs should use # swallow_logs(new_level=...) _test_states['loglevel'] = lgr.getEffectiveLevel() lgr.setLevel(100) # And we should also set it within environ so underlying commands also stay silent _test_states['DATALAD_LOG_LEVEL'] = DATALAD_LOG_LEVEL os.environ['DATALAD_LOG_LEVEL'] = '100' else: # We are not overriding them, since explicitly were asked to have some log level _test_states['loglevel'] = None # Set to non-interactive UI from datalad.ui import ui _test_states['ui_backend'] = ui.backend # obtain() since that one consults for the default value ui.set_backend(cfg.obtain('datalad.tests.ui.backend')) # Monkey patch nose so it does not ERROR out whenever code asks for fileno # of the output. See https://github.com/nose-devs/nose/issues/6 from io import StringIO as OrigStringIO class StringIO(OrigStringIO): fileno = lambda self: 1 encoding = None from nose.ext import dtcompat from nose.plugins import capture, multiprocess, plugintest dtcompat.StringIO = StringIO capture.StringIO = StringIO multiprocess.StringIO = StringIO plugintest.StringIO = StringIO
def setup_package(): import tempfile from pathlib import Path from datalad import consts from datalad.support.annexrepo import AnnexRepo from datalad.support.cookies import cookies_db from datalad.support.external_versions import external_versions from datalad.tests import _TEMP_PATHS_GENERATED from datalad.tests.utils_pytest import ( DEFAULT_BRANCH, DEFAULT_REMOTE, OBSCURE_FILENAME, HTTPPath, rmtemp, ) from datalad.ui import ui from datalad.utils import ( make_tempfile, on_osx, ) if on_osx: # enforce honoring TMPDIR (see gh-5307) tempfile.tempdir = os.environ.get('TMPDIR', tempfile.gettempdir()) with pytest.MonkeyPatch().context() as m: m.setattr(consts, "DATASETS_TOPURL", 'https://datasets-tests.datalad.org/') m.setenv('DATALAD_DATASETS_TOPURL', consts.DATASETS_TOPURL) m.setenv( "GIT_CONFIG_PARAMETERS", "'init.defaultBranch={}' 'clone.defaultRemoteName={}'".format( DEFAULT_BRANCH, DEFAULT_REMOTE)) def prep_tmphome(): # re core.askPass: # Don't let git ask for credentials in CI runs. Note, that this variable # technically is not a flag, but an executable (which is why name and value # are a bit confusing here - we just want a no-op basically). The environment # variable GIT_ASKPASS overwrites this, but neither env var nor this config # are supported by git-credential on all systems and git versions (most recent # ones should work either way, though). Hence use both across CI builds. gitconfig = """\ [user] name = DataLad Tester email = [email protected] [core] askPass = [datalad "log"] exc = 1 [annex "security"] # from annex 6.20180626 file:/// and http://localhost access isn't # allowed by default allowed-url-schemes = http https file allowed-http-addresses = all """ # TODO: split into a function + context manager with make_tempfile(mkdir=True) as new_home: pass # register for clean-up on exit _TEMP_PATHS_GENERATED.append(new_home) # populate default config new_home = Path(new_home) new_home.mkdir(parents=True, exist_ok=True) cfg_file = new_home / '.gitconfig' cfg_file.write_text(gitconfig) return new_home, cfg_file if external_versions['cmd:git'] < "2.32": # To overcome pybuild overriding HOME but us possibly wanting our # own HOME where we pre-setup git for testing (name, email) if 'GIT_HOME' in os.environ: m.setenv('HOME', os.environ['GIT_HOME']) else: # we setup our own new HOME, the BEST and HUGE one new_home, _ = prep_tmphome() for v, val in get_home_envvars(new_home).items(): m.setenv(v, val) else: _, cfg_file = prep_tmphome() m.setenv('GIT_CONFIG_GLOBAL', str(cfg_file)) # Re-load ConfigManager, since otherwise it won't consider global config # from new $HOME (see gh-4153 cfg.reload(force=True) # datalad.locations.sockets has likely changed. Discard any cached values. ssh_manager._socket_dir = None # To overcome pybuild by default defining http{,s}_proxy we would need # to define them to e.g. empty value so it wouldn't bother touching them. # But then haskell libraries do not digest empty value nicely, so we just # pop them out from the environment for ev in ('http_proxy', 'https_proxy'): if ev in os.environ and not (os.environ[ev]): lgr.debug("Removing %s from the environment since it is empty", ev) os.environ.pop(ev) # Prevent interactive credential entry (note "true" is the command to run) # See also the core.askPass setting above m.setenv('GIT_ASKPASS', 'true') # Set to non-interactive UI _test_states['ui_backend'] = ui.backend # obtain() since that one consults for the default value ui.set_backend(cfg.obtain('datalad.tests.ui.backend')) # in order to avoid having to fiddle with rather uncommon # file:// URLs in the tests, have a standard HTTP server # that serves an 'httpserve' directory in the test HOME # the URL will be available from datalad.test_http_server.url global test_http_server # Start the server only if not running already # Relevant: we have test_misc.py:test_test which runs datalad.test but # not doing teardown, so the original server might never get stopped if test_http_server is None: serve_path = tempfile.mkdtemp( dir=cfg.get("datalad.tests.temp.dir"), prefix='httpserve', ) test_http_server = HTTPPath(serve_path) test_http_server.start() _TEMP_PATHS_GENERATED.append(serve_path) if cfg.obtain('datalad.tests.setup.testrepos'): lgr.debug("Pre-populating testrepos") from datalad.tests.utils_pytest import with_testrepos with_testrepos()(lambda repo: 1)() yield lgr.debug("Printing versioning information collected so far") # Query for version of datalad, so it is included in ev.dumps below - useful while # testing extensions where version of datalad might differ in the environment. external_versions['datalad'] print(external_versions.dumps(query=True)) try: print("Obscure filename: str=%s repr=%r" % (OBSCURE_FILENAME.encode('utf-8'), OBSCURE_FILENAME)) except UnicodeEncodeError as exc: ce = CapturedException(exc) print("Obscure filename failed to print: %s" % ce) def print_dict(d): return " ".join("%s=%r" % v for v in d.items()) print("Encodings: %s" % print_dict(get_encoding_info())) print("Environment: %s" % print_dict(get_envvars_info())) if os.environ.get('DATALAD_TESTS_NOTEARDOWN'): return ui.set_backend(_test_states['ui_backend']) if test_http_server: test_http_server.stop() test_http_server = None else: lgr.debug( "For some reason global http_server was not set/running, thus not stopping" ) if len(_TEMP_PATHS_GENERATED): msg = "Removing %d dirs/files: %s" % ( len(_TEMP_PATHS_GENERATED), ', '.join(_TEMP_PATHS_GENERATED)) else: msg = "Nothing to remove" lgr.debug("Teardown tests. " + msg) for path in _TEMP_PATHS_GENERATED: rmtemp(str(path), ignore_errors=True) # Re-establish correct global config after changing $HOME. # Might be superfluous, since after teardown datalad.cfg shouldn't be # needed. However, maintaining a consistent state seems a good thing # either way. cfg.reload(force=True) ssh_manager._socket_dir = None cookies_db.close()
def setup_package(): import os from datalad import consts _test_states['HOME'] = os.environ.get('HOME', None) _test_states['DATASETS_TOPURL_ENV'] = os.environ.get('DATALAD_DATASETS_TOPURL', None) _test_states['DATASETS_TOPURL'] = consts.DATASETS_TOPURL os.environ['DATALAD_DATASETS_TOPURL'] = consts.DATASETS_TOPURL = 'http://datasets-tests.datalad.org/' # To overcome pybuild overriding HOME but us possibly wanting our # own HOME where we pre-setup git for testing (name, email) if 'GIT_HOME' in os.environ: os.environ['HOME'] = os.environ['GIT_HOME'] else: # we setup our own new HOME, the BEST and HUGE one from datalad.utils import make_tempfile from datalad.tests import _TEMP_PATHS_GENERATED # TODO: split into a function + context manager with make_tempfile(mkdir=True) as new_home: os.environ['HOME'] = new_home if not os.path.exists(new_home): os.makedirs(new_home) with open(os.path.join(new_home, '.gitconfig'), 'w') as f: f.write("""\ [user] name = DataLad Tester email = [email protected] """) _TEMP_PATHS_GENERATED.append(new_home) # For now we will just verify that it is ready to run the tests from datalad.support.gitrepo import check_git_configured check_git_configured() # To overcome pybuild by default defining http{,s}_proxy we would need # to define them to e.g. empty value so it wouldn't bother touching them. # But then haskell libraries do not digest empty value nicely, so we just # pop them out from the environment for ev in ('http_proxy', 'https_proxy'): if ev in os.environ and not (os.environ[ev]): lgr.debug("Removing %s from the environment since it is empty", ev) os.environ.pop(ev) # During tests we allow for "insecure" access to local file:// and # http://localhost URLs since all of them either generated as tests # fixtures or cloned from trusted sources from datalad.support.annexrepo import AnnexRepo AnnexRepo._ALLOW_LOCAL_URLS = True DATALAD_LOG_LEVEL = os.environ.get('DATALAD_LOG_LEVEL', None) if DATALAD_LOG_LEVEL is None: # very very silent. Tests introspecting logs should use # swallow_logs(new_level=...) _test_states['loglevel'] = lgr.getEffectiveLevel() lgr.setLevel(100) # And we should also set it within environ so underlying commands also stay silent _test_states['DATALAD_LOG_LEVEL'] = DATALAD_LOG_LEVEL os.environ['DATALAD_LOG_LEVEL'] = '100' else: # We are not overriding them, since explicitly were asked to have some log level _test_states['loglevel'] = None # Set to non-interactive UI from datalad.ui import ui _test_states['ui_backend'] = ui.backend # obtain() since that one consults for the default value ui.set_backend(cfg.obtain('datalad.tests.ui.backend'))
def _action_store(attrs, interactive, providers): # Determine the defaults to use for storing. In non-interactive mode, # this is what's going to be stored, in interactive mode user is # presented with them as default choice. # We don't really know what authentication type makes sense to store in # the provider config (this would be relevant for datalad using those # credentials w/o git). # However, pick 'http_basic_auth' in case of HTTP(S) URL and 'none' # otherwise as the default to pass into store routine. if attrs.get('protocol') in ['http', 'https']: authentication_type = 'http_basic_auth' else: authentication_type = 'none' # If we got a `path` component from git, usehttppath is set and thereby # git was instructed to include it when matching. Hence, do the same. url_re = "{pr}://{h}{p}.*".format(pr=attrs.get('protocol'), h=attrs.get('host'), p="/" + attrs.get('path') if "path" in attrs.keys() else "") name = attrs.get('host') credential_name = name credential_type = "user_password" if not interactive: # TODO: What about credential labels? This could already exist as # well. However, it's unlikely since the respective # "service name" for keyring is prepended with "datalad-". # For the same reason of how the keyring system is used by # datalad it's not very transparently accessible what labels # we'd need to check for. Rather than encoding knowledge about # datalad's internal handling here, let's address that in # datalad's provider and credential classes and have an easy # check to be called from here. if any(p.name == name for p in providers): print(f"Provider name '{name}' already exists. This can't be " "resolved in non-interactive mode.", file=sys.stderr) authenticator_class = AUTHENTICATION_TYPES[authentication_type] saved_provider = providers._store_new( url=attrs.get('url'), authentication_type=authentication_type, authenticator_class=authenticator_class, url_re=url_re, name=name, credential_name=credential_name, credential_type=credential_type, level='user' ) else: # use backend made for annex special remotes for interaction from a # subprocess whose stdin/stdout are in use for communication with # its parent from datalad.ui import ui ui.set_backend('annex') # ensure default is first in list (that's how `enter_new` determines # it's the default) auth_list = AUTHENTICATION_TYPES.copy() auth_list.pop(authentication_type, None) auth_list = [authentication_type] + list(auth_list.keys()) saved_provider = providers.enter_new( url=attrs.get('url'), auth_types=auth_list, url_re=url_re, name=name, credential_name=credential_name, credential_type=credential_type) saved_provider.credential.set(user=attrs['username'], password=attrs['password'])