def teardown_package():
import os
if os.environ.get('DATALAD_TESTS_NOTEARDOWN'):
return
from datalad.ui import ui
ui.set_backend(_test_states['ui_backend'])
if _test_states['loglevel'] is not None:
lgr.setLevel(_test_states['loglevel'])
if _test_states['DATALAD_LOG_LEVEL'] is None:
os.environ.pop('DATALAD_LOG_LEVEL')
else:
os.environ['DATALAD_LOG_LEVEL'] = _test_states['DATALAD_LOG_LEVEL']
from datalad.tests import _TEMP_PATHS_GENERATED
from datalad.tests.utils import rmtemp
if len(_TEMP_PATHS_GENERATED):
msg = "Removing %d dirs/files: %s" % (len(_TEMP_PATHS_GENERATED), ', '.join(_TEMP_PATHS_GENERATED))
else:
msg = "Nothing to remove"
lgr.debug("Teardown tests. " + msg)
for path in _TEMP_PATHS_GENERATED:
rmtemp(path, ignore_errors=True)
lgr.debug("Printing versioning information collected so far")
from datalad.support.external_versions import external_versions as ev
print(ev.dumps(query=True))
def teardown_package():
import os
if os.environ.get('DATALAD_TESTS_NOTEARDOWN'):
return
from datalad.ui import ui
ui.set_backend(_test_states['ui_backend'])
if _test_states['loglevel'] is not None:
lgr.setLevel(_test_states['loglevel'])
if _test_states['DATALAD_LOG_LEVEL'] is None:
os.environ.pop('DATALAD_LOG_LEVEL')
else:
os.environ['DATALAD_LOG_LEVEL'] = _test_states['DATALAD_LOG_LEVEL']
from datalad.tests import _TEMP_PATHS_GENERATED
from datalad.tests.utils import rmtemp
if len(_TEMP_PATHS_GENERATED):
msg = "Removing %d dirs/files: %s" % (len(_TEMP_PATHS_GENERATED),
', '.join(_TEMP_PATHS_GENERATED))
else:
msg = "Nothing to remove"
lgr.debug("Teardown tests. " + msg)
for path in _TEMP_PATHS_GENERATED:
rmtemp(path, ignore_errors=True)
lgr.debug("Printing versioning information collected so far")
from datalad.support.external_versions import external_versions as ev
# request versioning for few others which we do not check at runtime
for m in ('git', 'system-ssh'):
try: # Let's make sure to not blow up when we are almost done
ev[m]
except Exception:
pass
print(ev.dumps(query=True))
def setup_package():
import os
# To overcome pybuild overriding HOME but us possibly wanting our
# own HOME where we pre-setup git for testing (name, email)
if 'GIT_HOME' in os.environ:
os.environ['HOME'] = os.environ['GIT_HOME']
# To overcome pybuild by default defining http{,s}_proxy we would need
# to define them to e.g. empty value so it wouldn't bother touching them.
# But then haskell libraries do not digest empty value nicely, so we just
# pop them out from the environment
for ev in ('http_proxy', 'https_proxy'):
if ev in os.environ and not (os.environ[ev]):
lgr.debug("Removing %s from the environment since it is empty", ev)
os.environ.pop(ev)
DATALAD_LOG_LEVEL = os.environ.get('DATALAD_LOG_LEVEL', None)
if DATALAD_LOG_LEVEL is None:
# very very silent. Tests introspecting logs should use
# swallow_logs(new_level=...)
_test_states['loglevel'] = lgr.getEffectiveLevel()
lgr.setLevel(100)
# And we should also set it within environ so underlying commands also stay silent
_test_states['DATALAD_LOG_LEVEL'] = DATALAD_LOG_LEVEL
os.environ['DATALAD_LOG_LEVEL'] = '100'
else:
# We are not overriding them, since explicitly were asked to have some log level
_test_states['loglevel'] = None
# Set to non-interactive UI
from datalad.ui import ui
_test_states['ui_backend'] = ui.backend
ui.set_backend('tests-noninteractive')
def teardown_package():
import os
from datalad.tests.utils import rmtemp, OBSCURE_FILENAME
lgr.debug("Printing versioning information collected so far")
from datalad.support.external_versions import external_versions as ev
print(ev.dumps(query=True))
try:
print("Obscure filename: str=%s repr=%r" %
(OBSCURE_FILENAME.encode('utf-8'), OBSCURE_FILENAME))
except UnicodeEncodeError as exc:
from .dochelpers import exc_str
print("Obscure filename failed to print: %s" % exc_str(exc))
def print_dict(d):
return " ".join("%s=%r" % v for v in d.items())
print("Encodings: %s" % print_dict(get_encoding_info()))
print("Environment: %s" % print_dict(get_envvars_info()))
if os.environ.get('DATALAD_TESTS_NOTEARDOWN'):
return
from datalad.ui import ui
from datalad import consts
ui.set_backend(_test_states['ui_backend'])
if _test_states['loglevel'] is not None:
lgr.setLevel(_test_states['loglevel'])
from datalad.tests import _TEMP_PATHS_GENERATED
if len(_TEMP_PATHS_GENERATED):
msg = "Removing %d dirs/files: %s" % (len(_TEMP_PATHS_GENERATED),
', '.join(_TEMP_PATHS_GENERATED))
else:
msg = "Nothing to remove"
lgr.debug("Teardown tests. " + msg)
for path in _TEMP_PATHS_GENERATED:
rmtemp(path, ignore_errors=True)
# restore all the env variables
for v, val in _test_states['env'].items():
if val is not None:
os.environ[v] = val
else:
os.environ.pop(v)
# Re-establish correct global config after changing $HOME.
# Might be superfluous, since after teardown datalad.cfg shouldn't be
# needed. However, maintaining a consistent state seems a good thing
# either way.
cfg.reload(force=True)
consts.DATASETS_TOPURL = _test_states['DATASETS_TOPURL']
from datalad.support.cookies import cookies_db
cookies_db.close()
from datalad.support.annexrepo import AnnexRepo
AnnexRepo._ALLOW_LOCAL_URLS = False # stay safe!
def teardown_package():
import os
from datalad.tests.utils import rmtemp, OBSCURE_FILENAME
lgr.debug("Printing versioning information collected so far")
from datalad.support.external_versions import external_versions as ev
print(ev.dumps(query=True))
try:
print("Obscure filename: str=%s repr=%r" %
(OBSCURE_FILENAME.encode('utf-8'), OBSCURE_FILENAME))
except UnicodeEncodeError as exc:
from .dochelpers import exc_str
print("Obscure filename failed to print: %s" % exc_str(exc))
def print_dict(d):
return " ".join("%s=%r" % v for v in d.items())
print("Encodings: %s" % print_dict(get_encoding_info()))
print("Environment: %s" % print_dict(get_envvars_info()))
if os.environ.get('DATALAD_TESTS_NOTEARDOWN'):
return
from datalad.ui import ui
from datalad import consts
ui.set_backend(_test_states['ui_backend'])
if _test_states['loglevel'] is not None:
lgr.setLevel(_test_states['loglevel'])
if _test_states['DATALAD_LOG_LEVEL'] is None:
os.environ.pop('DATALAD_LOG_LEVEL')
else:
os.environ['DATALAD_LOG_LEVEL'] = _test_states['DATALAD_LOG_LEVEL']
from datalad.tests import _TEMP_PATHS_GENERATED
if len(_TEMP_PATHS_GENERATED):
msg = "Removing %d dirs/files: %s" % (len(_TEMP_PATHS_GENERATED),
', '.join(_TEMP_PATHS_GENERATED))
else:
msg = "Nothing to remove"
lgr.debug("Teardown tests. " + msg)
for path in _TEMP_PATHS_GENERATED:
rmtemp(path, ignore_errors=True)
if _test_states['HOME'] is not None:
os.environ['HOME'] = _test_states['HOME']
if _test_states['DATASETS_TOPURL_ENV']:
os.environ['DATALAD_DATASETS_TOPURL'] = _test_states[
'DATASETS_TOPURL_ENV']
consts.DATASETS_TOPURL = _test_states['DATASETS_TOPURL']
from datalad.support.cookies import cookies_db
cookies_db.close()
from datalad.support.annexrepo import AnnexRepo
AnnexRepo._ALLOW_LOCAL_URLS = False # stay safe!
def newfunc(*args, **kwargs):
from datalad.ui import ui
old_backend = ui.backend
try:
ui.set_backend('tests' if interactive else 'tests-noninteractive')
if responses:
ui.add_responses(responses)
ret = t(*args, **kwargs)
if responses:
responses_left = ui.get_responses()
assert not len(responses_left), "Some responses were left not used: %s" % str(responses_left)
return ret
finally:
ui.set_backend(old_backend)
def newfunc(*args, **kwargs):
from datalad.ui import ui
old_backend = ui.backend
try:
ui.set_backend('tests' if interactive else 'tests-noninteractive')
if responses:
ui.add_responses(responses)
ret = t(*args, **kwargs)
if responses:
responses_left = ui.get_responses()
assert not len(responses_left), "Some responses were left not used: %s" % str(responses_left)
return ret
finally:
ui.set_backend(old_backend)
def teardown_package():
import os
from datalad.tests.utils import rmtemp, OBSCURE_FILENAME
lgr.debug("Printing versioning information collected so far")
from datalad.support.external_versions import external_versions as ev
print(ev.dumps(query=True))
try:
print("Obscure filename: str=%s repr=%r"
% (OBSCURE_FILENAME.encode('utf-8'), OBSCURE_FILENAME))
except UnicodeEncodeError as exc:
from .dochelpers import exc_str
print("Obscure filename failed to print: %s" % exc_str(exc))
def print_dict(d):
return " ".join("%s=%r" % v for v in d.items())
print("Encodings: %s" % print_dict(get_encoding_info()))
print("Environment: %s" % print_dict(get_envvars_info()))
if os.environ.get('DATALAD_TESTS_NOTEARDOWN'):
return
from datalad.ui import ui
from datalad import consts
ui.set_backend(_test_states['ui_backend'])
if _test_states['loglevel'] is not None:
lgr.setLevel(_test_states['loglevel'])
if _test_states['DATALAD_LOG_LEVEL'] is None:
os.environ.pop('DATALAD_LOG_LEVEL')
else:
os.environ['DATALAD_LOG_LEVEL'] = _test_states['DATALAD_LOG_LEVEL']
from datalad.tests import _TEMP_PATHS_GENERATED
if len(_TEMP_PATHS_GENERATED):
msg = "Removing %d dirs/files: %s" % (len(_TEMP_PATHS_GENERATED), ', '.join(_TEMP_PATHS_GENERATED))
else:
msg = "Nothing to remove"
lgr.debug("Teardown tests. " + msg)
for path in _TEMP_PATHS_GENERATED:
rmtemp(path, ignore_errors=True)
if _test_states['HOME'] is not None:
os.environ['HOME'] = _test_states['HOME']
if _test_states['DATASETS_TOPURL_ENV']:
os.environ['DATALAD_DATASETS_TOPURL'] = _test_states['DATASETS_TOPURL_ENV']
consts.DATASETS_TOPURL = _test_states['DATASETS_TOPURL']
from datalad.support.cookies import cookies_db
cookies_db.close()
from datalad.support.annexrepo import AnnexRepo
AnnexRepo._ALLOW_LOCAL_URLS = False # stay safe!
def main(args=None, cls=None, remote_name=None, description=None):
import sys
parser = setup_parser(remote_name, description)
# parse cmd args
args = parser.parse_args(args)
# stdin/stdout will be used for interactions with annex
ui.set_backend('annex')
if args.common_debug:
# So we could see/stop clearly at the point of failure
setup_exceptionhook()
_main(args, cls)
else:
# Otherwise - guard and only log the summary. Postmortem is not
# as convenient if being caught in this ultimate except
try:
_main(args, cls)
except Exception as exc:
lgr.error('%s (%s)', str(exc), exc.__class__.__name__)
sys.exit(1)
def teardown_package():
import os
if os.environ.get('DATALAD_TESTS_NOTEARDOWN'):
return
from datalad.ui import ui
ui.set_backend(_test_states['ui_backend'])
if _test_states['loglevel'] is not None:
lgr.setLevel(_test_states['loglevel'])
if _test_states['DATALAD_LOG_LEVEL'] is None:
os.environ.pop('DATALAD_LOG_LEVEL')
else:
os.environ['DATALAD_LOG_LEVEL'] = _test_states['DATALAD_LOG_LEVEL']
from datalad.tests import _TEMP_PATHS_GENERATED
from datalad.tests.utils import rmtemp
if len(_TEMP_PATHS_GENERATED):
msg = "Removing %d dirs/files: %s" % (len(_TEMP_PATHS_GENERATED),
', '.join(_TEMP_PATHS_GENERATED))
else:
msg = "Nothing to remove"
lgr.debug("Teardown tests. " + msg)
for path in _TEMP_PATHS_GENERATED:
rmtemp(path, ignore_errors=True)
def setup_package():
import os
from datalad import consts
_test_states['HOME'] = os.environ.get('HOME', None)
_test_states['DATASETS_TOPURL_ENV'] = os.environ.get(
'DATALAD_DATASETS_TOPURL', None)
_test_states['DATASETS_TOPURL'] = consts.DATASETS_TOPURL
os.environ[
'DATALAD_DATASETS_TOPURL'] = consts.DATASETS_TOPURL = 'http://datasets-tests.datalad.org/'
# To overcome pybuild overriding HOME but us possibly wanting our
# own HOME where we pre-setup git for testing (name, email)
if 'GIT_HOME' in os.environ:
os.environ['HOME'] = os.environ['GIT_HOME']
else:
# we setup our own new HOME, the BEST and HUGE one
from datalad.utils import make_tempfile
from datalad.tests import _TEMP_PATHS_GENERATED
# TODO: split into a function + context manager
with make_tempfile(mkdir=True) as new_home:
os.environ['HOME'] = new_home
if not os.path.exists(new_home):
os.makedirs(new_home)
with open(os.path.join(new_home, '.gitconfig'), 'w') as f:
f.write("""\
[user]
name = DataLad Tester
email = [email protected]
""")
_TEMP_PATHS_GENERATED.append(new_home)
# For now we will just verify that it is ready to run the tests
from datalad.support.gitrepo import check_git_configured
check_git_configured()
# To overcome pybuild by default defining http{,s}_proxy we would need
# to define them to e.g. empty value so it wouldn't bother touching them.
# But then haskell libraries do not digest empty value nicely, so we just
# pop them out from the environment
for ev in ('http_proxy', 'https_proxy'):
if ev in os.environ and not (os.environ[ev]):
lgr.debug("Removing %s from the environment since it is empty", ev)
os.environ.pop(ev)
# During tests we allow for "insecure" access to local file:// and
# http://localhost URLs since all of them either generated as tests
# fixtures or cloned from trusted sources
from datalad.support.annexrepo import AnnexRepo
AnnexRepo._ALLOW_LOCAL_URLS = True
DATALAD_LOG_LEVEL = os.environ.get('DATALAD_LOG_LEVEL', None)
if DATALAD_LOG_LEVEL is None:
# very very silent. Tests introspecting logs should use
# swallow_logs(new_level=...)
_test_states['loglevel'] = lgr.getEffectiveLevel()
lgr.setLevel(100)
# And we should also set it within environ so underlying commands also stay silent
_test_states['DATALAD_LOG_LEVEL'] = DATALAD_LOG_LEVEL
os.environ['DATALAD_LOG_LEVEL'] = '100'
else:
# We are not overriding them, since explicitly were asked to have some log level
_test_states['loglevel'] = None
# Set to non-interactive UI
from datalad.ui import ui
_test_states['ui_backend'] = ui.backend
# obtain() since that one consults for the default value
ui.set_backend(cfg.obtain('datalad.tests.ui.backend'))
def setup_package():
import os
from datalad.utils import on_osx
from datalad.tests import _TEMP_PATHS_GENERATED
if on_osx:
# enforce honoring TMPDIR (see gh-5307)
import tempfile
tempfile.tempdir = os.environ.get('TMPDIR', tempfile.gettempdir())
from datalad import consts
_test_states['env'] = {}
def set_envvar(v, val):
"""Memoize and then set env var"""
_test_states['env'][v] = os.environ.get(v, None)
os.environ[v] = val
_test_states['DATASETS_TOPURL'] = consts.DATASETS_TOPURL
consts.DATASETS_TOPURL = 'http://datasets-tests.datalad.org/'
set_envvar('DATALAD_DATASETS_TOPURL', consts.DATASETS_TOPURL)
from datalad.tests.utils import (
DEFAULT_BRANCH,
DEFAULT_REMOTE,
)
set_envvar(
"GIT_CONFIG_PARAMETERS",
"'init.defaultBranch={}' 'clone.defaultRemoteName={}'".format(
DEFAULT_BRANCH, DEFAULT_REMOTE))
# To overcome pybuild overriding HOME but us possibly wanting our
# own HOME where we pre-setup git for testing (name, email)
if 'GIT_HOME' in os.environ:
set_envvar('HOME', os.environ['GIT_HOME'])
set_envvar('DATALAD_LOG_EXC', "1")
else:
# we setup our own new HOME, the BEST and HUGE one
from datalad.utils import make_tempfile
# TODO: split into a function + context manager
with make_tempfile(mkdir=True) as new_home:
pass
for v, val in get_home_envvars(new_home).items():
set_envvar(v, val)
if not os.path.exists(new_home):
os.makedirs(new_home)
with open(os.path.join(new_home, '.gitconfig'), 'w') as f:
f.write("""\
[user]
name = DataLad Tester
email = [email protected]
[datalad "log"]
exc = 1
""")
_TEMP_PATHS_GENERATED.append(new_home)
# Re-load ConfigManager, since otherwise it won't consider global config
# from new $HOME (see gh-4153
cfg.reload(force=True)
from datalad.interface.common_cfg import compute_cfg_defaults
compute_cfg_defaults()
# datalad.locations.sockets has likely changed. Discard any cached values.
ssh_manager._socket_dir = None
# To overcome pybuild by default defining http{,s}_proxy we would need
# to define them to e.g. empty value so it wouldn't bother touching them.
# But then haskell libraries do not digest empty value nicely, so we just
# pop them out from the environment
for ev in ('http_proxy', 'https_proxy'):
if ev in os.environ and not (os.environ[ev]):
lgr.debug("Removing %s from the environment since it is empty", ev)
os.environ.pop(ev)
# During tests we allow for "insecure" access to local file:// and
# http://localhost URLs since all of them either generated as tests
# fixtures or cloned from trusted sources
from datalad.support.annexrepo import AnnexRepo
AnnexRepo._ALLOW_LOCAL_URLS = True
DATALAD_LOG_LEVEL = os.environ.get('DATALAD_LOG_LEVEL', None)
if DATALAD_LOG_LEVEL is None:
# very very silent. Tests introspecting logs should use
# swallow_logs(new_level=...)
_test_states['loglevel'] = lgr.getEffectiveLevel()
lgr.setLevel(100)
# And we should also set it within environ so underlying commands also stay silent
set_envvar('DATALAD_LOG_LEVEL', '100')
else:
# We are not overriding them, since explicitly were asked to have some log level
_test_states['loglevel'] = None
# Set to non-interactive UI
from datalad.ui import ui
_test_states['ui_backend'] = ui.backend
# obtain() since that one consults for the default value
ui.set_backend(cfg.obtain('datalad.tests.ui.backend'))
# Monkey patch nose so it does not ERROR out whenever code asks for fileno
# of the output. See https://github.com/nose-devs/nose/issues/6
from io import StringIO as OrigStringIO
class StringIO(OrigStringIO):
fileno = lambda self: 1
encoding = None
from nose.ext import dtcompat
from nose.plugins import capture, multiprocess, plugintest
dtcompat.StringIO = StringIO
capture.StringIO = StringIO
multiprocess.StringIO = StringIO
plugintest.StringIO = StringIO
# in order to avoid having to fiddle with rather uncommon
# file:// URLs in the tests, have a standard HTTP server
# that serves an 'httpserve' directory in the test HOME
# the URL will be available from datalad.test_http_server.url
from datalad.tests.utils import HTTPPath
import tempfile
global test_http_server
serve_path = tempfile.mkdtemp(
dir=cfg.get("datalad.tests.temp.dir"),
prefix='httpserve',
)
test_http_server = HTTPPath(serve_path)
test_http_server.start()
_TEMP_PATHS_GENERATED.append(serve_path)
if cfg.obtain('datalad.tests.setup.testrepos'):
lgr.debug("Pre-populating testrepos")
from datalad.tests.utils import with_testrepos
with_testrepos()(lambda repo: 1)()
def setup_package():
import os
from datalad import consts
_test_states['HOME'] = os.environ.get('HOME', None)
_test_states['DATASETS_TOPURL_ENV'] = os.environ.get('DATALAD_DATASETS_TOPURL', None)
_test_states['DATASETS_TOPURL'] = consts.DATASETS_TOPURL
os.environ['DATALAD_DATASETS_TOPURL'] = consts.DATASETS_TOPURL = 'http://datasets-tests.datalad.org/'
# To overcome pybuild overriding HOME but us possibly wanting our
# own HOME where we pre-setup git for testing (name, email)
if 'GIT_HOME' in os.environ:
os.environ['HOME'] = os.environ['GIT_HOME']
else:
# we setup our own new HOME, the BEST and HUGE one
from datalad.utils import make_tempfile
from datalad.tests import _TEMP_PATHS_GENERATED
# TODO: split into a function + context manager
with make_tempfile(mkdir=True) as new_home:
os.environ['HOME'] = new_home
if not os.path.exists(new_home):
os.makedirs(new_home)
with open(os.path.join(new_home, '.gitconfig'), 'w') as f:
f.write("""\
[user]
name = DataLad Tester
email = [email protected]
""")
_TEMP_PATHS_GENERATED.append(new_home)
# If there is a bundled git, make sure GitPython uses it too
# (some parts of the test utilities still rely on GitPython)
from datalad.cmd import GitRunner
GitRunner._check_git_path()
if GitRunner._GIT_PATH:
import os
os.environ['GIT_PYTHON_GIT_EXECUTABLE'] = \
os.path.join(GitRunner._GIT_PATH, 'git')
# Re-load ConfigManager, since otherwise it won't consider global config
# from new $HOME (see gh-4153
cfg.reload(force=True)
# To overcome pybuild by default defining http{,s}_proxy we would need
# to define them to e.g. empty value so it wouldn't bother touching them.
# But then haskell libraries do not digest empty value nicely, so we just
# pop them out from the environment
for ev in ('http_proxy', 'https_proxy'):
if ev in os.environ and not (os.environ[ev]):
lgr.debug("Removing %s from the environment since it is empty", ev)
os.environ.pop(ev)
# During tests we allow for "insecure" access to local file:// and
# http://localhost URLs since all of them either generated as tests
# fixtures or cloned from trusted sources
from datalad.support.annexrepo import AnnexRepo
AnnexRepo._ALLOW_LOCAL_URLS = True
DATALAD_LOG_LEVEL = os.environ.get('DATALAD_LOG_LEVEL', None)
if DATALAD_LOG_LEVEL is None:
# very very silent. Tests introspecting logs should use
# swallow_logs(new_level=...)
_test_states['loglevel'] = lgr.getEffectiveLevel()
lgr.setLevel(100)
# And we should also set it within environ so underlying commands also stay silent
_test_states['DATALAD_LOG_LEVEL'] = DATALAD_LOG_LEVEL
os.environ['DATALAD_LOG_LEVEL'] = '100'
else:
# We are not overriding them, since explicitly were asked to have some log level
_test_states['loglevel'] = None
# Set to non-interactive UI
from datalad.ui import ui
_test_states['ui_backend'] = ui.backend
# obtain() since that one consults for the default value
ui.set_backend(cfg.obtain('datalad.tests.ui.backend'))
# Monkey patch nose so it does not ERROR out whenever code asks for fileno
# of the output. See https://github.com/nose-devs/nose/issues/6
from io import StringIO as OrigStringIO
class StringIO(OrigStringIO):
fileno = lambda self: 1
encoding = None
from nose.ext import dtcompat
from nose.plugins import capture, multiprocess, plugintest
dtcompat.StringIO = StringIO
capture.StringIO = StringIO
multiprocess.StringIO = StringIO
plugintest.StringIO = StringIO
def setup_package():
import tempfile
from pathlib import Path
from datalad import consts
from datalad.support.annexrepo import AnnexRepo
from datalad.support.cookies import cookies_db
from datalad.support.external_versions import external_versions
from datalad.tests import _TEMP_PATHS_GENERATED
from datalad.tests.utils_pytest import (
DEFAULT_BRANCH,
DEFAULT_REMOTE,
OBSCURE_FILENAME,
HTTPPath,
rmtemp,
)
from datalad.ui import ui
from datalad.utils import (
make_tempfile,
on_osx,
)
if on_osx:
# enforce honoring TMPDIR (see gh-5307)
tempfile.tempdir = os.environ.get('TMPDIR', tempfile.gettempdir())
with pytest.MonkeyPatch().context() as m:
m.setattr(consts, "DATASETS_TOPURL",
'https://datasets-tests.datalad.org/')
m.setenv('DATALAD_DATASETS_TOPURL', consts.DATASETS_TOPURL)
m.setenv(
"GIT_CONFIG_PARAMETERS",
"'init.defaultBranch={}' 'clone.defaultRemoteName={}'".format(
DEFAULT_BRANCH, DEFAULT_REMOTE))
def prep_tmphome():
# re core.askPass:
# Don't let git ask for credentials in CI runs. Note, that this variable
# technically is not a flag, but an executable (which is why name and value
# are a bit confusing here - we just want a no-op basically). The environment
# variable GIT_ASKPASS overwrites this, but neither env var nor this config
# are supported by git-credential on all systems and git versions (most recent
# ones should work either way, though). Hence use both across CI builds.
gitconfig = """\
[user]
name = DataLad Tester
email = [email protected]
[core]
askPass =
[datalad "log"]
exc = 1
[annex "security"]
# from annex 6.20180626 file:/// and http://localhost access isn't
# allowed by default
allowed-url-schemes = http https file
allowed-http-addresses = all
"""
# TODO: split into a function + context manager
with make_tempfile(mkdir=True) as new_home:
pass
# register for clean-up on exit
_TEMP_PATHS_GENERATED.append(new_home)
# populate default config
new_home = Path(new_home)
new_home.mkdir(parents=True, exist_ok=True)
cfg_file = new_home / '.gitconfig'
cfg_file.write_text(gitconfig)
return new_home, cfg_file
if external_versions['cmd:git'] < "2.32":
# To overcome pybuild overriding HOME but us possibly wanting our
# own HOME where we pre-setup git for testing (name, email)
if 'GIT_HOME' in os.environ:
m.setenv('HOME', os.environ['GIT_HOME'])
else:
# we setup our own new HOME, the BEST and HUGE one
new_home, _ = prep_tmphome()
for v, val in get_home_envvars(new_home).items():
m.setenv(v, val)
else:
_, cfg_file = prep_tmphome()
m.setenv('GIT_CONFIG_GLOBAL', str(cfg_file))
# Re-load ConfigManager, since otherwise it won't consider global config
# from new $HOME (see gh-4153
cfg.reload(force=True)
# datalad.locations.sockets has likely changed. Discard any cached values.
ssh_manager._socket_dir = None
# To overcome pybuild by default defining http{,s}_proxy we would need
# to define them to e.g. empty value so it wouldn't bother touching them.
# But then haskell libraries do not digest empty value nicely, so we just
# pop them out from the environment
for ev in ('http_proxy', 'https_proxy'):
if ev in os.environ and not (os.environ[ev]):
lgr.debug("Removing %s from the environment since it is empty",
ev)
os.environ.pop(ev)
# Prevent interactive credential entry (note "true" is the command to run)
# See also the core.askPass setting above
m.setenv('GIT_ASKPASS', 'true')
# Set to non-interactive UI
_test_states['ui_backend'] = ui.backend
# obtain() since that one consults for the default value
ui.set_backend(cfg.obtain('datalad.tests.ui.backend'))
# in order to avoid having to fiddle with rather uncommon
# file:// URLs in the tests, have a standard HTTP server
# that serves an 'httpserve' directory in the test HOME
# the URL will be available from datalad.test_http_server.url
global test_http_server
# Start the server only if not running already
# Relevant: we have test_misc.py:test_test which runs datalad.test but
# not doing teardown, so the original server might never get stopped
if test_http_server is None:
serve_path = tempfile.mkdtemp(
dir=cfg.get("datalad.tests.temp.dir"),
prefix='httpserve',
)
test_http_server = HTTPPath(serve_path)
test_http_server.start()
_TEMP_PATHS_GENERATED.append(serve_path)
if cfg.obtain('datalad.tests.setup.testrepos'):
lgr.debug("Pre-populating testrepos")
from datalad.tests.utils_pytest import with_testrepos
with_testrepos()(lambda repo: 1)()
yield
lgr.debug("Printing versioning information collected so far")
# Query for version of datalad, so it is included in ev.dumps below - useful while
# testing extensions where version of datalad might differ in the environment.
external_versions['datalad']
print(external_versions.dumps(query=True))
try:
print("Obscure filename: str=%s repr=%r" %
(OBSCURE_FILENAME.encode('utf-8'), OBSCURE_FILENAME))
except UnicodeEncodeError as exc:
ce = CapturedException(exc)
print("Obscure filename failed to print: %s" % ce)
def print_dict(d):
return " ".join("%s=%r" % v for v in d.items())
print("Encodings: %s" % print_dict(get_encoding_info()))
print("Environment: %s" % print_dict(get_envvars_info()))
if os.environ.get('DATALAD_TESTS_NOTEARDOWN'):
return
ui.set_backend(_test_states['ui_backend'])
if test_http_server:
test_http_server.stop()
test_http_server = None
else:
lgr.debug(
"For some reason global http_server was not set/running, thus not stopping"
)
if len(_TEMP_PATHS_GENERATED):
msg = "Removing %d dirs/files: %s" % (
len(_TEMP_PATHS_GENERATED), ', '.join(_TEMP_PATHS_GENERATED))
else:
msg = "Nothing to remove"
lgr.debug("Teardown tests. " + msg)
for path in _TEMP_PATHS_GENERATED:
rmtemp(str(path), ignore_errors=True)
# Re-establish correct global config after changing $HOME.
# Might be superfluous, since after teardown datalad.cfg shouldn't be
# needed. However, maintaining a consistent state seems a good thing
# either way.
cfg.reload(force=True)
ssh_manager._socket_dir = None
cookies_db.close()
def setup_package():
import os
from datalad import consts
_test_states['HOME'] = os.environ.get('HOME', None)
_test_states['DATASETS_TOPURL_ENV'] = os.environ.get('DATALAD_DATASETS_TOPURL', None)
_test_states['DATASETS_TOPURL'] = consts.DATASETS_TOPURL
os.environ['DATALAD_DATASETS_TOPURL'] = consts.DATASETS_TOPURL = 'http://datasets-tests.datalad.org/'
# To overcome pybuild overriding HOME but us possibly wanting our
# own HOME where we pre-setup git for testing (name, email)
if 'GIT_HOME' in os.environ:
os.environ['HOME'] = os.environ['GIT_HOME']
else:
# we setup our own new HOME, the BEST and HUGE one
from datalad.utils import make_tempfile
from datalad.tests import _TEMP_PATHS_GENERATED
# TODO: split into a function + context manager
with make_tempfile(mkdir=True) as new_home:
os.environ['HOME'] = new_home
if not os.path.exists(new_home):
os.makedirs(new_home)
with open(os.path.join(new_home, '.gitconfig'), 'w') as f:
f.write("""\
[user]
name = DataLad Tester
email = [email protected]
""")
_TEMP_PATHS_GENERATED.append(new_home)
# For now we will just verify that it is ready to run the tests
from datalad.support.gitrepo import check_git_configured
check_git_configured()
# To overcome pybuild by default defining http{,s}_proxy we would need
# to define them to e.g. empty value so it wouldn't bother touching them.
# But then haskell libraries do not digest empty value nicely, so we just
# pop them out from the environment
for ev in ('http_proxy', 'https_proxy'):
if ev in os.environ and not (os.environ[ev]):
lgr.debug("Removing %s from the environment since it is empty", ev)
os.environ.pop(ev)
# During tests we allow for "insecure" access to local file:// and
# http://localhost URLs since all of them either generated as tests
# fixtures or cloned from trusted sources
from datalad.support.annexrepo import AnnexRepo
AnnexRepo._ALLOW_LOCAL_URLS = True
DATALAD_LOG_LEVEL = os.environ.get('DATALAD_LOG_LEVEL', None)
if DATALAD_LOG_LEVEL is None:
# very very silent. Tests introspecting logs should use
# swallow_logs(new_level=...)
_test_states['loglevel'] = lgr.getEffectiveLevel()
lgr.setLevel(100)
# And we should also set it within environ so underlying commands also stay silent
_test_states['DATALAD_LOG_LEVEL'] = DATALAD_LOG_LEVEL
os.environ['DATALAD_LOG_LEVEL'] = '100'
else:
# We are not overriding them, since explicitly were asked to have some log level
_test_states['loglevel'] = None
# Set to non-interactive UI
from datalad.ui import ui
_test_states['ui_backend'] = ui.backend
# obtain() since that one consults for the default value
ui.set_backend(cfg.obtain('datalad.tests.ui.backend'))
def _action_store(attrs, interactive, providers):
# Determine the defaults to use for storing. In non-interactive mode,
# this is what's going to be stored, in interactive mode user is
# presented with them as default choice.
# We don't really know what authentication type makes sense to store in
# the provider config (this would be relevant for datalad using those
# credentials w/o git).
# However, pick 'http_basic_auth' in case of HTTP(S) URL and 'none'
# otherwise as the default to pass into store routine.
if attrs.get('protocol') in ['http', 'https']:
authentication_type = 'http_basic_auth'
else:
authentication_type = 'none'
# If we got a `path` component from git, usehttppath is set and thereby
# git was instructed to include it when matching. Hence, do the same.
url_re = "{pr}://{h}{p}.*".format(pr=attrs.get('protocol'),
h=attrs.get('host'),
p="/" + attrs.get('path')
if "path" in attrs.keys() else "")
name = attrs.get('host')
credential_name = name
credential_type = "user_password"
if not interactive:
# TODO: What about credential labels? This could already exist as
# well. However, it's unlikely since the respective
# "service name" for keyring is prepended with "datalad-".
# For the same reason of how the keyring system is used by
# datalad it's not very transparently accessible what labels
# we'd need to check for. Rather than encoding knowledge about
# datalad's internal handling here, let's address that in
# datalad's provider and credential classes and have an easy
# check to be called from here.
if any(p.name == name for p in providers):
print(f"Provider name '{name}' already exists. This can't be "
"resolved in non-interactive mode.",
file=sys.stderr)
authenticator_class = AUTHENTICATION_TYPES[authentication_type]
saved_provider = providers._store_new(
url=attrs.get('url'),
authentication_type=authentication_type,
authenticator_class=authenticator_class,
url_re=url_re,
name=name,
credential_name=credential_name,
credential_type=credential_type,
level='user'
)
else:
# use backend made for annex special remotes for interaction from a
# subprocess whose stdin/stdout are in use for communication with
# its parent
from datalad.ui import ui
ui.set_backend('annex')
# ensure default is first in list (that's how `enter_new` determines
# it's the default)
auth_list = AUTHENTICATION_TYPES.copy()
auth_list.pop(authentication_type, None)
auth_list = [authentication_type] + list(auth_list.keys())
saved_provider = providers.enter_new(
url=attrs.get('url'),
auth_types=auth_list,
url_re=url_re,
name=name,
credential_name=credential_name,
credential_type=credential_type)
saved_provider.credential.set(user=attrs['username'],
password=attrs['password'])
