Example #1
0
def teardown_package():
    import os
    from datalad.tests.utils import rmtemp, OBSCURE_FILENAME

    lgr.debug("Printing versioning information collected so far")
    from datalad.support.external_versions import external_versions as ev
    print(ev.dumps(query=True))
    try:
        print("Obscure filename: str=%s repr=%r" %
              (OBSCURE_FILENAME.encode('utf-8'), OBSCURE_FILENAME))
    except UnicodeEncodeError as exc:
        from .dochelpers import exc_str
        print("Obscure filename failed to print: %s" % exc_str(exc))

    def print_dict(d):
        return " ".join("%s=%r" % v for v in d.items())

    print("Encodings: %s" % print_dict(get_encoding_info()))
    print("Environment: %s" % print_dict(get_envvars_info()))

    if os.environ.get('DATALAD_TESTS_NOTEARDOWN'):
        return
    from datalad.ui import ui
    from datalad import consts
    ui.set_backend(_test_states['ui_backend'])
    if _test_states['loglevel'] is not None:
        lgr.setLevel(_test_states['loglevel'])

    from datalad.tests import _TEMP_PATHS_GENERATED
    if len(_TEMP_PATHS_GENERATED):
        msg = "Removing %d dirs/files: %s" % (len(_TEMP_PATHS_GENERATED),
                                              ', '.join(_TEMP_PATHS_GENERATED))
    else:
        msg = "Nothing to remove"
    lgr.debug("Teardown tests. " + msg)
    for path in _TEMP_PATHS_GENERATED:
        rmtemp(path, ignore_errors=True)

    # restore all the env variables
    for v, val in _test_states['env'].items():
        if val is not None:
            os.environ[v] = val
        else:
            os.environ.pop(v)

    # Re-establish correct global config after changing $HOME.
    # Might be superfluous, since after teardown datalad.cfg shouldn't be
    # needed. However, maintaining a consistent state seems a good thing
    # either way.
    cfg.reload(force=True)

    consts.DATASETS_TOPURL = _test_states['DATASETS_TOPURL']

    from datalad.support.cookies import cookies_db
    cookies_db.close()
    from datalad.support.annexrepo import AnnexRepo
    AnnexRepo._ALLOW_LOCAL_URLS = False  # stay safe!
Example #2
0
def teardown_package():
    import os
    from datalad.tests.utils import rmtemp, OBSCURE_FILENAME

    lgr.debug("Printing versioning information collected so far")
    from datalad.support.external_versions import external_versions as ev
    print(ev.dumps(query=True))
    try:
        print("Obscure filename: str=%s repr=%r" %
              (OBSCURE_FILENAME.encode('utf-8'), OBSCURE_FILENAME))
    except UnicodeEncodeError as exc:
        from .dochelpers import exc_str
        print("Obscure filename failed to print: %s" % exc_str(exc))

    def print_dict(d):
        return " ".join("%s=%r" % v for v in d.items())

    print("Encodings: %s" % print_dict(get_encoding_info()))
    print("Environment: %s" % print_dict(get_envvars_info()))

    if os.environ.get('DATALAD_TESTS_NOTEARDOWN'):
        return
    from datalad.ui import ui
    from datalad import consts
    ui.set_backend(_test_states['ui_backend'])
    if _test_states['loglevel'] is not None:
        lgr.setLevel(_test_states['loglevel'])
        if _test_states['DATALAD_LOG_LEVEL'] is None:
            os.environ.pop('DATALAD_LOG_LEVEL')
        else:
            os.environ['DATALAD_LOG_LEVEL'] = _test_states['DATALAD_LOG_LEVEL']

    from datalad.tests import _TEMP_PATHS_GENERATED
    if len(_TEMP_PATHS_GENERATED):
        msg = "Removing %d dirs/files: %s" % (len(_TEMP_PATHS_GENERATED),
                                              ', '.join(_TEMP_PATHS_GENERATED))
    else:
        msg = "Nothing to remove"
    lgr.debug("Teardown tests. " + msg)
    for path in _TEMP_PATHS_GENERATED:
        rmtemp(path, ignore_errors=True)

    if _test_states['HOME'] is not None:
        os.environ['HOME'] = _test_states['HOME']

    if _test_states['DATASETS_TOPURL_ENV']:
        os.environ['DATALAD_DATASETS_TOPURL'] = _test_states[
            'DATASETS_TOPURL_ENV']
    consts.DATASETS_TOPURL = _test_states['DATASETS_TOPURL']

    from datalad.support.cookies import cookies_db
    cookies_db.close()
    from datalad.support.annexrepo import AnnexRepo
    AnnexRepo._ALLOW_LOCAL_URLS = False  # stay safe!
Example #3
0
def teardown_package():
    import os
    from datalad.tests.utils import rmtemp, OBSCURE_FILENAME

    lgr.debug("Printing versioning information collected so far")
    from datalad.support.external_versions import external_versions as ev
    print(ev.dumps(query=True))
    try:
        print("Obscure filename: str=%s repr=%r"
                % (OBSCURE_FILENAME.encode('utf-8'), OBSCURE_FILENAME))
    except UnicodeEncodeError as exc:
        from .dochelpers import exc_str
        print("Obscure filename failed to print: %s" % exc_str(exc))
    def print_dict(d):
        return " ".join("%s=%r" % v for v in d.items())
    print("Encodings: %s" % print_dict(get_encoding_info()))
    print("Environment: %s" % print_dict(get_envvars_info()))

    if os.environ.get('DATALAD_TESTS_NOTEARDOWN'):
        return
    from datalad.ui import ui
    from datalad import consts
    ui.set_backend(_test_states['ui_backend'])
    if _test_states['loglevel'] is not None:
        lgr.setLevel(_test_states['loglevel'])
        if _test_states['DATALAD_LOG_LEVEL'] is None:
            os.environ.pop('DATALAD_LOG_LEVEL')
        else:
            os.environ['DATALAD_LOG_LEVEL'] = _test_states['DATALAD_LOG_LEVEL']

    from datalad.tests import _TEMP_PATHS_GENERATED
    if len(_TEMP_PATHS_GENERATED):
        msg = "Removing %d dirs/files: %s" % (len(_TEMP_PATHS_GENERATED), ', '.join(_TEMP_PATHS_GENERATED))
    else:
        msg = "Nothing to remove"
    lgr.debug("Teardown tests. " + msg)
    for path in _TEMP_PATHS_GENERATED:
        rmtemp(path, ignore_errors=True)

    if _test_states['HOME'] is not None:
        os.environ['HOME'] = _test_states['HOME']

    if _test_states['DATASETS_TOPURL_ENV']:
        os.environ['DATALAD_DATASETS_TOPURL'] = _test_states['DATASETS_TOPURL_ENV']
    consts.DATASETS_TOPURL = _test_states['DATASETS_TOPURL']

    from datalad.support.cookies import cookies_db
    cookies_db.close()
    from datalad.support.annexrepo import AnnexRepo
    AnnexRepo._ALLOW_LOCAL_URLS = False  # stay safe!
Example #4
0
def setup_package():
    import tempfile
    from pathlib import Path

    from datalad import consts
    from datalad.support.annexrepo import AnnexRepo
    from datalad.support.cookies import cookies_db
    from datalad.support.external_versions import external_versions
    from datalad.tests import _TEMP_PATHS_GENERATED
    from datalad.tests.utils_pytest import (
        DEFAULT_BRANCH,
        DEFAULT_REMOTE,
        OBSCURE_FILENAME,
        HTTPPath,
        rmtemp,
    )
    from datalad.ui import ui
    from datalad.utils import (
        make_tempfile,
        on_osx,
    )

    if on_osx:
        # enforce honoring TMPDIR (see gh-5307)
        tempfile.tempdir = os.environ.get('TMPDIR', tempfile.gettempdir())

    with pytest.MonkeyPatch().context() as m:
        m.setattr(consts, "DATASETS_TOPURL",
                  'https://datasets-tests.datalad.org/')
        m.setenv('DATALAD_DATASETS_TOPURL', consts.DATASETS_TOPURL)

        m.setenv(
            "GIT_CONFIG_PARAMETERS",
            "'init.defaultBranch={}' 'clone.defaultRemoteName={}'".format(
                DEFAULT_BRANCH, DEFAULT_REMOTE))

        def prep_tmphome():
            # re core.askPass:
            # Don't let git ask for credentials in CI runs. Note, that this variable
            # technically is not a flag, but an executable (which is why name and value
            # are a bit confusing here - we just want a no-op basically). The environment
            # variable GIT_ASKPASS overwrites this, but neither env var nor this config
            # are supported by git-credential on all systems and git versions (most recent
            # ones should work either way, though). Hence use both across CI builds.
            gitconfig = """\
[user]
        name = DataLad Tester
        email = [email protected]
[core]
	askPass =
[datalad "log"]
        exc = 1
[annex "security"]
	# from annex 6.20180626 file:/// and http://localhost access isn't
	# allowed by default
	allowed-url-schemes = http https file
	allowed-http-addresses = all
"""
            # TODO: split into a function + context manager
            with make_tempfile(mkdir=True) as new_home:
                pass
            # register for clean-up on exit
            _TEMP_PATHS_GENERATED.append(new_home)

            # populate default config
            new_home = Path(new_home)
            new_home.mkdir(parents=True, exist_ok=True)
            cfg_file = new_home / '.gitconfig'
            cfg_file.write_text(gitconfig)
            return new_home, cfg_file

        if external_versions['cmd:git'] < "2.32":
            # To overcome pybuild overriding HOME but us possibly wanting our
            # own HOME where we pre-setup git for testing (name, email)
            if 'GIT_HOME' in os.environ:
                m.setenv('HOME', os.environ['GIT_HOME'])
            else:
                # we setup our own new HOME, the BEST and HUGE one
                new_home, _ = prep_tmphome()
                for v, val in get_home_envvars(new_home).items():
                    m.setenv(v, val)
        else:
            _, cfg_file = prep_tmphome()
            m.setenv('GIT_CONFIG_GLOBAL', str(cfg_file))

        # Re-load ConfigManager, since otherwise it won't consider global config
        # from new $HOME (see gh-4153
        cfg.reload(force=True)

        # datalad.locations.sockets has likely changed. Discard any cached values.
        ssh_manager._socket_dir = None

        # To overcome pybuild by default defining http{,s}_proxy we would need
        # to define them to e.g. empty value so it wouldn't bother touching them.
        # But then haskell libraries do not digest empty value nicely, so we just
        # pop them out from the environment
        for ev in ('http_proxy', 'https_proxy'):
            if ev in os.environ and not (os.environ[ev]):
                lgr.debug("Removing %s from the environment since it is empty",
                          ev)
                os.environ.pop(ev)

        # Prevent interactive credential entry (note "true" is the command to run)
        # See also the core.askPass setting above
        m.setenv('GIT_ASKPASS', 'true')

        # Set to non-interactive UI
        _test_states['ui_backend'] = ui.backend
        # obtain() since that one consults for the default value
        ui.set_backend(cfg.obtain('datalad.tests.ui.backend'))

        # in order to avoid having to fiddle with rather uncommon
        # file:// URLs in the tests, have a standard HTTP server
        # that serves an 'httpserve' directory in the test HOME
        # the URL will be available from datalad.test_http_server.url

        global test_http_server
        # Start the server only if not running already
        # Relevant: we have test_misc.py:test_test which runs datalad.test but
        # not doing teardown, so the original server might never get stopped
        if test_http_server is None:
            serve_path = tempfile.mkdtemp(
                dir=cfg.get("datalad.tests.temp.dir"),
                prefix='httpserve',
            )
            test_http_server = HTTPPath(serve_path)
            test_http_server.start()
            _TEMP_PATHS_GENERATED.append(serve_path)

        if cfg.obtain('datalad.tests.setup.testrepos'):
            lgr.debug("Pre-populating testrepos")
            from datalad.tests.utils_pytest import with_testrepos
            with_testrepos()(lambda repo: 1)()

        yield

        lgr.debug("Printing versioning information collected so far")
        # Query for version of datalad, so it is included in ev.dumps below - useful while
        # testing extensions where version of datalad might differ in the environment.
        external_versions['datalad']
        print(external_versions.dumps(query=True))
        try:
            print("Obscure filename: str=%s repr=%r" %
                  (OBSCURE_FILENAME.encode('utf-8'), OBSCURE_FILENAME))
        except UnicodeEncodeError as exc:
            ce = CapturedException(exc)
            print("Obscure filename failed to print: %s" % ce)

        def print_dict(d):
            return " ".join("%s=%r" % v for v in d.items())

        print("Encodings: %s" % print_dict(get_encoding_info()))
        print("Environment: %s" % print_dict(get_envvars_info()))

        if os.environ.get('DATALAD_TESTS_NOTEARDOWN'):
            return

        ui.set_backend(_test_states['ui_backend'])

        if test_http_server:
            test_http_server.stop()
            test_http_server = None
        else:
            lgr.debug(
                "For some reason global http_server was not set/running, thus not stopping"
            )

        if len(_TEMP_PATHS_GENERATED):
            msg = "Removing %d dirs/files: %s" % (
                len(_TEMP_PATHS_GENERATED), ', '.join(_TEMP_PATHS_GENERATED))
        else:
            msg = "Nothing to remove"
        lgr.debug("Teardown tests. " + msg)
        for path in _TEMP_PATHS_GENERATED:
            rmtemp(str(path), ignore_errors=True)

    # Re-establish correct global config after changing $HOME.
    # Might be superfluous, since after teardown datalad.cfg shouldn't be
    # needed. However, maintaining a consistent state seems a good thing
    # either way.
    cfg.reload(force=True)

    ssh_manager._socket_dir = None

    cookies_db.close()