def test_ria_postclonecfg():
if not has_symlink_capability():
# This is needed to create an ORA remote using an URL for upload,
# that is then invalidated later on (delete the symlink it's based on).
raise SkipTest("Can't create symlinks")
from datalad.utils import make_tempfile
from datalad.tests.utils import HTTPPath
with make_tempfile(mkdir=True) as lcl, make_tempfile(mkdir=True) as store:
id = _postclonetest_prepare(lcl, store)
# test cloning via ria+file://
yield _test_ria_postclonecfg, Path(store).as_uri(), id
# Note: HTTP disabled for now. Requires proper implementation in ORA
# remote. See
# https://github.com/datalad/datalad/pull/4203#discussion_r410284649
# # test cloning via ria+http://
# with HTTPPath(store) as url:
# yield _test_ria_postclonecfg, url, id
# test cloning via ria+ssh://
yield skip_ssh(_test_ria_postclonecfg), \
"ssh://datalad-test:{}".format(Path(store).as_posix()), id
def create_postupdate_hook(path, ssh, dataset):
# location of post-update hook file, logs folder on remote target
hooks_remote_dir = opj(path, '.git', 'hooks')
hook_remote_target = opj(hooks_remote_dir, 'post-update')
# post-update hook should create its log directory if doesn't exist
logs_remote_dir = opj(path, WEB_META_LOG)
make_log_dir = 'mkdir -p "{}"'.format(logs_remote_dir)
# create json command for current dataset
json_command = r'''
mkdir -p {};
( which datalad > /dev/null \
&& ( cd ..; GIT_DIR=$PWD/.git datalad ls -a --json file '{}'; ) \
|| echo "no datalad found - skipping generation of indexes for web frontend"; \
) &> "{}/{}"
'''.format(logs_remote_dir,
str(path),
logs_remote_dir,
'datalad-publish-hook-$(date +%s).log' % TIMESTAMP_FMT)
# collate content for post_update hook
hook_content = '\n'.join(['#!/bin/bash', 'git update-server-info', make_log_dir, json_command])
with make_tempfile(content=hook_content) as tempf: # create post_update hook script
ssh.copy(tempf, hook_remote_target) # upload hook to dataset
ssh(['chmod', '+x', hook_remote_target]) # and make it executable
def create_postupdate_hook(path, ssh, dataset):
# location of post-update hook file, logs folder on remote target
hooks_remote_dir = opj(path, '.git', 'hooks')
hook_remote_target = opj(hooks_remote_dir, 'post-update')
# post-update hook should create its log directory if doesn't exist
logs_remote_dir = opj(path, WEB_META_LOG)
make_log_dir = 'mkdir -p "{}"'.format(logs_remote_dir)
# create json command for current dataset
json_command = r'''
mkdir -p {};
( which datalad > /dev/null \
&& ( cd ..; GIT_DIR=$PWD/.git datalad ls -a --json file '{}'; ) \
|| echo "no datalad found - skipping generation of indexes for web frontend"; \
) &> "{}/{}"
'''.format(logs_remote_dir, str(path), logs_remote_dir,
'datalad-publish-hook-$(date +%s).log' % TIMESTAMP_FMT)
# collate content for post_update hook
hook_content = '\n'.join([
'#!/bin/bash', 'git update-server-info', make_log_dir, json_command
])
with make_tempfile(content=hook_content
) as tempf: # create post_update hook script
ssh.copy(tempf, hook_remote_target) # upload hook to dataset
ssh(['chmod', '+x', hook_remote_target]) # and make it executable
def prep_tmphome():
# re core.askPass:
# Don't let git ask for credentials in CI runs. Note, that this variable
# technically is not a flag, but an executable (which is why name and value
# are a bit confusing here - we just want a no-op basically). The environment
# variable GIT_ASKPASS overwrites this, but neither env var nor this config
# are supported by git-credential on all systems and git versions (most recent
# ones should work either way, though). Hence use both across CI builds.
gitconfig = """\
[user]
name = DataLad Tester
email = [email protected]
[core]
askPass =
[datalad "log"]
exc = 1
[annex "security"]
# from annex 6.20180626 file:/// and http://localhost access isn't
# allowed by default
allowed-url-schemes = http https file
allowed-http-addresses = all
"""
# TODO: split into a function + context manager
with make_tempfile(mkdir=True) as new_home:
pass
# register for clean-up on exit
_TEMP_PATHS_GENERATED.append(new_home)
# populate default config
new_home = Path(new_home)
new_home.mkdir(parents=True, exist_ok=True)
cfg_file = new_home / '.gitconfig'
cfg_file.write_text(gitconfig)
return new_home, cfg_file
def _test_smoke_pipelines(func, args, kwargs={}):
with make_tempfile(mkdir=True) as tmpdir:
AnnexRepo(tmpdir, create=True)
with chpwd(tmpdir):
with swallow_logs():
for p in [func(*args, **kwargs)]:
ok_(len(p) > 1)
def upload_web_interface(path, ssh, shared, ui):
# path to web interface resources on local
webui_local = opj(dirname(datalad.__file__), 'resources', 'website')
# local html to dataset
html_local = opj(webui_local, "index.html")
# name and location of web-interface html on target
html_targetname = {True: ui, False: "index.html"}[isinstance(ui, str)]
html_target = opj(path, html_targetname)
# upload ui html to target
ssh.copy(html_local, html_target)
# upload assets to the dataset
webresources_local = opj(webui_local, 'assets')
webresources_remote = opj(path, WEB_HTML_DIR)
ssh(['mkdir', '-p', webresources_remote])
ssh.copy(webresources_local, webresources_remote, recursive=True)
# minimize and upload js assets
for js_file in glob(opj(webresources_local, 'js', '*.js')):
with open(js_file) as asset:
try:
from jsmin import jsmin
minified = jsmin(asset.read()) # minify asset
except ImportError:
lgr.warning(
"Will not minify web interface javascript, no jsmin available"
)
minified = asset.read() # no minify available
with make_tempfile(content=minified
) as tempf: # write minified to tempfile
js_name = js_file.split('/')[-1]
ssh.copy(tempf,
opj(webresources_remote, 'assets', 'js',
js_name)) # and upload js
# explicitly make web+metadata dir of dataset world-readable, if shared set to 'all'
mode = None
if shared in (True, 'true', 'all', 'world', 'everybody'):
mode = 'a+rX'
elif shared == 'group':
mode = 'g+rX'
elif str(shared).startswith('0'):
mode = shared
if mode:
ssh([
'chmod', mode, '-R',
dirname(webresources_remote),
opj(path, 'index.html')
])
def upload_web_interface(path, ssh, shared, ui):
# path to web interface resources on local
webui_local = opj(dirname(datalad.__file__), 'resources', 'website')
# local html to dataset
html_local = opj(webui_local, "index.html")
# name and location of web-interface html on target
html_targetname = {True: ui, False: "index.html"}[isinstance(ui, str)]
html_target = opj(path, html_targetname)
# upload ui html to target
ssh.put(html_local, html_target)
# upload assets to the dataset
webresources_local = opj(webui_local, 'assets')
webresources_remote = opj(path, WEB_HTML_DIR)
ssh('mkdir -p {}'.format(sh_quote(webresources_remote)))
ssh.put(webresources_local, webresources_remote, recursive=True)
# minimize and upload js assets
for js_file in glob(opj(webresources_local, 'js', '*.js')):
with open(js_file) as asset:
try:
from jsmin import jsmin
# jsmin = lambda x: x # no minimization
minified = jsmin(asset.read()) # minify asset
except ImportError:
lgr.warning(
"Will not minify web interface javascript, no jsmin available")
minified = asset.read() # no minify available
with make_tempfile(content=minified) as tempf: # write minified to tempfile
js_name = js_file.split('/')[-1]
ssh.put(tempf, opj(webresources_remote, 'assets', 'js', js_name)) # and upload js
# explicitly make web+metadata dir of dataset world-readable, if shared set to 'all'
mode = None
if shared in (True, 'true', 'all', 'world', 'everybody'):
mode = 'a+rX'
elif shared == 'group':
mode = 'g+rX'
elif str(shared).startswith('0'):
mode = shared
if mode:
ssh('chmod {} -R {} {}'.format(
mode,
sh_quote(dirname(webresources_remote)),
sh_quote(opj(path, 'index.html'))))
def create_postupdate_hook(path, ssh, dataset):
# location of post-update hook file, logs folder on remote target
hooks_remote_dir = opj(path, '.git', 'hooks')
# make sure hooks directory exists (see #1251)
ssh('mkdir -p {}'.format(sh_quote(hooks_remote_dir)))
hook_remote_target = opj(hooks_remote_dir, 'post-update')
# create json command for current dataset
log_filename = 'datalad-publish-hook-$(date +%s).log' % TIMESTAMP_FMT
hook_content = r'''#!/bin/bash
git update-server-info
#
# DataLad
#
# (Re)generate meta-data for DataLad Web UI and possibly init new submodules
dsdir="$(dirname $0)/../.."
logfile="$dsdir/{WEB_META_LOG}/{log_filename}"
if [ ! -e "$dsdir/.git" ]; then
echo Assumption of being under .git has failed >&2
exit 1
fi
mkdir -p "$dsdir/{WEB_META_LOG}" # assure logs directory exists
# Avoid file name collisions.
suffix=0
logfile_orig="$logfile"
while [ -f "$logfile" ]; do
suffix=$(( $suffix + 1 ))
logfile="$logfile_orig.$suffix"
done
( which datalad > /dev/null \
&& ( cd "$dsdir"; GIT_DIR="$PWD/.git" datalad ls -a --json file .; ) \
|| echo "E: no datalad found - skipping generation of indexes for web frontend"; \
) &> "$logfile"
'''.format(WEB_META_LOG=WEB_META_LOG, **locals())
with make_tempfile(content=hook_content) as tempf:
# create post_update hook script
# upload hook to dataset
ssh.put(tempf, hook_remote_target)
# and make it executable
ssh('chmod +x {}'.format(sh_quote(hook_remote_target)))
def create_postupdate_hook(path, ssh, dataset):
# location of post-update hook file, logs folder on remote target
hooks_remote_dir = opj(path, '.git', 'hooks')
# make sure hooks directory exists (see #1251)
ssh('mkdir -p {}'.format(sh_quote(hooks_remote_dir)))
hook_remote_target = opj(hooks_remote_dir, 'post-update')
# create json command for current dataset
log_filename = 'datalad-publish-hook-$(date +%s).log' % TIMESTAMP_FMT
hook_content = r'''#!/bin/bash
git update-server-info
#
# DataLad
#
# (Re)generate meta-data for DataLad Web UI and possibly init new submodules
dsdir="$(dirname $0)/../.."
logfile="$dsdir/{WEB_META_LOG}/{log_filename}"
if [ ! -e "$dsdir/.git" ]; then
echo Assumption of being under .git has failed >&2
exit 1
fi
mkdir -p "$dsdir/{WEB_META_LOG}" # assure logs directory exists
( which datalad > /dev/null \
&& ( cd "$dsdir"; GIT_DIR="$PWD/.git" datalad ls -a --json file .; ) \
|| echo "E: no datalad found - skipping generation of indexes for web frontend"; \
) &> "$logfile"
# Some submodules might have been added and thus we better init them
( cd "$dsdir"; git submodule update --init || : ; ) >> "$logfile" 2>&1
'''.format(WEB_META_LOG=WEB_META_LOG, **locals())
with make_tempfile(content=hook_content) as tempf:
# create post_update hook script
# upload hook to dataset
ssh.copy(tempf, hook_remote_target)
# and make it executable
ssh('chmod +x {}'.format(sh_quote(hook_remote_target)))
def setup_package():
import os
from datalad import consts
_test_states['HOME'] = os.environ.get('HOME', None)
_test_states['DATASETS_TOPURL_ENV'] = os.environ.get(
'DATALAD_DATASETS_TOPURL', None)
_test_states['DATASETS_TOPURL'] = consts.DATASETS_TOPURL
os.environ[
'DATALAD_DATASETS_TOPURL'] = consts.DATASETS_TOPURL = 'http://datasets-tests.datalad.org/'
# To overcome pybuild overriding HOME but us possibly wanting our
# own HOME where we pre-setup git for testing (name, email)
if 'GIT_HOME' in os.environ:
os.environ['HOME'] = os.environ['GIT_HOME']
else:
# we setup our own new HOME, the BEST and HUGE one
from datalad.utils import make_tempfile
from datalad.tests import _TEMP_PATHS_GENERATED
# TODO: split into a function + context manager
with make_tempfile(mkdir=True) as new_home:
os.environ['HOME'] = new_home
if not os.path.exists(new_home):
os.makedirs(new_home)
with open(os.path.join(new_home, '.gitconfig'), 'w') as f:
f.write("""\
[user]
name = DataLad Tester
email = [email protected]
""")
_TEMP_PATHS_GENERATED.append(new_home)
# For now we will just verify that it is ready to run the tests
from datalad.support.gitrepo import check_git_configured
check_git_configured()
# To overcome pybuild by default defining http{,s}_proxy we would need
# to define them to e.g. empty value so it wouldn't bother touching them.
# But then haskell libraries do not digest empty value nicely, so we just
# pop them out from the environment
for ev in ('http_proxy', 'https_proxy'):
if ev in os.environ and not (os.environ[ev]):
lgr.debug("Removing %s from the environment since it is empty", ev)
os.environ.pop(ev)
# During tests we allow for "insecure" access to local file:// and
# http://localhost URLs since all of them either generated as tests
# fixtures or cloned from trusted sources
from datalad.support.annexrepo import AnnexRepo
AnnexRepo._ALLOW_LOCAL_URLS = True
DATALAD_LOG_LEVEL = os.environ.get('DATALAD_LOG_LEVEL', None)
if DATALAD_LOG_LEVEL is None:
# very very silent. Tests introspecting logs should use
# swallow_logs(new_level=...)
_test_states['loglevel'] = lgr.getEffectiveLevel()
lgr.setLevel(100)
# And we should also set it within environ so underlying commands also stay silent
_test_states['DATALAD_LOG_LEVEL'] = DATALAD_LOG_LEVEL
os.environ['DATALAD_LOG_LEVEL'] = '100'
else:
# We are not overriding them, since explicitly were asked to have some log level
_test_states['loglevel'] = None
# Set to non-interactive UI
from datalad.ui import ui
_test_states['ui_backend'] = ui.backend
# obtain() since that one consults for the default value
ui.set_backend(cfg.obtain('datalad.tests.ui.backend'))
def postclonecfg_ria(ds, props):
"""Configure a dataset freshly cloned from a RIA store"""
repo = ds.repo
# RIA uses hashdir mixed, copying data to it via git-annex (if cloned via
# ssh) would make it see a bare repo and establish a hashdir lower annex
# object tree.
# Moreover, we want the ORA remote to receive all data for the store, so its
# objects could be moved into archives (the main point of a RIA store).
RIA_REMOTE_NAME = 'origin' # don't hardcode everywhere
ds.config.set(
'remote.{}.annex-ignore'.format(RIA_REMOTE_NAME), 'true',
where='local')
# chances are that if this dataset came from a RIA store, its subdatasets
# may live there too. Place a subdataset source candidate config that makes
# get probe this RIA store when obtaining subdatasets
ds.config.set(
# we use the label 'origin' for this candidate in order to not have to
# generate a complicated name from the actual source specification.
# we pick a cost of 200 to sort it before datalad's default candidates
# for non-RIA URLs, because they prioritize hierarchical layouts that
# cannot be found in a RIA store
'datalad.get.subdataset-source-candidate-200origin',
# use the entire original URL, up to the fragment + plus dataset ID
# placeholder, this should make things work with any store setup we
# support (paths, ports, ...)
props['source'].split('#', maxsplit=1)[0] + '#{id}',
where='local')
# setup publication dependency, if a corresponding special remote exists
# and was enabled (there could be RIA stores that actually only have repos)
# make this function be a generator
ora_remotes = [s for s in ds.siblings('query', result_renderer='disabled')
if s.get('annex-externaltype') == 'ora']
if not ora_remotes and any(
r.get('externaltype') == 'ora'
for r in (repo.get_special_remotes().values()
if hasattr(repo, 'get_special_remotes')
else [])):
# no ORA remote autoenabled, but configuration known about at least one.
# Let's check origin's config for datalad.ora-remote.uuid as stored by
# create-sibling-ria and enable try enabling that one.
lgr.debug("Found no autoenabled ORA special remote. Trying to look it "
"up in source config ...")
# First figure whether we cloned via SSH, HTTP or local path and then
# get that config file the same way:
config_content = None
scheme = props['giturl'].split(':', 1)[0]
if scheme in ['http', 'https']:
try:
config_content = download_url(
"{}{}config".format(
props['giturl'],
'/' if not props['giturl'].endswith('/') else ''))
except DownloadError as e:
lgr.debug("Failed to get config file from source:\n%s",
exc_str(e))
elif scheme == 'ssh':
# TODO: switch the following to proper command abstraction:
# SSHRemoteIO ignores the path part ATM. No remote CWD! (To be
# changed with command abstractions). So we need to get that part to
# have a valid path to origin's config file:
cfg_path = PurePosixPath(URL(props['giturl']).path) / 'config'
op = SSHRemoteIO(props['giturl'])
try:
config_content = op.read_file(cfg_path)
except RIARemoteError as e:
lgr.debug("Failed to get config file from source: %s",
exc_str(e))
elif scheme == 'file':
# TODO: switch the following to proper command abstraction:
op = LocalIO()
cfg_path = Path(URL(props['giturl']).localpath) / 'config'
try:
config_content = op.read_file(cfg_path)
except (RIARemoteError, OSError) as e:
lgr.debug("Failed to get config file from source: %s",
exc_str(e))
else:
lgr.debug("Unknown URL-Scheme %s in %s. Can handle SSH, HTTP or "
"FILE scheme URLs.", scheme, props['source'])
# 3. And read it
org_uuid = None
if config_content:
# TODO: We might be able to spare the saving to a file.
# "git config -f -" is not explicitly documented but happens
# to work and would read from stdin. Make sure we know this
# works for required git versions and on all platforms.
with make_tempfile(content=config_content) as cfg_file:
runner = GitWitlessRunner()
try:
result = runner.run(
['git', 'config', '-f', cfg_file,
'datalad.ora-remote.uuid'],
protocol=StdOutCapture
)
org_uuid = result['stdout'].strip()
except CommandError as e:
# doesn't contain what we are looking for
lgr.debug("Found no UUID for ORA special remote at "
"'%s' (%s)", RIA_REMOTE_NAME, exc_str(e))
# Now, enable it. If annex-init didn't fail to enable it as stored, we
# wouldn't end up here, so enable with store URL as suggested by the URL
# we cloned from.
if org_uuid:
srs = repo.get_special_remotes()
if org_uuid in srs.keys():
# TODO: - Double-check autoenable value and only do this when
# true?
# - What if still fails? -> Annex shouldn't change config
# in that case
# we only need the store:
new_url = props['source'].split('#')[0]
try:
repo.enable_remote(srs[org_uuid]['name'],
options=['url={}'.format(new_url)]
)
lgr.info("Reconfigured %s for %s",
srs[org_uuid]['name'], new_url)
# update ora_remotes for considering publication dependency
# below
ora_remotes = [s for s in
ds.siblings('query',
result_renderer='disabled')
if s.get('annex-externaltype', None) ==
'ora']
except CommandError as e:
lgr.debug("Failed to reconfigure ORA special remote: %s",
exc_str(e))
else:
lgr.debug("Unknown ORA special remote uuid at '%s': %s",
RIA_REMOTE_NAME, org_uuid)
if ora_remotes:
if len(ora_remotes) == 1:
yield from ds.siblings('configure',
name=RIA_REMOTE_NAME,
publish_depends=ora_remotes[0]['name'],
result_filter=None,
result_renderer='disabled')
else:
lgr.warning("Found multiple ORA remotes. Couldn't decide which "
"publishing to 'origin' should depend on: %s. Consider "
"running 'datalad siblings configure -s origin "
"--publish-depends ORAREMOTENAME' to set publication "
"dependency manually.",
[r['name'] for r in ora_remotes])
def test_make_tempfile():
# check if mkdir, content conflict caught
with assert_raises(ValueError):
with make_tempfile(content="blah", mkdir=True): # pragma: no cover
pass
def setup_package():
import os
from datalad.utils import on_osx
from datalad.tests import _TEMP_PATHS_GENERATED
if on_osx:
# enforce honoring TMPDIR (see gh-5307)
import tempfile
tempfile.tempdir = os.environ.get('TMPDIR', tempfile.gettempdir())
from datalad import consts
_test_states['env'] = {}
def set_envvar(v, val):
"""Memoize and then set env var"""
_test_states['env'][v] = os.environ.get(v, None)
os.environ[v] = val
_test_states['DATASETS_TOPURL'] = consts.DATASETS_TOPURL
consts.DATASETS_TOPURL = 'http://datasets-tests.datalad.org/'
set_envvar('DATALAD_DATASETS_TOPURL', consts.DATASETS_TOPURL)
from datalad.tests.utils import (
DEFAULT_BRANCH,
DEFAULT_REMOTE,
)
set_envvar(
"GIT_CONFIG_PARAMETERS",
"'init.defaultBranch={}' 'clone.defaultRemoteName={}'".format(
DEFAULT_BRANCH, DEFAULT_REMOTE))
# To overcome pybuild overriding HOME but us possibly wanting our
# own HOME where we pre-setup git for testing (name, email)
if 'GIT_HOME' in os.environ:
set_envvar('HOME', os.environ['GIT_HOME'])
set_envvar('DATALAD_LOG_EXC', "1")
else:
# we setup our own new HOME, the BEST and HUGE one
from datalad.utils import make_tempfile
# TODO: split into a function + context manager
with make_tempfile(mkdir=True) as new_home:
pass
for v, val in get_home_envvars(new_home).items():
set_envvar(v, val)
if not os.path.exists(new_home):
os.makedirs(new_home)
with open(os.path.join(new_home, '.gitconfig'), 'w') as f:
f.write("""\
[user]
name = DataLad Tester
email = [email protected]
[datalad "log"]
exc = 1
""")
_TEMP_PATHS_GENERATED.append(new_home)
# Re-load ConfigManager, since otherwise it won't consider global config
# from new $HOME (see gh-4153
cfg.reload(force=True)
from datalad.interface.common_cfg import compute_cfg_defaults
compute_cfg_defaults()
# datalad.locations.sockets has likely changed. Discard any cached values.
ssh_manager._socket_dir = None
# To overcome pybuild by default defining http{,s}_proxy we would need
# to define them to e.g. empty value so it wouldn't bother touching them.
# But then haskell libraries do not digest empty value nicely, so we just
# pop them out from the environment
for ev in ('http_proxy', 'https_proxy'):
if ev in os.environ and not (os.environ[ev]):
lgr.debug("Removing %s from the environment since it is empty", ev)
os.environ.pop(ev)
# During tests we allow for "insecure" access to local file:// and
# http://localhost URLs since all of them either generated as tests
# fixtures or cloned from trusted sources
from datalad.support.annexrepo import AnnexRepo
AnnexRepo._ALLOW_LOCAL_URLS = True
DATALAD_LOG_LEVEL = os.environ.get('DATALAD_LOG_LEVEL', None)
if DATALAD_LOG_LEVEL is None:
# very very silent. Tests introspecting logs should use
# swallow_logs(new_level=...)
_test_states['loglevel'] = lgr.getEffectiveLevel()
lgr.setLevel(100)
# And we should also set it within environ so underlying commands also stay silent
set_envvar('DATALAD_LOG_LEVEL', '100')
else:
# We are not overriding them, since explicitly were asked to have some log level
_test_states['loglevel'] = None
# Set to non-interactive UI
from datalad.ui import ui
_test_states['ui_backend'] = ui.backend
# obtain() since that one consults for the default value
ui.set_backend(cfg.obtain('datalad.tests.ui.backend'))
# Monkey patch nose so it does not ERROR out whenever code asks for fileno
# of the output. See https://github.com/nose-devs/nose/issues/6
from io import StringIO as OrigStringIO
class StringIO(OrigStringIO):
fileno = lambda self: 1
encoding = None
from nose.ext import dtcompat
from nose.plugins import capture, multiprocess, plugintest
dtcompat.StringIO = StringIO
capture.StringIO = StringIO
multiprocess.StringIO = StringIO
plugintest.StringIO = StringIO
# in order to avoid having to fiddle with rather uncommon
# file:// URLs in the tests, have a standard HTTP server
# that serves an 'httpserve' directory in the test HOME
# the URL will be available from datalad.test_http_server.url
from datalad.tests.utils import HTTPPath
import tempfile
global test_http_server
serve_path = tempfile.mkdtemp(
dir=cfg.get("datalad.tests.temp.dir"),
prefix='httpserve',
)
test_http_server = HTTPPath(serve_path)
test_http_server.start()
_TEMP_PATHS_GENERATED.append(serve_path)
if cfg.obtain('datalad.tests.setup.testrepos'):
lgr.debug("Pre-populating testrepos")
from datalad.tests.utils import with_testrepos
with_testrepos()(lambda repo: 1)()
def setup_package():
import os
from datalad import consts
_test_states['HOME'] = os.environ.get('HOME', None)
_test_states['DATASETS_TOPURL_ENV'] = os.environ.get('DATALAD_DATASETS_TOPURL', None)
_test_states['DATASETS_TOPURL'] = consts.DATASETS_TOPURL
os.environ['DATALAD_DATASETS_TOPURL'] = consts.DATASETS_TOPURL = 'http://datasets-tests.datalad.org/'
# To overcome pybuild overriding HOME but us possibly wanting our
# own HOME where we pre-setup git for testing (name, email)
if 'GIT_HOME' in os.environ:
os.environ['HOME'] = os.environ['GIT_HOME']
else:
# we setup our own new HOME, the BEST and HUGE one
from datalad.utils import make_tempfile
from datalad.tests import _TEMP_PATHS_GENERATED
# TODO: split into a function + context manager
with make_tempfile(mkdir=True) as new_home:
os.environ['HOME'] = new_home
if not os.path.exists(new_home):
os.makedirs(new_home)
with open(os.path.join(new_home, '.gitconfig'), 'w') as f:
f.write("""\
[user]
name = DataLad Tester
email = [email protected]
""")
_TEMP_PATHS_GENERATED.append(new_home)
# If there is a bundled git, make sure GitPython uses it too
# (some parts of the test utilities still rely on GitPython)
from datalad.cmd import GitRunner
GitRunner._check_git_path()
if GitRunner._GIT_PATH:
import os
os.environ['GIT_PYTHON_GIT_EXECUTABLE'] = \
os.path.join(GitRunner._GIT_PATH, 'git')
# Re-load ConfigManager, since otherwise it won't consider global config
# from new $HOME (see gh-4153
cfg.reload(force=True)
# To overcome pybuild by default defining http{,s}_proxy we would need
# to define them to e.g. empty value so it wouldn't bother touching them.
# But then haskell libraries do not digest empty value nicely, so we just
# pop them out from the environment
for ev in ('http_proxy', 'https_proxy'):
if ev in os.environ and not (os.environ[ev]):
lgr.debug("Removing %s from the environment since it is empty", ev)
os.environ.pop(ev)
# During tests we allow for "insecure" access to local file:// and
# http://localhost URLs since all of them either generated as tests
# fixtures or cloned from trusted sources
from datalad.support.annexrepo import AnnexRepo
AnnexRepo._ALLOW_LOCAL_URLS = True
DATALAD_LOG_LEVEL = os.environ.get('DATALAD_LOG_LEVEL', None)
if DATALAD_LOG_LEVEL is None:
# very very silent. Tests introspecting logs should use
# swallow_logs(new_level=...)
_test_states['loglevel'] = lgr.getEffectiveLevel()
lgr.setLevel(100)
# And we should also set it within environ so underlying commands also stay silent
_test_states['DATALAD_LOG_LEVEL'] = DATALAD_LOG_LEVEL
os.environ['DATALAD_LOG_LEVEL'] = '100'
else:
# We are not overriding them, since explicitly were asked to have some log level
_test_states['loglevel'] = None
# Set to non-interactive UI
from datalad.ui import ui
_test_states['ui_backend'] = ui.backend
# obtain() since that one consults for the default value
ui.set_backend(cfg.obtain('datalad.tests.ui.backend'))
# Monkey patch nose so it does not ERROR out whenever code asks for fileno
# of the output. See https://github.com/nose-devs/nose/issues/6
from io import StringIO as OrigStringIO
class StringIO(OrigStringIO):
fileno = lambda self: 1
encoding = None
from nose.ext import dtcompat
from nose.plugins import capture, multiprocess, plugintest
dtcompat.StringIO = StringIO
capture.StringIO = StringIO
multiprocess.StringIO = StringIO
plugintest.StringIO = StringIO
def setup_package():
import os
from datalad import consts
_test_states['HOME'] = os.environ.get('HOME', None)
_test_states['DATASETS_TOPURL_ENV'] = os.environ.get('DATALAD_DATASETS_TOPURL', None)
_test_states['DATASETS_TOPURL'] = consts.DATASETS_TOPURL
os.environ['DATALAD_DATASETS_TOPURL'] = consts.DATASETS_TOPURL = 'http://datasets-tests.datalad.org/'
# To overcome pybuild overriding HOME but us possibly wanting our
# own HOME where we pre-setup git for testing (name, email)
if 'GIT_HOME' in os.environ:
os.environ['HOME'] = os.environ['GIT_HOME']
else:
# we setup our own new HOME, the BEST and HUGE one
from datalad.utils import make_tempfile
from datalad.tests import _TEMP_PATHS_GENERATED
# TODO: split into a function + context manager
with make_tempfile(mkdir=True) as new_home:
os.environ['HOME'] = new_home
if not os.path.exists(new_home):
os.makedirs(new_home)
with open(os.path.join(new_home, '.gitconfig'), 'w') as f:
f.write("""\
[user]
name = DataLad Tester
email = [email protected]
""")
_TEMP_PATHS_GENERATED.append(new_home)
# For now we will just verify that it is ready to run the tests
from datalad.support.gitrepo import check_git_configured
check_git_configured()
# To overcome pybuild by default defining http{,s}_proxy we would need
# to define them to e.g. empty value so it wouldn't bother touching them.
# But then haskell libraries do not digest empty value nicely, so we just
# pop them out from the environment
for ev in ('http_proxy', 'https_proxy'):
if ev in os.environ and not (os.environ[ev]):
lgr.debug("Removing %s from the environment since it is empty", ev)
os.environ.pop(ev)
# During tests we allow for "insecure" access to local file:// and
# http://localhost URLs since all of them either generated as tests
# fixtures or cloned from trusted sources
from datalad.support.annexrepo import AnnexRepo
AnnexRepo._ALLOW_LOCAL_URLS = True
DATALAD_LOG_LEVEL = os.environ.get('DATALAD_LOG_LEVEL', None)
if DATALAD_LOG_LEVEL is None:
# very very silent. Tests introspecting logs should use
# swallow_logs(new_level=...)
_test_states['loglevel'] = lgr.getEffectiveLevel()
lgr.setLevel(100)
# And we should also set it within environ so underlying commands also stay silent
_test_states['DATALAD_LOG_LEVEL'] = DATALAD_LOG_LEVEL
os.environ['DATALAD_LOG_LEVEL'] = '100'
else:
# We are not overriding them, since explicitly were asked to have some log level
_test_states['loglevel'] = None
# Set to non-interactive UI
from datalad.ui import ui
_test_states['ui_backend'] = ui.backend
# obtain() since that one consults for the default value
ui.set_backend(cfg.obtain('datalad.tests.ui.backend'))
