def test_ria_postclonecfg(): if not has_symlink_capability(): # This is needed to create an ORA remote using an URL for upload, # that is then invalidated later on (delete the symlink it's based on). raise SkipTest("Can't create symlinks") from datalad.utils import make_tempfile from datalad.tests.utils import HTTPPath with make_tempfile(mkdir=True) as lcl, make_tempfile(mkdir=True) as store: id = _postclonetest_prepare(lcl, store) # test cloning via ria+file:// yield _test_ria_postclonecfg, Path(store).as_uri(), id # Note: HTTP disabled for now. Requires proper implementation in ORA # remote. See # https://github.com/datalad/datalad/pull/4203#discussion_r410284649 # # test cloning via ria+http:// # with HTTPPath(store) as url: # yield _test_ria_postclonecfg, url, id # test cloning via ria+ssh:// yield skip_ssh(_test_ria_postclonecfg), \ "ssh://datalad-test:{}".format(Path(store).as_posix()), id
def create_postupdate_hook(path, ssh, dataset): # location of post-update hook file, logs folder on remote target hooks_remote_dir = opj(path, '.git', 'hooks') hook_remote_target = opj(hooks_remote_dir, 'post-update') # post-update hook should create its log directory if doesn't exist logs_remote_dir = opj(path, WEB_META_LOG) make_log_dir = 'mkdir -p "{}"'.format(logs_remote_dir) # create json command for current dataset json_command = r''' mkdir -p {}; ( which datalad > /dev/null \ && ( cd ..; GIT_DIR=$PWD/.git datalad ls -a --json file '{}'; ) \ || echo "no datalad found - skipping generation of indexes for web frontend"; \ ) &> "{}/{}" '''.format(logs_remote_dir, str(path), logs_remote_dir, 'datalad-publish-hook-$(date +%s).log' % TIMESTAMP_FMT) # collate content for post_update hook hook_content = '\n'.join(['#!/bin/bash', 'git update-server-info', make_log_dir, json_command]) with make_tempfile(content=hook_content) as tempf: # create post_update hook script ssh.copy(tempf, hook_remote_target) # upload hook to dataset ssh(['chmod', '+x', hook_remote_target]) # and make it executable
def create_postupdate_hook(path, ssh, dataset): # location of post-update hook file, logs folder on remote target hooks_remote_dir = opj(path, '.git', 'hooks') hook_remote_target = opj(hooks_remote_dir, 'post-update') # post-update hook should create its log directory if doesn't exist logs_remote_dir = opj(path, WEB_META_LOG) make_log_dir = 'mkdir -p "{}"'.format(logs_remote_dir) # create json command for current dataset json_command = r''' mkdir -p {}; ( which datalad > /dev/null \ && ( cd ..; GIT_DIR=$PWD/.git datalad ls -a --json file '{}'; ) \ || echo "no datalad found - skipping generation of indexes for web frontend"; \ ) &> "{}/{}" '''.format(logs_remote_dir, str(path), logs_remote_dir, 'datalad-publish-hook-$(date +%s).log' % TIMESTAMP_FMT) # collate content for post_update hook hook_content = '\n'.join([ '#!/bin/bash', 'git update-server-info', make_log_dir, json_command ]) with make_tempfile(content=hook_content ) as tempf: # create post_update hook script ssh.copy(tempf, hook_remote_target) # upload hook to dataset ssh(['chmod', '+x', hook_remote_target]) # and make it executable
def prep_tmphome(): # re core.askPass: # Don't let git ask for credentials in CI runs. Note, that this variable # technically is not a flag, but an executable (which is why name and value # are a bit confusing here - we just want a no-op basically). The environment # variable GIT_ASKPASS overwrites this, but neither env var nor this config # are supported by git-credential on all systems and git versions (most recent # ones should work either way, though). Hence use both across CI builds. gitconfig = """\ [user] name = DataLad Tester email = [email protected] [core] askPass = [datalad "log"] exc = 1 [annex "security"] # from annex 6.20180626 file:/// and http://localhost access isn't # allowed by default allowed-url-schemes = http https file allowed-http-addresses = all """ # TODO: split into a function + context manager with make_tempfile(mkdir=True) as new_home: pass # register for clean-up on exit _TEMP_PATHS_GENERATED.append(new_home) # populate default config new_home = Path(new_home) new_home.mkdir(parents=True, exist_ok=True) cfg_file = new_home / '.gitconfig' cfg_file.write_text(gitconfig) return new_home, cfg_file
def _test_smoke_pipelines(func, args, kwargs={}): with make_tempfile(mkdir=True) as tmpdir: AnnexRepo(tmpdir, create=True) with chpwd(tmpdir): with swallow_logs(): for p in [func(*args, **kwargs)]: ok_(len(p) > 1)
def upload_web_interface(path, ssh, shared, ui): # path to web interface resources on local webui_local = opj(dirname(datalad.__file__), 'resources', 'website') # local html to dataset html_local = opj(webui_local, "index.html") # name and location of web-interface html on target html_targetname = {True: ui, False: "index.html"}[isinstance(ui, str)] html_target = opj(path, html_targetname) # upload ui html to target ssh.copy(html_local, html_target) # upload assets to the dataset webresources_local = opj(webui_local, 'assets') webresources_remote = opj(path, WEB_HTML_DIR) ssh(['mkdir', '-p', webresources_remote]) ssh.copy(webresources_local, webresources_remote, recursive=True) # minimize and upload js assets for js_file in glob(opj(webresources_local, 'js', '*.js')): with open(js_file) as asset: try: from jsmin import jsmin minified = jsmin(asset.read()) # minify asset except ImportError: lgr.warning( "Will not minify web interface javascript, no jsmin available" ) minified = asset.read() # no minify available with make_tempfile(content=minified ) as tempf: # write minified to tempfile js_name = js_file.split('/')[-1] ssh.copy(tempf, opj(webresources_remote, 'assets', 'js', js_name)) # and upload js # explicitly make web+metadata dir of dataset world-readable, if shared set to 'all' mode = None if shared in (True, 'true', 'all', 'world', 'everybody'): mode = 'a+rX' elif shared == 'group': mode = 'g+rX' elif str(shared).startswith('0'): mode = shared if mode: ssh([ 'chmod', mode, '-R', dirname(webresources_remote), opj(path, 'index.html') ])
def upload_web_interface(path, ssh, shared, ui): # path to web interface resources on local webui_local = opj(dirname(datalad.__file__), 'resources', 'website') # local html to dataset html_local = opj(webui_local, "index.html") # name and location of web-interface html on target html_targetname = {True: ui, False: "index.html"}[isinstance(ui, str)] html_target = opj(path, html_targetname) # upload ui html to target ssh.put(html_local, html_target) # upload assets to the dataset webresources_local = opj(webui_local, 'assets') webresources_remote = opj(path, WEB_HTML_DIR) ssh('mkdir -p {}'.format(sh_quote(webresources_remote))) ssh.put(webresources_local, webresources_remote, recursive=True) # minimize and upload js assets for js_file in glob(opj(webresources_local, 'js', '*.js')): with open(js_file) as asset: try: from jsmin import jsmin # jsmin = lambda x: x # no minimization minified = jsmin(asset.read()) # minify asset except ImportError: lgr.warning( "Will not minify web interface javascript, no jsmin available") minified = asset.read() # no minify available with make_tempfile(content=minified) as tempf: # write minified to tempfile js_name = js_file.split('/')[-1] ssh.put(tempf, opj(webresources_remote, 'assets', 'js', js_name)) # and upload js # explicitly make web+metadata dir of dataset world-readable, if shared set to 'all' mode = None if shared in (True, 'true', 'all', 'world', 'everybody'): mode = 'a+rX' elif shared == 'group': mode = 'g+rX' elif str(shared).startswith('0'): mode = shared if mode: ssh('chmod {} -R {} {}'.format( mode, sh_quote(dirname(webresources_remote)), sh_quote(opj(path, 'index.html'))))
def create_postupdate_hook(path, ssh, dataset): # location of post-update hook file, logs folder on remote target hooks_remote_dir = opj(path, '.git', 'hooks') # make sure hooks directory exists (see #1251) ssh('mkdir -p {}'.format(sh_quote(hooks_remote_dir))) hook_remote_target = opj(hooks_remote_dir, 'post-update') # create json command for current dataset log_filename = 'datalad-publish-hook-$(date +%s).log' % TIMESTAMP_FMT hook_content = r'''#!/bin/bash git update-server-info # # DataLad # # (Re)generate meta-data for DataLad Web UI and possibly init new submodules dsdir="$(dirname $0)/../.." logfile="$dsdir/{WEB_META_LOG}/{log_filename}" if [ ! -e "$dsdir/.git" ]; then echo Assumption of being under .git has failed >&2 exit 1 fi mkdir -p "$dsdir/{WEB_META_LOG}" # assure logs directory exists # Avoid file name collisions. suffix=0 logfile_orig="$logfile" while [ -f "$logfile" ]; do suffix=$(( $suffix + 1 )) logfile="$logfile_orig.$suffix" done ( which datalad > /dev/null \ && ( cd "$dsdir"; GIT_DIR="$PWD/.git" datalad ls -a --json file .; ) \ || echo "E: no datalad found - skipping generation of indexes for web frontend"; \ ) &> "$logfile" '''.format(WEB_META_LOG=WEB_META_LOG, **locals()) with make_tempfile(content=hook_content) as tempf: # create post_update hook script # upload hook to dataset ssh.put(tempf, hook_remote_target) # and make it executable ssh('chmod +x {}'.format(sh_quote(hook_remote_target)))
def create_postupdate_hook(path, ssh, dataset): # location of post-update hook file, logs folder on remote target hooks_remote_dir = opj(path, '.git', 'hooks') # make sure hooks directory exists (see #1251) ssh('mkdir -p {}'.format(sh_quote(hooks_remote_dir))) hook_remote_target = opj(hooks_remote_dir, 'post-update') # create json command for current dataset log_filename = 'datalad-publish-hook-$(date +%s).log' % TIMESTAMP_FMT hook_content = r'''#!/bin/bash git update-server-info # # DataLad # # (Re)generate meta-data for DataLad Web UI and possibly init new submodules dsdir="$(dirname $0)/../.." logfile="$dsdir/{WEB_META_LOG}/{log_filename}" if [ ! -e "$dsdir/.git" ]; then echo Assumption of being under .git has failed >&2 exit 1 fi mkdir -p "$dsdir/{WEB_META_LOG}" # assure logs directory exists ( which datalad > /dev/null \ && ( cd "$dsdir"; GIT_DIR="$PWD/.git" datalad ls -a --json file .; ) \ || echo "E: no datalad found - skipping generation of indexes for web frontend"; \ ) &> "$logfile" # Some submodules might have been added and thus we better init them ( cd "$dsdir"; git submodule update --init || : ; ) >> "$logfile" 2>&1 '''.format(WEB_META_LOG=WEB_META_LOG, **locals()) with make_tempfile(content=hook_content) as tempf: # create post_update hook script # upload hook to dataset ssh.copy(tempf, hook_remote_target) # and make it executable ssh('chmod +x {}'.format(sh_quote(hook_remote_target)))
def setup_package(): import os from datalad import consts _test_states['HOME'] = os.environ.get('HOME', None) _test_states['DATASETS_TOPURL_ENV'] = os.environ.get( 'DATALAD_DATASETS_TOPURL', None) _test_states['DATASETS_TOPURL'] = consts.DATASETS_TOPURL os.environ[ 'DATALAD_DATASETS_TOPURL'] = consts.DATASETS_TOPURL = 'http://datasets-tests.datalad.org/' # To overcome pybuild overriding HOME but us possibly wanting our # own HOME where we pre-setup git for testing (name, email) if 'GIT_HOME' in os.environ: os.environ['HOME'] = os.environ['GIT_HOME'] else: # we setup our own new HOME, the BEST and HUGE one from datalad.utils import make_tempfile from datalad.tests import _TEMP_PATHS_GENERATED # TODO: split into a function + context manager with make_tempfile(mkdir=True) as new_home: os.environ['HOME'] = new_home if not os.path.exists(new_home): os.makedirs(new_home) with open(os.path.join(new_home, '.gitconfig'), 'w') as f: f.write("""\ [user] name = DataLad Tester email = [email protected] """) _TEMP_PATHS_GENERATED.append(new_home) # For now we will just verify that it is ready to run the tests from datalad.support.gitrepo import check_git_configured check_git_configured() # To overcome pybuild by default defining http{,s}_proxy we would need # to define them to e.g. empty value so it wouldn't bother touching them. # But then haskell libraries do not digest empty value nicely, so we just # pop them out from the environment for ev in ('http_proxy', 'https_proxy'): if ev in os.environ and not (os.environ[ev]): lgr.debug("Removing %s from the environment since it is empty", ev) os.environ.pop(ev) # During tests we allow for "insecure" access to local file:// and # http://localhost URLs since all of them either generated as tests # fixtures or cloned from trusted sources from datalad.support.annexrepo import AnnexRepo AnnexRepo._ALLOW_LOCAL_URLS = True DATALAD_LOG_LEVEL = os.environ.get('DATALAD_LOG_LEVEL', None) if DATALAD_LOG_LEVEL is None: # very very silent. Tests introspecting logs should use # swallow_logs(new_level=...) _test_states['loglevel'] = lgr.getEffectiveLevel() lgr.setLevel(100) # And we should also set it within environ so underlying commands also stay silent _test_states['DATALAD_LOG_LEVEL'] = DATALAD_LOG_LEVEL os.environ['DATALAD_LOG_LEVEL'] = '100' else: # We are not overriding them, since explicitly were asked to have some log level _test_states['loglevel'] = None # Set to non-interactive UI from datalad.ui import ui _test_states['ui_backend'] = ui.backend # obtain() since that one consults for the default value ui.set_backend(cfg.obtain('datalad.tests.ui.backend'))
def postclonecfg_ria(ds, props): """Configure a dataset freshly cloned from a RIA store""" repo = ds.repo # RIA uses hashdir mixed, copying data to it via git-annex (if cloned via # ssh) would make it see a bare repo and establish a hashdir lower annex # object tree. # Moreover, we want the ORA remote to receive all data for the store, so its # objects could be moved into archives (the main point of a RIA store). RIA_REMOTE_NAME = 'origin' # don't hardcode everywhere ds.config.set( 'remote.{}.annex-ignore'.format(RIA_REMOTE_NAME), 'true', where='local') # chances are that if this dataset came from a RIA store, its subdatasets # may live there too. Place a subdataset source candidate config that makes # get probe this RIA store when obtaining subdatasets ds.config.set( # we use the label 'origin' for this candidate in order to not have to # generate a complicated name from the actual source specification. # we pick a cost of 200 to sort it before datalad's default candidates # for non-RIA URLs, because they prioritize hierarchical layouts that # cannot be found in a RIA store 'datalad.get.subdataset-source-candidate-200origin', # use the entire original URL, up to the fragment + plus dataset ID # placeholder, this should make things work with any store setup we # support (paths, ports, ...) props['source'].split('#', maxsplit=1)[0] + '#{id}', where='local') # setup publication dependency, if a corresponding special remote exists # and was enabled (there could be RIA stores that actually only have repos) # make this function be a generator ora_remotes = [s for s in ds.siblings('query', result_renderer='disabled') if s.get('annex-externaltype') == 'ora'] if not ora_remotes and any( r.get('externaltype') == 'ora' for r in (repo.get_special_remotes().values() if hasattr(repo, 'get_special_remotes') else [])): # no ORA remote autoenabled, but configuration known about at least one. # Let's check origin's config for datalad.ora-remote.uuid as stored by # create-sibling-ria and enable try enabling that one. lgr.debug("Found no autoenabled ORA special remote. Trying to look it " "up in source config ...") # First figure whether we cloned via SSH, HTTP or local path and then # get that config file the same way: config_content = None scheme = props['giturl'].split(':', 1)[0] if scheme in ['http', 'https']: try: config_content = download_url( "{}{}config".format( props['giturl'], '/' if not props['giturl'].endswith('/') else '')) except DownloadError as e: lgr.debug("Failed to get config file from source:\n%s", exc_str(e)) elif scheme == 'ssh': # TODO: switch the following to proper command abstraction: # SSHRemoteIO ignores the path part ATM. No remote CWD! (To be # changed with command abstractions). So we need to get that part to # have a valid path to origin's config file: cfg_path = PurePosixPath(URL(props['giturl']).path) / 'config' op = SSHRemoteIO(props['giturl']) try: config_content = op.read_file(cfg_path) except RIARemoteError as e: lgr.debug("Failed to get config file from source: %s", exc_str(e)) elif scheme == 'file': # TODO: switch the following to proper command abstraction: op = LocalIO() cfg_path = Path(URL(props['giturl']).localpath) / 'config' try: config_content = op.read_file(cfg_path) except (RIARemoteError, OSError) as e: lgr.debug("Failed to get config file from source: %s", exc_str(e)) else: lgr.debug("Unknown URL-Scheme %s in %s. Can handle SSH, HTTP or " "FILE scheme URLs.", scheme, props['source']) # 3. And read it org_uuid = None if config_content: # TODO: We might be able to spare the saving to a file. # "git config -f -" is not explicitly documented but happens # to work and would read from stdin. Make sure we know this # works for required git versions and on all platforms. with make_tempfile(content=config_content) as cfg_file: runner = GitWitlessRunner() try: result = runner.run( ['git', 'config', '-f', cfg_file, 'datalad.ora-remote.uuid'], protocol=StdOutCapture ) org_uuid = result['stdout'].strip() except CommandError as e: # doesn't contain what we are looking for lgr.debug("Found no UUID for ORA special remote at " "'%s' (%s)", RIA_REMOTE_NAME, exc_str(e)) # Now, enable it. If annex-init didn't fail to enable it as stored, we # wouldn't end up here, so enable with store URL as suggested by the URL # we cloned from. if org_uuid: srs = repo.get_special_remotes() if org_uuid in srs.keys(): # TODO: - Double-check autoenable value and only do this when # true? # - What if still fails? -> Annex shouldn't change config # in that case # we only need the store: new_url = props['source'].split('#')[0] try: repo.enable_remote(srs[org_uuid]['name'], options=['url={}'.format(new_url)] ) lgr.info("Reconfigured %s for %s", srs[org_uuid]['name'], new_url) # update ora_remotes for considering publication dependency # below ora_remotes = [s for s in ds.siblings('query', result_renderer='disabled') if s.get('annex-externaltype', None) == 'ora'] except CommandError as e: lgr.debug("Failed to reconfigure ORA special remote: %s", exc_str(e)) else: lgr.debug("Unknown ORA special remote uuid at '%s': %s", RIA_REMOTE_NAME, org_uuid) if ora_remotes: if len(ora_remotes) == 1: yield from ds.siblings('configure', name=RIA_REMOTE_NAME, publish_depends=ora_remotes[0]['name'], result_filter=None, result_renderer='disabled') else: lgr.warning("Found multiple ORA remotes. Couldn't decide which " "publishing to 'origin' should depend on: %s. Consider " "running 'datalad siblings configure -s origin " "--publish-depends ORAREMOTENAME' to set publication " "dependency manually.", [r['name'] for r in ora_remotes])
def test_make_tempfile(): # check if mkdir, content conflict caught with assert_raises(ValueError): with make_tempfile(content="blah", mkdir=True): # pragma: no cover pass
def setup_package(): import os from datalad.utils import on_osx from datalad.tests import _TEMP_PATHS_GENERATED if on_osx: # enforce honoring TMPDIR (see gh-5307) import tempfile tempfile.tempdir = os.environ.get('TMPDIR', tempfile.gettempdir()) from datalad import consts _test_states['env'] = {} def set_envvar(v, val): """Memoize and then set env var""" _test_states['env'][v] = os.environ.get(v, None) os.environ[v] = val _test_states['DATASETS_TOPURL'] = consts.DATASETS_TOPURL consts.DATASETS_TOPURL = 'http://datasets-tests.datalad.org/' set_envvar('DATALAD_DATASETS_TOPURL', consts.DATASETS_TOPURL) from datalad.tests.utils import ( DEFAULT_BRANCH, DEFAULT_REMOTE, ) set_envvar( "GIT_CONFIG_PARAMETERS", "'init.defaultBranch={}' 'clone.defaultRemoteName={}'".format( DEFAULT_BRANCH, DEFAULT_REMOTE)) # To overcome pybuild overriding HOME but us possibly wanting our # own HOME where we pre-setup git for testing (name, email) if 'GIT_HOME' in os.environ: set_envvar('HOME', os.environ['GIT_HOME']) set_envvar('DATALAD_LOG_EXC', "1") else: # we setup our own new HOME, the BEST and HUGE one from datalad.utils import make_tempfile # TODO: split into a function + context manager with make_tempfile(mkdir=True) as new_home: pass for v, val in get_home_envvars(new_home).items(): set_envvar(v, val) if not os.path.exists(new_home): os.makedirs(new_home) with open(os.path.join(new_home, '.gitconfig'), 'w') as f: f.write("""\ [user] name = DataLad Tester email = [email protected] [datalad "log"] exc = 1 """) _TEMP_PATHS_GENERATED.append(new_home) # Re-load ConfigManager, since otherwise it won't consider global config # from new $HOME (see gh-4153 cfg.reload(force=True) from datalad.interface.common_cfg import compute_cfg_defaults compute_cfg_defaults() # datalad.locations.sockets has likely changed. Discard any cached values. ssh_manager._socket_dir = None # To overcome pybuild by default defining http{,s}_proxy we would need # to define them to e.g. empty value so it wouldn't bother touching them. # But then haskell libraries do not digest empty value nicely, so we just # pop them out from the environment for ev in ('http_proxy', 'https_proxy'): if ev in os.environ and not (os.environ[ev]): lgr.debug("Removing %s from the environment since it is empty", ev) os.environ.pop(ev) # During tests we allow for "insecure" access to local file:// and # http://localhost URLs since all of them either generated as tests # fixtures or cloned from trusted sources from datalad.support.annexrepo import AnnexRepo AnnexRepo._ALLOW_LOCAL_URLS = True DATALAD_LOG_LEVEL = os.environ.get('DATALAD_LOG_LEVEL', None) if DATALAD_LOG_LEVEL is None: # very very silent. Tests introspecting logs should use # swallow_logs(new_level=...) _test_states['loglevel'] = lgr.getEffectiveLevel() lgr.setLevel(100) # And we should also set it within environ so underlying commands also stay silent set_envvar('DATALAD_LOG_LEVEL', '100') else: # We are not overriding them, since explicitly were asked to have some log level _test_states['loglevel'] = None # Set to non-interactive UI from datalad.ui import ui _test_states['ui_backend'] = ui.backend # obtain() since that one consults for the default value ui.set_backend(cfg.obtain('datalad.tests.ui.backend')) # Monkey patch nose so it does not ERROR out whenever code asks for fileno # of the output. See https://github.com/nose-devs/nose/issues/6 from io import StringIO as OrigStringIO class StringIO(OrigStringIO): fileno = lambda self: 1 encoding = None from nose.ext import dtcompat from nose.plugins import capture, multiprocess, plugintest dtcompat.StringIO = StringIO capture.StringIO = StringIO multiprocess.StringIO = StringIO plugintest.StringIO = StringIO # in order to avoid having to fiddle with rather uncommon # file:// URLs in the tests, have a standard HTTP server # that serves an 'httpserve' directory in the test HOME # the URL will be available from datalad.test_http_server.url from datalad.tests.utils import HTTPPath import tempfile global test_http_server serve_path = tempfile.mkdtemp( dir=cfg.get("datalad.tests.temp.dir"), prefix='httpserve', ) test_http_server = HTTPPath(serve_path) test_http_server.start() _TEMP_PATHS_GENERATED.append(serve_path) if cfg.obtain('datalad.tests.setup.testrepos'): lgr.debug("Pre-populating testrepos") from datalad.tests.utils import with_testrepos with_testrepos()(lambda repo: 1)()
def setup_package(): import os from datalad import consts _test_states['HOME'] = os.environ.get('HOME', None) _test_states['DATASETS_TOPURL_ENV'] = os.environ.get('DATALAD_DATASETS_TOPURL', None) _test_states['DATASETS_TOPURL'] = consts.DATASETS_TOPURL os.environ['DATALAD_DATASETS_TOPURL'] = consts.DATASETS_TOPURL = 'http://datasets-tests.datalad.org/' # To overcome pybuild overriding HOME but us possibly wanting our # own HOME where we pre-setup git for testing (name, email) if 'GIT_HOME' in os.environ: os.environ['HOME'] = os.environ['GIT_HOME'] else: # we setup our own new HOME, the BEST and HUGE one from datalad.utils import make_tempfile from datalad.tests import _TEMP_PATHS_GENERATED # TODO: split into a function + context manager with make_tempfile(mkdir=True) as new_home: os.environ['HOME'] = new_home if not os.path.exists(new_home): os.makedirs(new_home) with open(os.path.join(new_home, '.gitconfig'), 'w') as f: f.write("""\ [user] name = DataLad Tester email = [email protected] """) _TEMP_PATHS_GENERATED.append(new_home) # If there is a bundled git, make sure GitPython uses it too # (some parts of the test utilities still rely on GitPython) from datalad.cmd import GitRunner GitRunner._check_git_path() if GitRunner._GIT_PATH: import os os.environ['GIT_PYTHON_GIT_EXECUTABLE'] = \ os.path.join(GitRunner._GIT_PATH, 'git') # Re-load ConfigManager, since otherwise it won't consider global config # from new $HOME (see gh-4153 cfg.reload(force=True) # To overcome pybuild by default defining http{,s}_proxy we would need # to define them to e.g. empty value so it wouldn't bother touching them. # But then haskell libraries do not digest empty value nicely, so we just # pop them out from the environment for ev in ('http_proxy', 'https_proxy'): if ev in os.environ and not (os.environ[ev]): lgr.debug("Removing %s from the environment since it is empty", ev) os.environ.pop(ev) # During tests we allow for "insecure" access to local file:// and # http://localhost URLs since all of them either generated as tests # fixtures or cloned from trusted sources from datalad.support.annexrepo import AnnexRepo AnnexRepo._ALLOW_LOCAL_URLS = True DATALAD_LOG_LEVEL = os.environ.get('DATALAD_LOG_LEVEL', None) if DATALAD_LOG_LEVEL is None: # very very silent. Tests introspecting logs should use # swallow_logs(new_level=...) _test_states['loglevel'] = lgr.getEffectiveLevel() lgr.setLevel(100) # And we should also set it within environ so underlying commands also stay silent _test_states['DATALAD_LOG_LEVEL'] = DATALAD_LOG_LEVEL os.environ['DATALAD_LOG_LEVEL'] = '100' else: # We are not overriding them, since explicitly were asked to have some log level _test_states['loglevel'] = None # Set to non-interactive UI from datalad.ui import ui _test_states['ui_backend'] = ui.backend # obtain() since that one consults for the default value ui.set_backend(cfg.obtain('datalad.tests.ui.backend')) # Monkey patch nose so it does not ERROR out whenever code asks for fileno # of the output. See https://github.com/nose-devs/nose/issues/6 from io import StringIO as OrigStringIO class StringIO(OrigStringIO): fileno = lambda self: 1 encoding = None from nose.ext import dtcompat from nose.plugins import capture, multiprocess, plugintest dtcompat.StringIO = StringIO capture.StringIO = StringIO multiprocess.StringIO = StringIO plugintest.StringIO = StringIO
def setup_package(): import os from datalad import consts _test_states['HOME'] = os.environ.get('HOME', None) _test_states['DATASETS_TOPURL_ENV'] = os.environ.get('DATALAD_DATASETS_TOPURL', None) _test_states['DATASETS_TOPURL'] = consts.DATASETS_TOPURL os.environ['DATALAD_DATASETS_TOPURL'] = consts.DATASETS_TOPURL = 'http://datasets-tests.datalad.org/' # To overcome pybuild overriding HOME but us possibly wanting our # own HOME where we pre-setup git for testing (name, email) if 'GIT_HOME' in os.environ: os.environ['HOME'] = os.environ['GIT_HOME'] else: # we setup our own new HOME, the BEST and HUGE one from datalad.utils import make_tempfile from datalad.tests import _TEMP_PATHS_GENERATED # TODO: split into a function + context manager with make_tempfile(mkdir=True) as new_home: os.environ['HOME'] = new_home if not os.path.exists(new_home): os.makedirs(new_home) with open(os.path.join(new_home, '.gitconfig'), 'w') as f: f.write("""\ [user] name = DataLad Tester email = [email protected] """) _TEMP_PATHS_GENERATED.append(new_home) # For now we will just verify that it is ready to run the tests from datalad.support.gitrepo import check_git_configured check_git_configured() # To overcome pybuild by default defining http{,s}_proxy we would need # to define them to e.g. empty value so it wouldn't bother touching them. # But then haskell libraries do not digest empty value nicely, so we just # pop them out from the environment for ev in ('http_proxy', 'https_proxy'): if ev in os.environ and not (os.environ[ev]): lgr.debug("Removing %s from the environment since it is empty", ev) os.environ.pop(ev) # During tests we allow for "insecure" access to local file:// and # http://localhost URLs since all of them either generated as tests # fixtures or cloned from trusted sources from datalad.support.annexrepo import AnnexRepo AnnexRepo._ALLOW_LOCAL_URLS = True DATALAD_LOG_LEVEL = os.environ.get('DATALAD_LOG_LEVEL', None) if DATALAD_LOG_LEVEL is None: # very very silent. Tests introspecting logs should use # swallow_logs(new_level=...) _test_states['loglevel'] = lgr.getEffectiveLevel() lgr.setLevel(100) # And we should also set it within environ so underlying commands also stay silent _test_states['DATALAD_LOG_LEVEL'] = DATALAD_LOG_LEVEL os.environ['DATALAD_LOG_LEVEL'] = '100' else: # We are not overriding them, since explicitly were asked to have some log level _test_states['loglevel'] = None # Set to non-interactive UI from datalad.ui import ui _test_states['ui_backend'] = ui.backend # obtain() since that one consults for the default value ui.set_backend(cfg.obtain('datalad.tests.ui.backend'))