def check_run_and_get_output(cmd): runner = Runner() try: output = runner.run(["datalad", "--help"]) except CommandError, e: raise AssertionError("'datalad --help' failed to start normally. " "Exited with %d and output %s" % (e.code, (e.stdout, e.stderr)))
def check_runner_heavy_output(log_online): # TODO: again, no automatic detection of this resulting in being stucked yet. runner = Runner() cmd = '%s -c "import datalad.tests.heavyoutput;"' % sys.executable with swallow_outputs() as cm: ret = runner.run(cmd, log_stderr=False, log_stdout=False, expect_stderr=True) eq_(cm.err, cm.out) # they are identical in that script eq_(cm.out[:10], "[0, 1, 2, ") eq_(cm.out[-15:], "997, 998, 999]\n") #do it again with capturing: ret = runner.run(cmd, log_stderr=True, log_stdout=True, expect_stderr=True) # and now original problematic command with a massive single line if not log_online: # We know it would get stuck in online mode cmd = '%s -c "import sys; x=str(list(range(1000))); [(sys.stdout.write(x), sys.stderr.write(x)) for i in xrange(100)];"' % sys.executable ret = runner.run(cmd, log_stderr=True, log_stdout=True, expect_stderr=True)
def test_runner_log_stdout(): # TODO: no idea of how to check correct logging via any kind of assertion yet. runner = Runner(dry=False) cmd_ = ['echo', 'stdout-Message should be logged'] for cmd in [cmd_, ' '.join(cmd_)]: # should be identical runs, either as a string or as a list kw = {} # on Windows it can't find echo if ran outside the shell if on_windows and isinstance(cmd, list): kw['shell'] = True with swallow_logs(logging.DEBUG) as cm: ret = runner.run(cmd, log_stdout=True, **kw) eq_(cm.lines[0], "Running: %s" % cmd) if not on_windows: # we can just count on sanity eq_(cm.lines[1], "stdout| stdout-Message should be logged") else: # echo outputs quoted lines for some reason, so relax check ok_("stdout-Message should be logged" in cm.lines[1]) assert_equal(runner.commands, [], "Run of: %s resulted in non-empty buffer: %s" % (cmd, runner.commands.__str__())) cmd = 'echo stdout-Message should not be logged' with swallow_outputs() as cmo: with swallow_logs(new_level=logging.INFO) as cml: ret = runner.run(cmd, log_stdout=False) eq_(cmo.out, "stdout-Message should not be logged\n") eq_(cml.out, "") assert_equal(runner.commands, [], "Run of: %s resulted in non-empty buffer: %s" % (cmd, runner.commands.__str__()))
def compress_files(files, archive, path=None, overwrite=True): """Compress `files` into an `archive` file Parameters ---------- files : list of str archive : str path : str Alternative directory under which compressor will be invoked, to e.g. take into account relative paths of files and/or archive overwrite : bool Whether to allow overwriting the target archive file if one already exists """ runner = Runner(cwd=path) apath = Path(archive) if apath.exists(): if overwrite: apath.unlink() else: raise ValueError( 'Target archive {} already exists and overwrite is forbidden'. format(apath)) if len(apath.suffixes) > 1 and apath.suffixes[-2] == '.tar': cmd = '7z u .tar -so -- {} | 7z u -si -- {}'.format( ' '.join(quote_cmdlinearg(f) for f in files), quote_cmdlinearg(str(apath)), ) else: cmd = ['7z', 'u', str(apath), '--'] + files runner.run(cmd)
def test_runner_log_stdout(): # TODO: no idea of how to check correct logging via any kind of assertion yet. runner = Runner(dry=False) cmd_ = ['echo', 'stdout-Message should be logged'] for cmd in [cmd_, ' '.join(cmd_)]: # should be identical runs, either as a string or as a list kw = {} # on Windows it can't find echo if ran outside the shell if on_windows and isinstance(cmd, list): kw['shell'] = True with swallow_logs(logging.DEBUG) as cm: ret = runner.run(cmd, log_stdout=True, **kw) eq_(cm.lines[0], "Running: %s" % cmd) if not on_windows: # we can just count on sanity eq_(cm.lines[1], "stdout| stdout-Message should be logged") else: # echo outputs quoted lines for some reason, so relax check ok_("stdout-Message should be logged" in cm.lines[1]) assert_equal( runner.commands, [], "Run of: %s resulted in non-empty buffer: %s" % (cmd, runner.commands.__str__())) cmd = 'echo stdout-Message should not be logged' with swallow_outputs() as cmo: with swallow_logs(new_level=logging.INFO) as cml: ret = runner.run(cmd, log_stdout=False) eq_(cmo.out, "stdout-Message should not be logged\n") eq_(cml.out, "") assert_equal( runner.commands, [], "Run of: %s resulted in non-empty buffer: %s" % (cmd, runner.commands.__str__()))
def get_singularity_jobspec(cmd): """Extract the runscript of a singularity container used as an executable Parameters ---------- cmd : list A command as an argument list. Returns ------- None or str, None or list If no singularity is available, or the executable in the command is not a singularity image given by its path, None is return. Otherwise the runscript of the container is returned a string. The second value is None if the first is None, or a list of arguments to the runscript. """ # get the path to the command's executable exec_path = cmd[0] runner = Runner() if not op.exists(exec_path): # probably a command from PATH return # this is a real file, not just a command on the path try: stdout, stderr = runner.run( ['singularity', '--version'], log_stdout=True, log_stderr=True, expect_stderr=True, expect_fail=True, ) # TODO could be used to tailor handling to particular versions except CommandError as e: # pragma: no cover # we do not have a singularity installation that we can handle # log debug, because there is no guarantee that the executable # actually was a singularity container lgr.debug('No suitable singularity version installed: %s', exc_str(e)) return # we have singularity try: stdout, stderr = runner.run( # stringification only needed for pythons older than 3.6 ['singularity', 'exec', exec_path, 'cat', '/singularity'], log_stdout=True, log_stderr=True, expect_stderr=True, expect_fail=True, ) # TODO could be used to tailor handling to particular versions except CommandError as e: # we do not have a singularity installation that we can handle # log debug, because there is no guarantee that the executable # actually was a singularity container lgr.debug('%s is not a singularity image: %s', exec_path, exc_str(e)) return # all but the container itself are the arguments return exec_path, cmd[1:]
def setup(self): self.runner = Runner() # older versions might not have it try: from datalad.cmd import GitRunner self.git_runner = GitRunner() except ImportError: pass
def create_info_file(self): runner = Runner() git_version = runner.run("git --version")[0].split()[2] self.create_file('INFO.txt', "git: %s\n" "datalad: %s\n" % (git_version, __version__), annex=False)
def test_cfg_passthrough(path): runner = Runner() _ = runner.run([ 'datalad', '-c', 'annex.tune.objecthash1=true', '-c', 'annex.tune.objecthashlower=true', 'create', path ]) ds = Dataset(path) eq_(ds.config.get('annex.tune.objecthash1', None), 'true') eq_(ds.config.get('annex.tune.objecthashlower', None), 'true')
def test_no_stdin_swallow(fname): # will relay actual exit code on CommandError cmd = ['datalad', 'sshrun', 'localhost', 'cat'] out, err = Runner().run(cmd, stdin=open(fname)) assert_equal(out.rstrip(), '123magic') # test with -n switch now, which we could place even at the end out, err = Runner().run(cmd + ['-n'], stdin=open(fname)) assert_equal(out, '')
def test_cfg_passthrough(path): runner = Runner() _ = runner.run( ['datalad', '-c', 'annex.tune.objecthash1=true', '-c', 'annex.tune.objecthashlower=true', 'create', path]) ds = Dataset(path) eq_(ds.config.get('annex.tune.objecthash1', None), 'true') eq_(ds.config.get('annex.tune.objecthashlower', None), 'true')
def check_run_and_get_output(cmd): runner = Runner() try: # suppress log output happen it was set to high values with patch.dict('os.environ', {'DATALAD_LOG_LEVEL': 'WARN'}): output = runner.run(["datalad", "--help"]) except CommandError as e: raise AssertionError("'datalad --help' failed to start normally. " "Exited with %d and output %s" % (e.code, (e.stdout, e.stderr))) return output
def test_runner_failure(dir): runner = Runner() failing_cmd = ['git', 'annex', 'add', 'notexistent.dat'] assert_raises(CommandError, runner.run, failing_cmd, cwd=dir) try: runner.run(failing_cmd, cwd=dir) except CommandError, e: assert_equal(1, e.code) assert_in('notexistent.dat not found', e.stderr)
def check_run_and_get_output(cmd): runner = Runner() try: # suppress log output happen it was set to high values with patch.dict('os.environ', {'DATALAD_LOGLEVEL': 'WARN'}): output = runner.run(["datalad", "--help"]) except CommandError as e: raise AssertionError("'datalad --help' failed to start normally. " "Exited with %d and output %s" % (e.code, (e.stdout, e.stderr))) return output
def test_runner_log_stderr(): # TODO: no idea of how to check correct logging via any kind of assertion yet. runner = Runner(dry=False) cmd = 'echo stderr-Message should be logged >&2' ret = runner.run(cmd, log_stderr=True, expect_stderr=True) assert_equal(runner.commands, [], "Run of: %s resulted in non-empty buffer: %s" % (cmd, runner.commands.__str__())) cmd = 'echo stderr-Message should not be logged >&2' with swallow_outputs() as cmo: with swallow_logs(new_level=logging.INFO) as cml: ret = runner.run(cmd, log_stderr=False) eq_(cmo.err.rstrip(), "stderr-Message should not be logged") eq_(cml.out, "") assert_equal(runner.commands, [], "Run of: %s resulted in non-empty buffer: %s" % (cmd, runner.commands.__str__()))
def test_runner(tempfile): # test non-dry command call runner = Runner(dry=False) cmd = 'echo Testing real run > %s' % tempfile ret = runner.run(cmd) assert_equal(runner.commands, [], "Run of: %s resulted in non-empty buffer: %s" % (cmd, runner.commands.__str__())) assert_true(os.path.exists(tempfile), "Run of: %s resulted with non-existing file %s" % (cmd, tempfile)) # test non-dry python function call output = runner.call(os.path.join, 'foo', 'bar') assert_equal(os.path.join('foo', 'bar'), output, "Drycall of: os.path.join, 'foo', 'bar' returned %s" % output) assert_equal(runner.commands.__str__().find('os.path.join'), -1, "Drycall of: os.path.join, 'foo', 'bar' resulted in buffer: %s" % runner.commands.__str__())
def test_runner_instance_callable_dry(): cmd_ = ['echo', 'Testing __call__ with string'] for cmd in [cmd_, ' '.join(cmd_)]: runner = Runner(dry=True) ret = runner(cmd) eq_(ret, ("DRY", "DRY" )) # (stdout, stderr) is returned. But in dry -- ("DRY","DRY") assert_equal( runner.commands.__str__(), ("[%r]" % cmd), "Dry run of Runner.__call__ didn't record command: %s.\n" "Buffer: %s" % (cmd, runner.commands.__str__())) ret = runner(os.path.join, 'foo', 'bar') eq_(ret, None) assert_greater( runner.commands.__str__().find('join'), -1, "Dry run of Runner.__call__ didn't record function join()." "Buffer: %s" % runner.commands.__str__()) assert_greater( runner.commands.__str__().find('args='), -1, "Dry run of Runner.__call__ didn't record function join()." "Buffer: %s" % runner.commands.__str__())
def test_duecredit(path): # Just to check that no obvious side-effects run = Runner(cwd=path).run cmd = [ sys.executable, "-c", "from datalad.support.gitrepo import GitRepo; GitRepo(%r, create=True)" % path ] env = os.environ.copy() # Test with duecredit not enabled for sure env.pop('DUECREDIT_ENABLE', None) # Alternative workaround for what to be fixed by # https://github.com/datalad/datalad/pull/3215 # where underlying datalad process might issue a warning since our temp # cwd is not matching possibly present PWD env variable env.pop('PWD', None) out, err = run(cmd, env=env, expect_stderr=True) outs = out + err # Let's not depend on where duecredit decides to spit out # All quiet eq_(outs, '') # and now enable DUECREDIT - output could come to stderr env['DUECREDIT_ENABLE'] = '1' out, err = run(cmd, env=env, expect_stderr=True) outs = out + err if external_versions['duecredit']: assert_in('Data management and distribution platform', outs) else: eq_(outs, '')
def test_script_shims(): runner = Runner() for script in [ 'datalad', 'git-annex-remote-datalad-archives', 'git-annex-remote-datalad']: if not on_windows: # those must be available for execution, and should not contain which, _ = runner(['which', script]) # test if there is no easy install shim in there with open(which.rstrip()) as f: content = f.read() else: from distutils.spawn import find_executable content = find_executable(script) assert_not_in('EASY', content) # NOTHING easy should be there assert_not_in('pkg_resources', content) # and let's check that it is our script out, err = runner([script, '--version']) version = (out + err).splitlines()[0].split(' ', 1)[1] # we can get git and non git .dev version... so for now # relax get_numeric_portion = lambda v: [x for x in v.split('.') if x.isdigit()] # extract numeric portion assert get_numeric_portion(version) # that my lambda is correctish assert_equal(get_numeric_portion(__version__), get_numeric_portion(version))
def test_our_metadataset_search(tdir): # smoke test for basic search operations on our super-megadataset # expensive operation but ok ds = install(path=tdir, source=DATASETS_TOPURL, result_xfm='datasets', return_type='item-or-list') res_haxby = list(ds.search('haxby')) assert_greater(len(res_haxby), 10) # default search should be case insensitive # but somehow it is not fully -- we get 12 here #res_Haxby = list(ds.search('Haxby')) #eq_(len(res_haxby), len(res_Haxby)) assert_result_count(ds.search('id:873a6eae-7ae6-11e6-a6c8-002590f97d84', mode='textblob'), 1, type='dataset', path=op.join(ds.path, 'crcns', 'pfc-2')) # there is a problem with argparse not decoding into utf8 in PY2 from datalad.cmdline.tests.test_main import run_main # TODO: make it into an independent lean test from datalad.cmd import Runner out, err = Runner(cwd=ds.path)('datalad search Buzsáki') assert_in('crcns/pfc-2 ', out) # has it in description # and then another aspect: this entry it among multiple authors, need to # check if aggregating them into a searchable entity was done correctly assert_in('crcns/hc-1 ', out)
def test_our_metadataset_search(tdir): # TODO renable when a dataset with new aggregated metadata is # available at some public location raise SkipTest # smoke test for basic search operations on our super-megadataset # expensive operation but ok #ds = install( # path=tdir, # # TODO renable test when /// metadata actually conforms to the new metadata # #source="///", # source="smaug:/mnt/btrfs/datasets-meta6-4/datalad/crawl", # result_xfm='datasets', return_type='item-or-list') assert list(ds.search('haxby')) assert_result_count(ds.search('id:873a6eae-7ae6-11e6-a6c8-002590f97d84', mode='textblob'), 1, type='dataset', path=opj(ds.path, 'crcns', 'pfc-2')) # there is a problem with argparse not decoding into utf8 in PY2 from datalad.cmdline.tests.test_main import run_main # TODO: make it into an independent lean test from datalad.cmd import Runner out, err = Runner(cwd=ds.path)('datalad search Buzsáki') assert_in('crcns/pfc-2 ', out) # has it in description # and then another aspect: this entry it among multiple authors, need to # check if aggregating them into a searchable entity was done correctly assert_in('crcns/hc-1 ', out)
def test_runner_failure(dir_): runner = Runner() with assert_raises(CommandError) as cme: runner.run( py2cmd('import sys; sys.exit(53)') ) eq_(53, cme.exception.code)
def populate(self): super(SubmoduleDataset, self).populate() # add submodules annex = BasicAnnexTestRepo() annex.create() from datalad.cmd import Runner runner = Runner() kw = dict(cwd=self.path, expect_stderr=True) runner.run(['git', 'submodule', 'add', annex.url, 'sub1'], **kw) runner.run(['git', 'submodule', 'add', annex.url, 'sub2'], **kw) runner.run(['git', 'commit', '-m', 'Added sub1 and sub2.'], **kw) runner.run(['git', 'submodule', 'update', '--init', '--recursive'], **kw) # init annex in subdatasets for s in ('sub1', 'sub2'): runner.run(['git', 'annex', 'init'], cwd=opj(self.path, s), expect_stderr=True)
def __init__(self, dataset=None, dataset_only=False): # store in a simple dict # no subclassing, because we want to be largely read-only, and implement # config writing separately self._store = {} self._dataset = dataset self._dataset_only = dataset_only # Since configs could contain sensitive information, to prevent # any "facilitated" leakage -- just disable loging of outputs for # this runner run_kwargs = dict(log_outputs=False) if dataset is not None: # make sure we run the git config calls in the dataset # to pick up the right config files run_kwargs['cwd'] = dataset.path self._runner = Runner(**run_kwargs) self.reload()
def test_runner_dry(tempfile): runner = Runner(dry=True) # test dry command call cmd = 'echo Testing dry run > %s' % tempfile ret = runner.run(cmd) assert_equal(("DRY", "DRY"), ret, "Dry run of: %s resulted in output %s" % (cmd, ret)) assert_greater(runner.commands.__str__().find('echo Testing dry run'), -1, "Dry run of: %s resulted in buffer: %s" % (cmd, runner.commands.__str__())) assert_false(os.path.exists(tempfile)) # test dry python function call output = runner.call(os.path.join, 'foo', 'bar') assert_is(None, output, "Drycall of: os.path.join, 'foo', 'bar' returned %s" % output) assert_greater(runner.commands.__str__().find('join'), -1, "Drycall of: os.path.join, 'foo', 'bar' resulted in buffer: %s" % runner.commands.__str__())
def _move2store(storepath, d): # make a bare clone of it into a local that matches the organization # of a ria dataset store store_loc = str(storepath / d.id[:3] / d.id[3:]) d.repo.call_git(['clone', '--bare', d.path, store_loc]) d.siblings('configure', name='store', url=str(store_loc), result_renderer='disabled') Runner(cwd=store_loc).run(['git', 'update-server-info'])
class runner(SuprocBenchmarks): """Some rudimentary tests to see if there is no major slowdowns from Runner """ def setup(self): from datalad.cmd import Runner self.runner = Runner() # older versions might not have it try: from datalad.cmd import GitRunner self.git_runner = GitRunner() except ImportError: pass def time_echo(self): self.runner.run("echo") def time_echo_gitrunner(self): self.git_runner.run("echo")
def test_runner_stdout_capture(): runner = Runner() test_msg = "stdout-Message" res = runner.run( py2cmd('import sys; print(%r, file=sys.stdout)' % test_msg), protocol=StdOutErrCapture, ) eq_(res['stdout'].rstrip(), test_msg) ok_(not res['stderr'])
def test_create(path): ds = Dataset(path) ds.create( description="funny", # custom git init option initopts=dict(shared='world')) ok_(ds.is_installed()) assert_repo_status(ds.path, annex=True) # check default backend eq_(ds.config.get("annex.backends"), 'MD5E') eq_(ds.config.get("core.sharedrepository"), '2') runner = Runner() # check description in `info` cmd = ['git', 'annex', 'info'] cmlout = runner.run(cmd, cwd=path) assert_in('funny [here]', cmlout[0]) # check datset ID eq_(ds.config.get_value('datalad.dataset', 'id'), ds.id)
class RunnerSuite(SuprocBenchmarks): """Some rudimentary tests to see if there is no major slowdowns from Runner """ def setup(self): from datalad.cmd import Runner self.runner = Runner() # older versions might not have it try: from datalad.cmd import GitRunner self.git_runner = GitRunner() except ImportError: pass def time_echo(self): self.runner.run("echo") def time_echo_gitrunner(self): self.git_runner.run("echo")
def test_runner_fix_PWD(path): env = os.environ.copy() env['PWD'] = orig_cwd = os.getcwd() runner = Runner(cwd=path, env=env) res = runner.run( py2cmd('import os; print(os.environ["PWD"])'), protocol=StdOutCapture, ) eq_(res['stdout'].strip(), path) # was fixed up to point to point to cwd's path eq_(env['PWD'], orig_cwd) # no side-effect
def test_runner_log_stderr(): # TODO: no idea of how to check correct logging via any kind of assertion yet. runner = Runner(dry=False) cmd = 'echo stderr-Message should be logged >&2' ret = runner.run(cmd, log_stderr=True, expect_stderr=True) assert_equal( runner.commands, [], "Run of: %s resulted in non-empty buffer: %s" % (cmd, runner.commands.__str__())) cmd = 'echo stderr-Message should not be logged >&2' with swallow_outputs() as cmo: with swallow_logs(new_level=logging.INFO) as cml: ret = runner.run(cmd, log_stderr=False) eq_(cmo.err.rstrip(), "stderr-Message should not be logged") eq_(cml.out, "") assert_equal( runner.commands, [], "Run of: %s resulted in non-empty buffer: %s" % (cmd, runner.commands.__str__()))
def test_runner(tempfile): runner = Runner() content = 'Testing real run' if on_windows else 'Testing äöü東 real run' cmd = 'echo %s > %s' % (content, tempfile) res = runner.run(cmd) # no capture of any kind, by default ok_(not res['stdout']) ok_(not res['stderr']) ok_file_has_content(tempfile, content, strip=True) os.unlink(tempfile)
def test_replace_and_relative_sshpath(src_path, dst_path): # We need to come up with the path relative to our current home directory # https://github.com/datalad/datalad/issues/1653 # but because we override HOME the HOME on the remote end would be # different even though a localhost. So we need to query it from datalad import ssh_manager ssh = ssh_manager.get_connection('localhost') remote_home, err = ssh('pwd') assert not err remote_home = remote_home.rstrip('\n') dst_relpath = os.path.relpath(dst_path, remote_home) url = 'localhost:%s' % dst_relpath ds = Dataset(src_path).create() create_tree(ds.path, {'sub.dat': 'lots of data'}) ds.add('sub.dat') ds.create_sibling(url, ui=True) published = ds.publish(to='localhost', transfer_data='all') assert_result_count(published, 1, path=opj(ds.path, 'sub.dat')) # verify that hook runs and there is nothing in stderr # since it exits with 0 exit even if there was a problem out, err = Runner(cwd=opj(dst_path, '.git'))(_path_('hooks/post-update')) assert_false(out) assert_false(err) # Verify that we could replace and publish no problem # https://github.com/datalad/datalad/issues/1656 # Strangely it spits outs IncompleteResultsError exception atm... so just # checking that it fails somehow res = ds.create_sibling(url, on_failure='ignore') assert_status('error', res) assert_in('already configured', res[0]['message'][0]) # "Settings" such as UI do not persist, so we specify it again # for the test below depending on it ds.create_sibling(url, existing='replace', ui=True) published2 = ds.publish(to='localhost', transfer_data='all') assert_result_count(published2, 1, path=opj(ds.path, 'sub.dat')) # and one more test since in above test it would not puke ATM but just # not even try to copy since it assumes that file is already there create_tree(ds.path, {'sub2.dat': 'more data'}) ds.add('sub2.dat') published3 = ds.publish(to='localhost', transfer_data='none') # we publish just git assert_result_count(published3, 0, path=opj(ds.path, 'sub2.dat')) # now publish "with" data, which should also trigger the hook! # https://github.com/datalad/datalad/issues/1658 from glob import glob from datalad.consts import WEB_META_LOG logs_prior = glob(_path_(dst_path, WEB_META_LOG, '*')) published4 = ds.publish(to='localhost', transfer_data='all') assert_result_count(published4, 1, path=opj(ds.path, 'sub2.dat')) logs_post = glob(_path_(dst_path, WEB_META_LOG, '*')) eq_(len(logs_post), len(logs_prior) + 1) assert_postupdate_hooks(dst_path)
def test_runner_instance_callable_wet(): runner = Runner() cmd = [sys.executable, "-c", "print('Testing')"] out = runner(cmd) eq_(out[0].rstrip(), ('Testing')) eq_(out[1], '') ret = runner(os.path.join, 'foo', 'bar') eq_(ret, os.path.join('foo', 'bar'))
def test_runner_cwd_encoding(path): env = os.environ.copy() # Add PWD to env so that runner will temporarily adjust it to point to cwd. env['PWD'] = os.getcwd() cwd = Path(path) / OBSCURE_FILENAME cwd.mkdir() # Running doesn't fail if cwd or env has unicode value. Runner(cwd=cwd, env=env).run( py2cmd( 'from pathlib import Path; (Path.cwd() / "foo").write_text("t")')) (cwd / 'foo').exists()
def test_create(path): ds = Dataset(path) ds.create(description="funny", native_metadata_type=['bim', 'bam', 'bum'], shared_access='world') ok_(ds.is_installed()) ok_clean_git(ds.path, annex=True) # check default backend eq_(ds.config.get("annex.backends"), 'MD5E') eq_(ds.config.get("core.sharedrepository"), '2') runner = Runner() # check description in `info` cmd = ['git-annex', 'info'] cmlout = runner.run(cmd, cwd=path) assert_in('funny [here]', cmlout[0]) # check datset ID eq_(ds.config.get_value('datalad.dataset', 'id'), ds.id) assert_equal(ds.config.get_value('datalad.metadata', 'nativetype'), ('bim', 'bam', 'bum'))
def _execute_command(command, pwd, expected_exit=None): from datalad.cmd import Runner exc = None cmd_exitcode = None runner = Runner(cwd=pwd) try: lgr.info("== Command start (output follows) =====") runner.run( command, # immediate output log_online=True, # not yet sure what we should do with the command output # IMHO `run` itself should be very silent and let the command talk log_stdout=False, log_stderr=False, expect_stderr=True, expect_fail=True, # TODO stdin ) except CommandError as e: # strip our own info from the exception. The original command output # went to stdout/err -- we just have to exitcode in the same way exc = e cmd_exitcode = e.code if expected_exit is not None and expected_exit != cmd_exitcode: # we failed in a different way during a rerun. This can easily # happen if we try to alter a locked file # # TODO add the ability to `git reset --hard` the dataset tree on failure # we know that we started clean, so we could easily go back, needs gh-1424 # to be able to do it recursively raise exc lgr.info("== Command exit (modification check follows) =====") return cmd_exitcode or 0, exc
def __init__(self, dataset=None, dataset_only=False): # store in a simple dict # no subclassing, because we want to be largely read-only, and implement # config writing separately self._store = {} self._dataset_path = dataset.path if dataset else None self._dataset_only = dataset_only # Since configs could contain sensitive information, to prevent # any "facilitated" leakage -- just disable logging of outputs for # this runner run_kwargs = dict(log_outputs=False) if dataset is not None: # make sure we run the git config calls in the dataset # to pick up the right config files run_kwargs['cwd'] = dataset.path self._runner = Runner(**run_kwargs) self.reload()
from datalad.cmd import Runner from datalad.support.protocol import DryRunProtocol import logging import time import json # Original SATA interface ones # DRIVES_PREFIX = "ata-ST4000NM0033-9ZM170_S1Z0" # SAS 6TB DRIVES_PREFIX = "scsi-35000c50084" lgr = logging.getLogger("datalad.benchmarking") run = Runner() dryrun = Runner() def get_drives(prefix, even=True): """Return letters for corresponding drives""" drives = [d for d in glob.glob('/dev/disk/by-id/%s*' % prefix) if not '-part' in d] drives = sorted([ os.path.realpath(d)[-3:] for d in drives ]) for d in drives: assert(d.startswith('sd')) if even and len(drives) > 1: drives = drives[:len(drives)//2*2] assert(len(drives) == 1 or len(drives) % 2 == 0) return drives
def __call__(sshurl, target=None, target_dir=None, target_url=None, target_pushurl=None, dataset=None, recursive=False, existing='raise', shared=False): if sshurl is None: raise ValueError("""insufficient information for target creation (needs at least a dataset and a SSH URL).""") if target is None and (target_url is not None or target_pushurl is not None): raise ValueError("""insufficient information for adding the target as a sibling (needs at least a name)""") # shortcut ds = dataset if ds is not None and not isinstance(ds, Dataset): ds = Dataset(ds) if ds is None: # try to find a dataset at or above CWD dspath = GitRepo.get_toppath(abspath(getpwd())) if dspath is None: raise ValueError("""No dataset found at or above {0}.""".format(getpwd())) ds = Dataset(dspath) lgr.debug("Resolved dataset for target creation: {0}".format(ds)) assert(ds is not None and sshurl is not None) if not ds.is_installed(): raise ValueError("""Dataset {0} is not installed yet.""".format(ds)) assert(ds.repo is not None) # determine target parameters: parsed_target = urlparse(sshurl) host_name = parsed_target.netloc # TODO: Sufficient to fail on this condition? if not parsed_target.netloc: raise ValueError("Malformed URL: {0}".format(sshurl)) if target_dir is None: if parsed_target.path: target_dir = parsed_target.path else: target_dir = '.' # TODO: centralize and generalize template symbol handling replicate_local_structure = False if "%NAME" not in target_dir: replicate_local_structure = True # collect datasets to use: datasets = dict() datasets[basename(ds.path)] = ds if recursive: for subds in ds.get_dataset_handles(recursive=True): sub_path = opj(ds.path, subds) # TODO: when enhancing Dataset/*Repo classes and therefore # adapt to moved code, make proper distinction between name and # path of a submodule, which are technically different. This # probably will become important on windows as well as whenever # we want to allow for moved worktrees. datasets[basename(ds.path) + '/' + subds] = \ Dataset(sub_path) # setup SSH Connection: # TODO: Make the entire setup a helper to use it when pushing via # publish? # - build control master: from datalad.utils import assure_dir not_supported_on_windows("TODO") from os import geteuid # Linux specific import var_run_user_datalad = "/var/run/user/%s/datalad" % geteuid() assure_dir(var_run_user_datalad) control_path = "%s/%s" % (var_run_user_datalad, host_name) control_path += ":%s" % parsed_target.port if parsed_target.port else "" # - start control master: cmd = "ssh -o ControlMaster=yes -o \"ControlPath=%s\" " \ "-o ControlPersist=yes %s exit" % (control_path, host_name) lgr.debug("Try starting control master by calling:\n%s" % cmd) import subprocess proc = subprocess.Popen(cmd, shell=True) proc.communicate(input="\n") # why the f.. this is necessary? runner = Runner() ssh_cmd = ["ssh", "-S", control_path, host_name] lgr.info("Creating target datasets ...") for current_dataset in datasets: if not replicate_local_structure: path = target_dir.replace("%NAME", current_dataset.replace("/", "-")) else: # TODO: opj depends on local platform, not the remote one. # check how to deal with it. Does windows ssh server accept # posix paths? vice versa? Should planned SSH class provide # tools for this issue? path = normpath(opj(target_dir, relpath(datasets[current_dataset].path, start=ds.path))) if path != '.': # check if target exists # TODO: Is this condition valid for != '.' only? path_exists = True cmd = ssh_cmd + ["ls", path] try: out, err = runner.run(cmd, expect_fail=True, expect_stderr=True) except CommandError as e: if "No such file or directory" in e.stderr and \ path in e.stderr: path_exists = False else: raise # It's an unexpected failure here if path_exists: if existing == 'raise': raise RuntimeError( "Target directory %s already exists." % path) elif existing == 'skip': continue elif existing == 'replace': pass else: raise ValueError("Do not know how to hand existing=%s" % repr(existing)) cmd = ssh_cmd + ["mkdir", "-p", path] try: runner.run(cmd) except CommandError as e: lgr.error("Remotely creating target directory failed at " "%s.\nError: %s" % (path, str(e))) continue # init git repo cmd = ssh_cmd + ["git", "-C", path, "init"] if shared: cmd.append("--shared=%s" % shared) try: runner.run(cmd) except CommandError as e: lgr.error("Remotely initializing git repository failed at %s." "\nError: %s\nSkipping ..." % (path, str(e))) continue # check git version on remote end: cmd = ssh_cmd + ["git", "version"] try: out, err = runner.run(cmd) git_version = out.lstrip("git version").strip() lgr.debug("Detected git version on server: %s" % git_version) if git_version < "2.4": lgr.error("Git version >= 2.4 needed to configure remote." " Version detected on server: %s\nSkipping ..." % git_version) continue except CommandError as e: lgr.warning( "Failed to determine git version on remote.\n" "Error: {0}\nTrying to configure anyway " "...".format(e.message)) # allow for pushing to checked out branch cmd = ssh_cmd + ["git", "-C", path, "config", "receive.denyCurrentBranch", "updateInstead"] try: runner.run(cmd) except CommandError as e: lgr.warning("git config failed at remote location %s.\n" "You will not be able to push to checked out " "branch." % path) # enable post-update hook: cmd = ssh_cmd + ["mv", opj(path, ".git/hooks/post-update.sample"), opj(path, ".git/hooks/post-update")] try: runner.run(cmd) except CommandError as e: lgr.error("Failed to enable post update hook.\n" "Error: %s" % e.message) # initially update server info "manually": cmd = ssh_cmd + ["git", "-C", path, "update-server-info"] try: runner.run(cmd) except CommandError as e: lgr.error("Failed to update server info.\n" "Error: %s" % e.message) # stop controlmaster (close ssh connection): cmd = ["ssh", "-O", "stop", "-S", control_path, host_name] out, err = runner.run(cmd, expect_stderr=True) if target: # add the sibling(s): if target_url is None: target_url = sshurl if target_pushurl is None: target_pushurl = sshurl result_adding = AddSibling()(dataset=ds, name=target, url=target_url, pushurl=target_pushurl, recursive=recursive, force=existing in {'replace'})
def test_clone(src, tempdir): # Verify that all our repos are clonable r = Runner() output = r.run(["git" , "clone", src, tempdir], log_online=True) #status, output = getstatusoutput("git clone %(src)s %(tempdir)s" % locals()) ok_(os.path.exists(os.path.join(tempdir, ".git")))
class runner(SuprocBenchmarks): """Some rudimentary tests to see if there is no major slowdowns from Runner """ def setup(self): self.runner = Runner() # older versions might not have it try: from datalad.cmd import GitRunner self.git_runner = GitRunner() except ImportError: pass def time_echo(self): self.runner.run("echo") def time_echo_gitrunner(self): self.git_runner.run("echo") # Following "track" measures computing overhead comparing to the simplest # os.system call on the same command without carrying for in/out unit = "% overhead" def _get_overhead(self, cmd, nrepeats=3, **run_kwargs): """Estimate overhead over running command via the simplest os.system and to not care about any output """ # asv does not repeat tracking ones I think, so nrepeats overheads = [] for _ in range(nrepeats): t0 = time() os.system(cmd + " >/dev/null 2>&1") t1 = time() self.runner.run(cmd, **run_kwargs) t2 = time() overhead = 100 * ((t2 - t1) / (t1 - t0) - 1.0) # print("O :", t1 - t0, t2 - t0, overhead) overheads.append(overhead) overhead = round(sum(overheads) / len(overheads), 2) #overhead = round(min(overheads), 2) return overhead def track_overhead_echo(self): return self._get_overhead("echo") # 100ms chosen below as providing some sensible stability for me. # at 10ms -- too much variability def track_overhead_100ms(self): return self._get_overhead("sleep 0.1") def track_overhead_heavyout(self): # run busyloop for 100ms outputing as much as it could return self._get_overhead(heavyout_cmd) def track_overhead_heavyout_online_through(self): return self._get_overhead(heavyout_cmd, log_stderr='offline', # needed to would get stuck log_online=True) def track_overhead_heavyout_online_process(self): return self._get_overhead(heavyout_cmd, log_stdout=lambda s: '', log_stderr='offline', # needed to would get stuck log_online=True) # # Probably not really interesting, and good lord wobbles around 0 # def track_overhead_heavyout_offline(self): # return self._get_overhead(heavyout_cmd, # log_stdout='offline', # log_stderr='offline') # TODO: track the one with in/out, i.e. for those BatchedProcesses
def __call__(dataset=None, name=None, url=None, pushurl=None, recursive=False, force=False): # TODO: Detect malformed URL and fail? if name is None or (url is None and pushurl is None): raise ValueError("""insufficient information to add a sibling (needs at least a dataset, a name and an URL).""") if url is None: url = pushurl # shortcut ds = dataset if ds is not None and not isinstance(ds, Dataset): ds = Dataset(ds) if ds is None: # try to find a dataset at or above CWD dspath = GitRepo.get_toppath(abspath(getpwd())) if dspath is None: raise ValueError( "No dataset found at or above {0}.".format(getpwd())) ds = Dataset(dspath) lgr.debug("Resolved dataset for target creation: {0}".format(ds)) assert(ds is not None and name is not None and url is not None) if not ds.is_installed(): raise ValueError("Dataset {0} is not installed yet.".format(ds)) assert(ds.repo is not None) ds_basename = basename(ds.path) repos = { ds_basename: {'repo': ds.repo} } if recursive: for subds in ds.get_dataset_handles(recursive=True): sub_path = opj(ds.path, subds) repos[ds_basename + '/' + subds] = { # repos[subds] = { 'repo': GitRepo(sub_path, create=False) } # Note: This is copied from create_publication_target_sshwebserver # as it is the same logic as for its target_dir. # TODO: centralize and generalize template symbol handling # TODO: Check pushurl for template symbols too. Probably raise if only # one of them uses such symbols replicate_local_structure = False if "%NAME" not in url: replicate_local_structure = True for repo in repos: if not replicate_local_structure: repos[repo]['url'] = url.replace("%NAME", repo.replace("/", "-")) if pushurl: repos[repo]['pushurl'] = pushurl.replace("%NAME", repo.replace("/", "-")) else: repos[repo]['url'] = url if pushurl: repos[repo]['pushurl'] = pushurl if repo != ds_basename: repos[repo]['url'] = _urljoin(repos[repo]['url'], repo[len(ds_basename)+1:]) if pushurl: repos[repo]['pushurl'] = _urljoin(repos[repo]['pushurl'], repo[len(ds_basename)+1:]) # collect existing remotes: already_existing = list() conflicting = list() for repo in repos: if name in repos[repo]['repo'].git_get_remotes(): already_existing.append(repo) lgr.debug("""Remote '{0}' already exists in '{1}'.""".format(name, repo)) existing_url = repos[repo]['repo'].git_get_remote_url(name) existing_pushurl = \ repos[repo]['repo'].git_get_remote_url(name, push=True) if repos[repo]['url'].rstrip('/') != existing_url.rstrip('/') \ or (pushurl and existing_pushurl and repos[repo]['pushurl'].rstrip('/') != existing_pushurl.rstrip('/')) \ or (pushurl and not existing_pushurl): conflicting.append(repo) if not force and conflicting: raise RuntimeError("Sibling '{0}' already exists with conflicting" " URL for {1} dataset(s). {2}".format( name, len(conflicting), conflicting)) runner = Runner() successfully_added = list() for repo in repos: if repo in already_existing: if repo not in conflicting: lgr.debug("Skipping {0}. Nothing to do.".format(repo)) continue # rewrite url cmd = ["git", "remote", "set-url", name, repos[repo]['url']] runner.run(cmd, cwd=repos[repo]['repo'].path) else: # add the remote cmd = ["git", "remote", "add", name, repos[repo]['url']] runner.run(cmd, cwd=repos[repo]['repo'].path) if pushurl: cmd = ["git", "remote", "set-url", "--push", name, repos[repo]['pushurl']] runner.run(cmd, cwd=repos[repo]['repo'].path) successfully_added.append(repo) return successfully_added
def populate(self): super(NestedDataset, self).populate() ds = SubmoduleDataset() ds.create() from datalad.cmd import Runner runner = Runner() kw = dict(expect_stderr=True) runner.run(['git', 'submodule', 'add', ds.url, 'subdataset'], cwd=self.path, **kw) runner.run(['git', 'submodule', 'add', ds.url, 'subsubdataset'], cwd=opj(self.path, 'subdataset'), **kw) runner.run(['git', 'commit', '-m', 'Added subdataset.'], cwd=opj(self.path, 'subdataset'), **kw) runner.run(['git', 'commit', '-a', '-m', 'Added subdatasets.'], cwd=self.path, **kw) runner.run(['git', 'submodule', 'update', '--init', '--recursive'], cwd=self.path, **kw) # init all annexes for s in ('', 'subdataset', opj('subdataset', 'subsubdataset')): runner.run(['git', 'annex', 'init'], cwd=opj(self.path, s), expect_stderr=True)
class ConfigManager(object): """Thin wrapper around `git-config` with support for a dataset configuration. The general idea is to have an object that is primarily used to read/query configuration option. Upon creation, current configuration is read via one (or max two, in the case of the presence of dataset-specific configuration) calls to `git config`. If this class is initialized with a Dataset instance, it supports reading and writing configuration from ``.datalad/config`` inside a dataset too. This file is committed to Git and hence useful to ship certain configuration items with a dataset. The API aims to provide the most significant read-access API of a dictionary, the Python ConfigParser, and GitPython's config parser implementations. This class is presently not capable of efficiently writing multiple configurations items at once. Instead, each modification results in a dedicated call to `git config`. This author thinks this is OK, as he cannot think of a situation where a large number of items need to be written during normal operation. If such need arises, various solutions are possible (via GitPython, or an independent writer). Any DATALAD_* environment variable is also presented as a configuration item. Settings read from environment variables are not stored in any of the configuration file, but are read dynamically from the environment at each `reload()` call. Their values take precedence over any specification in configuration files. Parameters ---------- dataset : Dataset, optional If provided, all `git config` calls are executed in this dataset's directory. Moreover, any modifications are, by default, directed to this dataset's configuration file (which will be created on demand) dataset_only : bool If True, configuration items are only read from a datasets persistent configuration file, if any present (the one in ``.datalad/config``, not ``.git/config``). """ def __init__(self, dataset=None, dataset_only=False): # store in a simple dict # no subclassing, because we want to be largely read-only, and implement # config writing separately self._store = {} self._dataset_path = dataset.path if dataset else None self._dataset_only = dataset_only # Since configs could contain sensitive information, to prevent # any "facilitated" leakage -- just disable logging of outputs for # this runner run_kwargs = dict(log_outputs=False) if dataset is not None: # make sure we run the git config calls in the dataset # to pick up the right config files run_kwargs['cwd'] = dataset.path self._runner = Runner(**run_kwargs) self.reload() def reload(self): """Reload all configuration items from the configured sources""" self._store = {} # 2-step strategy: # - load datalad dataset config from dataset # - load git config from all supported by git sources # in doing so we always stay compatible with where Git gets its # config from, but also allow to override persistent information # from dataset locally or globally if self._dataset_path: # now any dataset config dscfg_fname = opj(self._dataset_path, '.datalad', 'config') if exists(dscfg_fname): stdout, stderr = self._run(['-z', '-l', '--file', dscfg_fname], log_stderr=True) # overwrite existing value, do not amend to get multi-line # values self._store = _parse_gitconfig_dump( stdout, self._store, replace=False) if not self._dataset_only: stdout, stderr = self._run(['-z', '-l'], log_stderr=True) self._store = _parse_gitconfig_dump( stdout, self._store, replace=True) # override with environment variables self._store = _parse_env(self._store) @_where_reload def obtain(self, var, default=None, dialog_type=None, valtype=None, store=False, where=None, reload=True, **kwargs): """ Convenience method to obtain settings interactively, if needed A UI will be used to ask for user input in interactive sessions. Questions to ask, and additional explanations can be passed directly as arguments, or retrieved from a list of pre-configured items. Additionally, this method allows for type conversion and storage of obtained settings. Both aspects can also be pre-configured. Parameters ---------- var : str Variable name including any section like `git config` expects them, e.g. 'core.editor' default : any type In interactive sessions and if `store` is True, this default value will be presented to the user for confirmation (or modification). In all other cases, this value will be silently assigned unless there is an existing configuration setting. dialog_type : {'question', 'yesno', None} Which dialog type to use in interactive sessions. If `None`, pre-configured UI options are used. store : bool Whether to store the obtained value (or default) %s `**kwargs` Additional arguments for the UI function call, such as a question `text`. """ # do local import, as this module is import prominently and the # could theroetically import all kind of weired things for type # conversion from datalad.interface.common_cfg import definitions as cfg_defs # fetch what we know about this variable cdef = cfg_defs.get(var, {}) # type conversion setup if valtype is None and 'type' in cdef: valtype = cdef['type'] if valtype is None: valtype = lambda x: x # any default? if default is None and 'default' in cdef: default = cdef['default'] _value = None if var in self: # nothing needs to be obtained, it is all here already _value = self[var] elif store is False and default is not None: # nothing will be stored, and we have a default -> no user confirmation # we cannot use logging, because we want to use the config to confiugre # the logging #lgr.debug('using default {} for config setting {}'.format(default, var)) _value = default if _value is not None: # we got everything we need and can exit early try: return valtype(_value) except Exception as e: raise ValueError( "value '{}' of existing configuration for '{}' cannot be " "converted to the desired type '{}' ({})".format( _value, var, valtype, exc_str(e))) # now we need to try to obtain something from the user from datalad.ui import ui # configure UI dialog_opts = kwargs if dialog_type is None: # no override # check for common knowledge on how to obtain a value if 'ui' in cdef: dialog_type = cdef['ui'][0] # pull standard dialog settings dialog_opts = cdef['ui'][1] # update with input dialog_opts.update(kwargs) if (not ui.is_interactive or dialog_type is None) and default is None: raise RuntimeError( "cannot obtain value for configuration item '{}', " "not preconfigured, no default, no UI available".format(var)) if not hasattr(ui, dialog_type): raise ValueError("UI '{}' does not support dialog type '{}'".format( ui, dialog_type)) # configure storage destination, if needed if store: if where is None and 'destination' in cdef: where = cdef['destination'] if where is None: raise ValueError( "request to store configuration item '{}', but no " "storage destination specified".format(var)) # obtain via UI dialog = getattr(ui, dialog_type) _value = dialog(default=default, **dialog_opts) if _value is None: # we got nothing if default is None: raise RuntimeError( "could not obtain value for configuration item '{}', " "not preconfigured, no default".format(var)) # XXX maybe we should return default here, even it was returned # from the UI -- if that is even possible # execute type conversion before storing to check that we got # something that looks like what we want try: value = valtype(_value) except Exception as e: raise ValueError( "cannot convert user input `{}` to desired type ({})".format( _value, exc_str(e))) # XXX we could consider "looping" until we have a value of proper # type in case of a user typo... if store: # store value as it was before any conversion, needs to be str # anyway # needs string conversion nevertheless, because default could come # in as something else self.add(var, '{}'.format(_value), where=where, reload=reload) return value # # Compatibility with dict API # def __len__(self): return len(self._store) def __getitem__(self, key): return self._store.__getitem__(key) def __contains__(self, key): return self._store.__contains__(key) def keys(self): """Returns list of configuration item names""" return self._store.keys() def get(self, key, default=None): """D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.""" return self._store.get(key, default) # # Compatibility with ConfigParser API # def sections(self): """Returns a list of the sections available""" return list(set([cfg_section_regex.match(k).group(1) for k in self._store])) def options(self, section): """Returns a list of options available in the specified section.""" opts = [] for k in self._store: sec, opt = cfg_sectionoption_regex.match(k).groups() if sec == section: opts.append(opt) return opts def has_section(self, section): """Indicates whether a section is present in the configuration""" for k in self._store: if k.startswith(section): return True return False def has_option(self, section, option): """If the given section exists, and contains the given option""" for k in self._store: sec, opt = cfg_sectionoption_regex.match(k).groups() if sec == section and opt == option: return True return False def getint(self, section, option): """A convenience method which coerces the option value to an integer""" return int(self.get_value(section, option)) def getbool(self, section, option, default=None): """A convenience method which coerces the option value to a bool Values "on", "yes", "true" and any int!=0 are considered True Values which evaluate to bool False, "off", "no", "false" are considered False TypeError is raised for other values. """ val = self.get_value(section, option, default=default) return anything2bool(val) def getfloat(self, section, option): """A convenience method which coerces the option value to a float""" return float(self.get_value(section, option)) # this is a hybrid of ConfigParser and dict API def items(self, section=None): """Return a list of (name, value) pairs for each option Optionally limited to a given section. """ if section is None: return self._store.items() return [(k, v) for k, v in self._store.items() if cfg_section_regex.match(k).group(1) == section] # # Compatibility with GitPython's ConfigParser # def get_value(self, section, option, default=None): """Like `get()`, but with an optional default value If the default is not None, the given default value will be returned in case the option did not exist. This behavior immitates GitPython's config parser. """ try: return self['.'.join((section, option))] except KeyError as e: # this strange dance is needed because gitpython does it this way if default is not None: return default else: raise e # # Modify configuration (proxy respective git-config call) # @_where_reload def _run(self, args, where=None, reload=False, **kwargs): """Centralized helper to run "git config" calls Parameters ---------- args : list Arguments to pass for git config %s **kwargs Keywords arguments for Runner's call """ if where: args = self._get_location_args(where) + args out = self._runner.run(['git', 'config'] + args, **kwargs) if reload: self.reload() return out def _get_location_args(self, where, args=None): if args is None: args = [] cfg_labels = ('dataset', 'local', 'global') if where not in cfg_labels: raise ValueError( "unknown configuration label '{}' (not in {})".format( where, cfg_labels)) if where == 'dataset': if not self._dataset_path: raise ValueError( 'ConfigManager cannot store to configuration to dataset, none specified') # create an empty config file if none exists, `git config` will # fail otherwise dscfg_dirname = opj(self._dataset_path, '.datalad') dscfg_fname = opj(dscfg_dirname, 'config') if not exists(dscfg_dirname): os.makedirs(dscfg_dirname) if not exists(dscfg_fname): open(dscfg_fname, 'w').close() args.extend(['--file', opj(self._dataset_path, '.datalad', 'config')]) elif where == 'global': args.append('--global') elif where == 'local': args.append('--local') return args @_where_reload def add(self, var, value, where='dataset', reload=True): """Add a configuration variable and value Parameters ---------- var : str Variable name including any section like `git config` expects them, e.g. 'core.editor' value : str Variable value %s""" self._run(['--add', var, value], where=where, reload=reload, log_stderr=True) @_where_reload def set(self, var, value, where='dataset', reload=True, force=False): """Set a variable to a value. In opposition to `add`, this replaces the value of `var` if there is one already. Parameters ---------- var : str Variable name including any section like `git config` expects them, e.g. 'core.editor' value : str Variable value force: bool if set, replaces all occurrences of `var` by a single one with the given `value`. Otherwise raise if multiple entries for `var` exist already %s""" from datalad.support.gitrepo import to_options self._run(to_options(replace_all=force) + [var, value], where=where, reload=reload, log_stderr=True) @_where_reload def rename_section(self, old, new, where='dataset', reload=True): """Rename a configuration section Parameters ---------- old : str Name of the section to rename. new : str Name of the section to rename to. %s""" self._run(['--rename-section', old, new], where=where, reload=reload) @_where_reload def remove_section(self, sec, where='dataset', reload=True): """Rename a configuration section Parameters ---------- sec : str Name of the section to remove. %s""" self._run(['--remove-section', sec], where=where, reload=reload) @_where_reload def unset(self, var, where='dataset', reload=True): """Remove all occurrences of a variable Parameters ---------- var : str Name of the variable to remove %s""" # use unset all as it is simpler for now self._run(['--unset-all', var], where=where, reload=reload)