def check_compress_file(ext, annex, path=None, name=None):
# we base the archive name on the filename, in order to also
# be able to properly test compressors where the corresponding
# archive format has no capability of storing a filename
# (i.e. where the archive name itself determines the filename
# of the decompressed file, like .xz)
archive = op.join(name, _filename + ext)
compress_files([_filename], archive, path=path)
assert_true(op.exists(archive))
if annex:
# It should work even when file is annexed and is a symlink to the
# key
from datalad.support.annexrepo import AnnexRepo
repo = AnnexRepo(path, init=True)
repo.add(_filename)
repo.commit(files=[_filename], msg="commit")
dir_extracted = name + "_extracted"
try:
decompress_file(archive, dir_extracted)
except MissingExternalDependency as exc:
raise SkipTest() from exc
_filepath = op.join(dir_extracted, _filename)
ok_file_has_content(_filepath, 'content')
def test_timeout_nothing():
# Expect timeout protocol calls for the process on long running processes,
# if the specified timeout is short enough.
class TestProtocol(NoCapture):
def __init__(self,
timeout_queue: List):
NoCapture.__init__(self)
self.timeout_queue = timeout_queue
self.counter = count()
def timeout(self, fd: Optional[int]) -> bool:
self.timeout_queue.append(fd)
return False
stdin_queue = queue.Queue()
for i in range(12):
stdin_queue.put(b"\x00" * 1024)
stdin_queue.put(None)
timeout_queue = []
run_command(
py2cmd("import time; time.sleep(.4)\n"),
stdin=stdin_queue,
protocol=TestProtocol,
timeout=.1,
protocol_kwargs=dict(timeout_queue=timeout_queue)
)
# Ensure that we have only process timeouts and at least one
assert_true(all(map(lambda e: e is None, timeout_queue)))
assert_true(len(timeout_queue) > 0)
def test_exit_4():
rt = ThreadedRunner(cmd=["sleep", "4"],
stdin=None,
protocol_class=GenNothing,
timeout=.5)
tuple(rt.run())
assert_true(rt.process.poll() is not None)
def test_timeout_process():
# Expect timeouts on stdin, stdout, stderr, and the process
class TestProtocol(StdOutErrCapture):
def __init__(self,
timeout_queue: List):
StdOutErrCapture.__init__(self)
self.timeout_queue = timeout_queue
self.counter = count()
def timeout(self, fd: Optional[int]) -> bool:
self.timeout_queue.append((self.counter.__next__(), fd))
return False
stdin_queue = queue.Queue()
for i in range(12):
stdin_queue.put(b"\x00" * 1024)
stdin_queue.put(None)
timeout_queue = []
run_command(
py2cmd("import time;time.sleep(.5)\n"),
stdin=stdin_queue,
protocol=TestProtocol,
timeout=.1,
protocol_kwargs=dict(timeout_queue=timeout_queue)
)
# Expect at least one timeout for stdout and stderr.
# there might be more.
sources = (1, 2)
assert_true(len(timeout_queue) >= len(sources))
for source in sources:
assert_true(any(filter(lambda t: t[1] == source, timeout_queue)))
def test_path_and_url(path, url):
def _urlopen(url, auth=None):
req = Request(url)
if auth:
req.add_header(
"Authorization", b"Basic " + base64.standard_b64encode(
'{0}:{1}'.format(*auth).encode('utf-8')))
return urlopen(req)
# @serve_ should remove http_proxy from the os.environ if was present
if not on_windows:
assert_false('http_proxy' in os.environ)
# get the "dir-view"
dirurl = url + test_fpath.parent.as_posix()
u = _urlopen(dirurl, auth)
assert_true(u.getcode() == 200)
html = u.read()
# get the actual content
file_html = _urlopen(url + url_quote(test_fpath.as_posix()),
auth).read().decode()
# verify we got the right one
eq_(file_html, test_fpath_full.read_text())
if bs4 is None:
return
# MIH is not sure what this part below is supposed to do
# possibly some kind of internal consistency test
soup = bs4.BeautifulSoup(html, "html.parser")
href_links = [txt.get('href') for txt in soup.find_all('a')]
assert_true(len(href_links) == 1)
parsed_url = f"{dirurl}/{href_links[0]}"
u = _urlopen(parsed_url, auth)
html = u.read().decode()
eq_(html, file_html)
def decorated_test1(url):
# we expect a file-scheme url to a cached version of `ds_url`
expect_origin_path = cache_dir / name_in_cache
assert_equal(expect_origin_path.as_uri(), url)
origin = Dataset(expect_origin_path)
assert_true(origin.is_installed())
assert_false(origin.repo.file_has_content(str(annexed_file)))
def test_ExtractedArchive(path=None):
archive = op.join(path, fn_archive_obscure_ext)
earchive = ExtractedArchive(archive)
assert_false(op.exists(earchive.path))
# no longer the case -- just using hash for now
# assert_in(os.path.basename(archive), earchive.path)
fpath = op.join(
fn_archive_obscure, # lead directory
fn_in_archive_obscure)
extracted = earchive.get_extracted_filename(fpath)
eq_(extracted, op.join(earchive.path, fpath))
assert_false(op.exists(extracted)) # not yet
extracted_ = earchive.get_extracted_file(fpath)
eq_(extracted, extracted_)
assert_true(op.exists(extracted)) # now it should
extracted_files = earchive.get_extracted_files()
ok_generator(extracted_files)
eq_(
sorted(extracted_files),
sorted([
# ['bbc/3.txt', 'bbc/abc']
op.join(fn_archive_obscure, fn_in_archive_obscure),
op.join(fn_archive_obscure, '3.txt')
]))
earchive.clean()
if not dl_cfg.get('datalad.tests.temp.keep'):
assert_false(op.exists(earchive.path))
def test_ignore_nondatasets(path=None):
# we want to ignore the version/commits for this test
def _kill_time(meta):
for m in meta:
for k in ('version', 'shasum'):
if k in m:
del m[k]
return meta
ds = Dataset(path).create()
meta = _kill_time(ds.metadata(reporton='datasets', on_failure='ignore'))
n_subm = 0
# placing another repo in the dataset has no effect on metadata
for cls, subpath in ((GitRepo, 'subm'), (AnnexRepo, 'annex_subm')):
subm_path = opj(ds.path, subpath)
r = cls(subm_path, create=True)
with open(opj(subm_path, 'test'), 'w') as f:
f.write('test')
r.add('test')
r.commit('some')
assert_true(Dataset(subm_path).is_installed())
assert_equal(meta, _kill_time(ds.metadata(reporton='datasets', on_failure='ignore')))
# making it a submodule has no effect either
ds.save(subpath)
assert_equal(len(ds.subdatasets()), n_subm + 1)
assert_equal(meta, _kill_time(ds.metadata(reporton='datasets', on_failure='ignore')))
n_subm += 1
def test_configs(path=None):
# set up dataset with registered procedure (c&p from test_basics):
ds = Dataset(path).create(force=True)
ds.run_procedure('cfg_yoda')
# configure dataset to look for procedures in its code folder
ds.config.add('datalad.locations.dataset-procedures',
'code',
scope='branch')
# 1. run procedure based on execution guessing by run_procedure:
ds.run_procedure(spec=['datalad_test_proc', 'some_arg'])
# look for traces
ok_file_has_content(op.join(ds.path, 'fromproc.txt'), 'some_arg\n')
# 2. now configure specific call format including usage of substitution config
# for run:
ds.config.add('datalad.procedures.datalad_test_proc.call-format',
u'%s {script} {ds} {{mysub}} {args}' %
quote_cmdlinearg(sys.executable),
scope='branch')
ds.config.add('datalad.run.substitutions.mysub',
'dataset-call-config',
scope='branch')
# TODO: Should we allow for --inputs/--outputs arguments for run_procedure
# (to be passed into run)?
ds.unlock("fromproc.txt")
# run again:
ds.run_procedure(spec=['datalad_test_proc', 'some_arg'])
# look for traces
ok_file_has_content(op.join(ds.path, 'fromproc.txt'),
'dataset-call-config\n')
# 3. have a conflicting config at user-level, which should override the
# config on dataset level:
ds.config.add('datalad.procedures.datalad_test_proc.call-format',
u'%s {script} {ds} local {args}' %
quote_cmdlinearg(sys.executable),
scope='local')
ds.unlock("fromproc.txt")
# run again:
ds.run_procedure(spec=['datalad_test_proc', 'some_arg'])
# look for traces
ok_file_has_content(op.join(ds.path, 'fromproc.txt'), 'local\n')
# 4. get configured help message:
r = ds.run_procedure('datalad_test_proc',
help_proc=True,
on_failure='ignore')
assert_true(len(r) == 1)
assert_in_results(r, status="impossible")
ds.config.add('datalad.procedures.datalad_test_proc.help',
"This is a help message",
scope='branch')
r = ds.run_procedure('datalad_test_proc', help_proc=True)
assert_true(len(r) == 1)
assert_in_results(r, message="This is a help message", status='ok')
def test_noannex_fail_if_has_annexed(path=None):
ds = Dataset(path).create(force=True)
ds.save()
assert_true(isinstance(ds.repo, AnnexRepo))
# internally procedure raises RuntimeError, but since we run it via runner, we
# get CommandError here
with assert_raises(CommandError):
ds.run_procedure('cfg_noannex') # we are killing annex while ds.repo
def test_exit_3():
# Expect the process to be closed after
# the generator exits.
rt = ThreadedRunner(cmd=["sleep", "4"],
stdin=None,
protocol_class=GenStdoutStderr,
timeout=.5,
exception_on_error=False)
tuple(rt.run())
assert_true(rt.process.poll() is not None)
def test_external_versions_popular_packages():
ev = ExternalVersions()
for modname in ('scipy', 'numpy', 'mvpa2', 'sklearn', 'statsmodels',
'pandas', 'matplotlib', 'psychopy', 'github'):
_test_external(ev, modname)
# more of a smoke test
assert_false(linesep in ev.dumps())
assert_true(ev.dumps(indent=True).endswith(linesep))
def test_external_versions_rogue_module(topd=None):
ev = ExternalVersions()
# if module throws some other non-ImportError exception upon import
# we must not crash, but issue a warning
modname = 'verycustomrogue__'
create_tree(topd, {modname + '.py': 'raise Exception("pickaboo")'})
with patch('sys.path', [topd]), \
swallow_logs(new_level=logging.WARNING) as cml:
assert ev[modname] is None
assert_true(ev.dumps(indent=True).endswith(linesep))
assert_in('pickaboo', cml.out)
def test_bare(src=None, path=None):
# create a proper datalad dataset with all bells and whistles
ds = Dataset(src).create()
dlconfig_sha = ds.repo.call_git(['rev-parse', 'HEAD:.datalad/config'])
# can we handle a bare repo version of it?
gr = AnnexRepo.clone(src,
path,
clone_options=['--bare', '-b', DEFAULT_BRANCH])
# we had to specifically checkout the standard branch, because on crippled
# FS, HEAD will point to an adjusted branch by default, and the test logic
# below does not account for this case.
# this should just make sure the bare repo has the expected setup,
# but it should still be bare. Let's check that to be sure
assert_true(gr.bare)
# do we read the correct local config?
assert_in(gr.pathobj / 'config', gr.config._stores['git']['files'])
# do we pick up the default branch config too?
assert_in('blob:HEAD:.datalad/config',
gr.config._stores['branch']['files'])
# and track its reload stamp via its file shasum
assert_equal(
dlconfig_sha,
gr.config._stores['branch']['stats']['blob:HEAD:.datalad/config'])
# check that we can pick up the dsid from the commit branch config
assert_equal(ds.id, gr.config.get('datalad.dataset.id'))
# and it is coming from the correct source
assert_equal(ds.id,
gr.config.get_from_source('branch', 'datalad.dataset.id'))
assert_equal(None, gr.config.get_from_source('local',
'datalad.dataset.id'))
# any sensible (and also our CI) test environment(s) should have this
assert_in('user.name', gr.config)
# not set something that wasn't there
obscure_key = 'sec.reallyobscurename!@@.key'
assert_not_in(obscure_key, gr.config)
# to the local config, which is easily accessible
gr.config.set(obscure_key, 'myvalue', scope='local')
assert_equal(gr.config.get(obscure_key), 'myvalue')
# now make sure the config is where we think it is
assert_in(obscure_key.split('.')[1], (gr.pathobj / 'config').read_text())
# update committed config and check update
old_id = ds.id
ds.config.set('datalad.dataset.id', 'surprise!', scope='branch')
ds.save()
# fetch into default branch (like `update`, but for bare-repos)
gr.call_git(
['fetch', f'{DEFAULT_REMOTE}', f'{DEFAULT_BRANCH}:{DEFAULT_BRANCH}'])
# without a reload, no state change, like with non-bare repos
assert_equal(old_id,
gr.config.get_from_source('branch', 'datalad.dataset.id'))
# a non-forced reload() must be enough, because state change
# detection kicks in
gr.config.reload()
assert_equal('surprise!', gr.config.get('datalad.dataset.id'))
def test_batched_close_ok():
# Expect a long wait and no timeout if the process runs longer than timeout
# seconds and the config for "datalad.runtime.stalled-external" has its
# default value.
bc = BatchedCommand(cmd=[sys.executable, "-i", "-u", "-q", "-"], timeout=2)
# Send at least one instruction to start the subprocess
response = bc("import time; print('a')")
assert_equal(response, "a")
bc.stdin_queue.put("time.sleep(.5); exit(3)\n".encode())
bc.close(return_stderr=False)
assert_true(bc.wait_timed_out is False)
assert_equal(bc.return_code, 3)
def test_ephemeral(ds_path=None, store_path=None, clone_path=None):
dspath = Path(ds_path)
store = Path(store_path)
file_test = Path('file1.txt')
file_testsub = Path('sub') / 'other.txt'
# create the original dataset
ds = Dataset(dspath)
ds.create(force=True)
ds.save()
# put into store:
ds.create_sibling_ria("ria+{}".format(store.as_uri()),
"riastore",
new_store_ok=True)
ds.push(to="riastore", data="anything")
# now, get an ephemeral clone from the RIA store:
eph_clone = clone('ria+{}#{}'.format(store.as_uri(), ds.id),
clone_path,
reckless="ephemeral")
# ephemeral clone was properly linked (store has bare repos!):
clone_annex = (eph_clone.repo.dot_git / 'annex')
assert_true(clone_annex.is_symlink())
assert_true(clone_annex.resolve().samefile(store / ds.id[:3] / ds.id[3:] /
'annex'))
if not eph_clone.repo.is_managed_branch():
# TODO: We can't properly handle adjusted branch yet
# we don't need to get files in order to access them:
assert_equal((eph_clone.pathobj / file_test).read_text(), "some")
assert_equal((eph_clone.pathobj / file_testsub).read_text(), "other")
# can we unlock those files?
eph_clone.unlock(file_test)
# change content
(eph_clone.pathobj / file_test).write_text("new content")
eph_clone.save()
# new content should already be in store
# (except the store doesn't know yet)
res = eph_clone.repo.fsck(remote="riastore-storage", fast=True)
assert_equal(len(res), 2)
assert_result_count(res, 1, success=True, file=file_test.as_posix())
assert_result_count(res, 1, success=True, file=file_testsub.as_posix())
# push back git history
eph_clone.push(to=DEFAULT_REMOTE, data="nothing")
# get an update in origin
ds.update(merge=True, reobtain_data=True)
assert_equal((ds.pathobj / file_test).read_text(), "new content")
def test_batched_close_abandon():
# Expect a timeout if the process runs longer than timeout and the config
# for "datalad.runtime.stalled-external" is "abandon".
bc = BatchedCommand(cmd=[sys.executable, "-i", "-u", "-q", "-"],
timeout=.1)
# Send at least one instruction to start the subprocess
response = bc("import time; print('a')")
assert_equal(response, "a")
bc.stdin_queue.put("time.sleep(2); exit(1)\n".encode())
with unittest.mock.patch("datalad.cfg") as cfg_mock:
cfg_mock.configure_mock(**{"obtain.return_value": "abandon"})
bc.close(return_stderr=False)
assert_true(bc.wait_timed_out is True)
assert_is_none(bc.return_code)
def test_zip_archive(path=None):
ds = Dataset(opj(path, 'ds')).create(force=True, annex=False)
ds.save()
with chpwd(path):
ds.export_archive(filename='my', archivetype='zip')
assert_true(os.path.exists('my.zip'))
custom1_md5 = md5sum('my.zip')
time.sleep(1.1)
ds.export_archive(filename='my', archivetype='zip')
assert_equal(md5sum('my.zip'), custom1_md5)
# should be able to export without us cd'ing to that ds directory
ds.export_archive(filename=ds.path, archivetype='zip')
default_name = 'datalad_{}.zip'.format(ds.id)
assert_true(os.path.exists(os.path.join(ds.path, default_name)))
def test_repo_cache(path=None):
ds = Dataset(path)
# none by default
eq_(ds.repo, None)
# make Git repo manually
git = GitRepo(path=path, create=True)
repo = ds.repo
# got one
assert_false(repo is None)
# stays that one
assert_true(ds.repo is repo)
# now turn into an annex
annex = AnnexRepo(path=path, create=True)
# repo instance must change
assert_false(ds.repo is repo)
assert_true(isinstance(ds.repo, AnnexRepo))
def test_text2git(path=None):
# Test if files being correctly annexed in a ds configured with text2git.
TEXT_FILES = ('JSON', 'YAML', 'MARKDOWN', 'empty')
BINARY_FILES = ('0blob', 'emptyline')
ds = Dataset(path).create(force=True)
ds.run_procedure('cfg_text2git')
ds.save(path=TEXT_FILES + BINARY_FILES, message="added all files")
assert_repo_status(ds.path)
# check that text files are not annexed
for f in TEXT_FILES:
assert_false(ds.repo.is_under_annex(f))
# and trivial binaries - annexed
for f in BINARY_FILES:
assert_true(ds.repo.is_under_annex(f))
def test_basic_setup():
# the import alone will verify that all default values match their
# constraints
from datalad import api
# random pick of something that should be there
assert_true(hasattr(api, 'install'))
assert_true(hasattr(api, 'create'))
# make sure all helper utilities do not pollute the namespace
# and we end up only with __...__ attributes
assert_false(
list(
filter(lambda s: s.startswith('_') and not re.match('__.*__', s),
dir(api))))
assert_in('Parameters', api.Dataset.install.__doc__)
assert_in('Parameters', api.Dataset.create.__doc__)
def test_dataset_id(path=None):
ds = Dataset(path)
assert_equal(ds.id, None)
ds.create()
dsorigid = ds.id
# ID is always a UUID
assert_equal(ds.id.count('-'), 4)
assert_equal(len(ds.id), 36)
# Ben: The following part of the test is concerned with creating new objects
# and therefore used to reset the flyweight dict while keeping a ref to
# the old object for comparison etc. This is ugly and in parts
# retesting what is already tested in `test_Dataset_flyweight`. No need
# for that. If we del the last ref to an instance and gc.collect(),
# then we get a new instance on next request. This test should trust
# the result of `test_Dataset_flyweight`.
# creating a new object for the same path
# yields the same ID
del ds
newds = Dataset(path)
assert_equal(dsorigid, newds.id)
# recreating the dataset does NOT change the id
del newds
ds = Dataset(path)
ds.create(annex=False, force=True)
assert_equal(ds.id, dsorigid)
# even adding an annex doesn't
del ds
ds = Dataset(path)
ds.create(force=True)
assert_equal(ds.id, dsorigid)
# dataset ID and annex UUID have nothing to do with each other
# if an ID was already generated
assert_true(ds.repo.uuid != ds.id)
# even if we generate a dataset from scratch with an annex UUID right away,
# this is also not the ID
annexds = Dataset(opj(path, 'scratch')).create()
assert_true(annexds.id != annexds.repo.uuid)
def test_add_delete_after_and_drop_subdir(self=None):
os.mkdir(opj(self.annex.path, 'subdir'))
mv_out = self.annex.call_git(['mv', '1.tar', 'subdir'])
self.annex.commit("moved into subdir")
with chpwd(self.annex.path):
# was failing since deleting without considering if tarball
# was extracted in that tarball directory
commits_prior_master = list(self.annex.get_branch_commits_())
commits_prior = list(self.annex.get_branch_commits_('git-annex'))
add_out = self.ds.add_archive_content(opj('subdir', '1.tar'),
delete_after=True,
drop_after=True)
assert_repo_status(self.annex.path)
if not self.annex.is_managed_branch():
# whole counting logic here is ignorant of adjusted branches
commits_after_master = list(self.annex.get_branch_commits_())
commits_after = list(
self.annex.get_branch_commits_('git-annex'))
# There should be a single commit for all additions +1 to
# initiate datalad-archives gh-1258. If faking dates,
# there should be another +1 because annex.alwayscommit
# isn't set to false.
assert_equal(
len(commits_after),
len(commits_prior) + 2 + self.annex.fake_dates_enabled)
assert_equal(len(commits_after_master),
len(commits_prior_master))
# there should be no .datalad temporary files hanging around
self.assert_no_trash_left_behind()
# and if we add some untracked file, redo, there should be no changes
# to master and file should remain not committed
create_tree(self.annex.path, {'dummy.txt': '123'})
assert_true(self.annex.dirty) # untracked file
add_out = add_archive_content(opj('subdir', '1.tar'),
delete_after=True,
drop_after=True,
allow_dirty=True)
assert_repo_status(self.annex.path, untracked=['dummy.txt'])
assert_equal(len(list(self.annex.get_branch_commits_())),
len(commits_prior_master))
# there should be no .datalad temporary files hanging around
self.assert_no_trash_left_behind()
def test_interactive_communication():
class BidirectionalProtocol(Protocol):
proc_out = True
proc_err = True
def __init__(self, result_pool: dict):
super().__init__()
self.state = 0
self.result_pool = result_pool
def connection_made(self, process):
super().connection_made(process)
os.write(self.process.stdin.fileno(), b"1 + 1\n")
def connection_lost(self, exc):
self.result_pool["connection_lost_called"] = True
def process_exited(self):
self.result_pool["process_exited_called"] = True
def pipe_data_received(self, fd, data):
super().pipe_data_received(fd, data)
if self.state == 0:
self.state += 1
os.write(self.process.stdin.fileno(), b"2 ** 3\n")
if self.state == 1:
self.state += 1
os.write(self.process.stdin.fileno(), b"exit(0)\n")
result_pool = dict()
result = run_command([sys.executable, "-i"],
BidirectionalProtocol,
stdin=subprocess.PIPE,
protocol_kwargs={
"result_pool": result_pool
})
lines = [line.strip() for line in result["stdout"].splitlines()]
eq_(lines, ["2", "8"])
assert_true(result_pool["connection_lost_called"], True)
assert_true(result_pool["process_exited_called"], True)
def test_property_reevaluation(repo1=None):
ds = Dataset(repo1)
assert_is_none(ds.repo)
assert_is_not_none(ds.config)
first_config = ds.config
assert_false(ds._cfg_bound)
assert_is_none(ds.id)
ds.create()
assert_repo_status(repo1)
# after creation, we have `repo`, and `config` was reevaluated to point
# to the repo's config:
assert_is_not_none(ds.repo)
assert_is_not_none(ds.config)
second_config = ds.config
assert_true(ds._cfg_bound)
assert_is(ds.config, ds.repo.config)
assert_is_not(first_config, second_config)
assert_is_not_none(ds.id)
first_id = ds.id
ds.drop(what='all', reckless='kill', recursive=True)
# repo is gone, and config is again reevaluated to only provide user/system
# level config:
assert_false(lexists(ds.path))
assert_is_none(ds.repo)
assert_is_not_none(ds.config)
third_config = ds.config
assert_false(ds._cfg_bound)
assert_is_not(second_config, third_config)
assert_is_none(ds.id)
ds.create()
assert_repo_status(repo1)
# after recreation everything is sane again:
assert_is_not_none(ds.repo)
assert_is_not_none(ds.config)
assert_is(ds.config, ds.repo.config)
forth_config = ds.config
assert_true(ds._cfg_bound)
assert_is_not(third_config, forth_config)
assert_is_not_none(ds.id)
assert_not_equal(ds.id, first_id)
def test_param():
# having a parameter with no information is fine
# it doesn't need a name, because it comes from the signature
# of the actual implementation that is described
p = Parameter()
pname = 'testname'
# minimal docstring
assert_equal(pname, p.get_autodoc('testname'))
doc = 'somedoc'
p = Parameter(doc=doc)
assert_equal('%s\n %s.' % (pname, doc), p.get_autodoc('testname'))
# constraints
p = Parameter(doc=doc, constraints=cnstr.EnsureInt() | cnstr.EnsureStr())
autodoc = p.get_autodoc('testname')
assert_true('int or str' in autodoc)
with assert_raises(ValueError) as cmr:
Parameter(unknown_arg=123)
assert_in('Detected unknown argument(s) for the Parameter: unknown_arg',
str(cmr.value))
def test_subprocess_return_code_capture():
class KillProtocol(Protocol):
proc_out = True
proc_err = True
def __init__(self, signal_to_send: int, result_pool: dict):
super().__init__()
self.signal_to_send = signal_to_send
self.result_pool = result_pool
def connection_made(self, process):
super().connection_made(process)
process.send_signal(self.signal_to_send)
def connection_lost(self, exc):
self.result_pool["connection_lost_called"] = (True, exc)
def process_exited(self):
self.result_pool["process_exited_called"] = True
# windows doesn't support SIGINT but would need a Ctrl-C
signal_to_send = signal.SIGTERM if on_windows else signal.SIGINT
result_pool = dict()
result = run_command(["waitfor", "/T", "10000", "TheComputerTurnsIntoATulip"]
if on_windows
else ["sleep", "10000"],
KillProtocol,
None,
{
"signal_to_send": signal_to_send,
"result_pool": result_pool
},
exception_on_error=False)
if not on_windows:
# this one specifically tests the SIGINT case, which is not supported
# on windows
eq_(result["code"], -signal_to_send)
assert_true(result_pool["connection_lost_called"][0])
assert_true(result_pool["process_exited_called"])
def test_symlinked_dataset_properties(repo1=None,
repo2=None,
repo3=None,
non_repo=None,
symlink=None):
ds = Dataset(repo1).create()
# now, let ds be a symlink and change that symlink to point to different
# things:
ar2 = AnnexRepo(repo2)
ar3 = AnnexRepo(repo3)
assert_true(os.path.isabs(non_repo))
os.symlink(repo1, symlink)
ds_link = Dataset(symlink)
assert_is(ds_link.repo, ds.repo) # same Repo instance
assert_is_not(ds_link, ds) # but not the same Dataset instance
assert_is(ds_link.config, ds.repo.config)
assert_true(ds_link._cfg_bound)
assert_is_not_none(ds_link.id)
# same id, although different Dataset instance:
assert_equal(ds_link.id, ds.id)
os.unlink(symlink)
os.symlink(repo2, symlink)
assert_is(ds_link.repo, ar2) # same Repo instance
assert_is(ds_link.config, ar2.config)
assert_true(ds_link._cfg_bound)
# id is None again, since this repository is an annex but there was no
# Dataset.create() called yet.
assert_is_none(ds_link.id)
os.unlink(symlink)
os.symlink(repo3, symlink)
assert_is(ds_link.repo, ar3) # same Repo instance
assert_is(ds_link.config, ar3.config)
assert_true(ds_link._cfg_bound)
# id is None again, since this repository is an annex but there was no
# Dataset.create() called yet.
assert_is_none(ds_link.id)
os.unlink(symlink)
os.symlink(non_repo, symlink)
assert_is_none(ds_link.repo)
assert_is_not(ds_link.config, ar3.config)
assert_false(ds_link._cfg_bound)
assert_is_none(ds_link.id)
def test_external_versions_basic():
ev = ExternalVersions()
our_module = 'datalad'
assert_equal(ev.versions, {})
assert_equal(ev[our_module], __version__)
# and it could be compared
assert_greater_equal(ev[our_module], __version__)
# We got some odd failure in this test not long are after switching to versionner
# https://github.com/datalad/datalad/issues/5785. Verify that we do get expected
# data types
our_version = ev[our_module].version
assert isinstance(
our_version,
(str, list)), f"Got {our_version!r} of type {type(our_version)}"
assert_greater(ev[our_module], '0.1')
assert_equal(list(ev.keys()), [our_module])
assert_true(our_module in ev)
assert_false('unknown' in ev)
# all are LooseVersions now
assert_true(isinstance(ev[our_module], LooseVersion))
version_str = __version__
assert_equal(ev.dumps(), "Versions: %s=%s" % (our_module, version_str))
# For non-existing one we get None
assert_equal(ev['custom__nonexisting'], None)
# and nothing gets added to _versions for nonexisting
assert_equal(set(ev.versions.keys()), {our_module})
# but if it is a module without version, we get it set to UNKNOWN
assert_equal(ev['os'], ev.UNKNOWN)
# And get a record on that inside
assert_equal(ev.versions.get('os'), ev.UNKNOWN)
# And that thing is "True", i.e. present
assert (ev['os'])
# but not comparable with anything besides itself (was above)
assert_raises(TypeError, cmp, ev['os'], '0')
assert_raises(TypeError, assert_greater, ev['os'], '0')
return
def test_blocking_thread_exit():
read_queue = queue.Queue()
(read_descriptor, write_descriptor) = os.pipe()
read_file = os.fdopen(read_descriptor, "rb")
read_thread = ReadThread(
identifier="test thread",
user_info=read_descriptor,
source=read_file,
destination_queue=read_queue,
signal_queues=[]
)
read_thread.start()
os.write(write_descriptor, b"some data")
assert_true(read_thread.is_alive())
identifier, state, data = read_queue.get()
eq_(data, b"some data")
read_thread.request_exit()
# Check the blocking part
sleep(.3)
assert_true(read_thread.is_alive())
# Check actual exit, we will not get
# "more data" when exit was requested,
# because the thread will not attempt
# a write
os.write(write_descriptor, b"more data")
read_thread.join()
print(read_queue.queue)
assert_true(read_queue.empty())
