def check_python_eval(cmd, path):
ds = Dataset(path).create()
res = ds.foreach_dataset(cmd, cmd_type='eval')
eq_(len(res), 1)
expected_variables = {'ds', 'pwd', 'refds'}
eq_(expected_variables.intersection(res[0]['result']), expected_variables)
# besides expected, there could be few more ATM, +5 arbitrarily just to test
# that we are not leaking too much
assert_greater(len(expected_variables) + 5, len(res[0]['result']))
def _test_external(ev, modname):
try:
exec("import %s" % modname, globals(), locals())
except ImportError:
raise SkipTest("External %s not present" % modname)
except Exception as e:
raise SkipTest("External %s fails to import" % modname) from e
assert (ev[modname] is not ev.UNKNOWN)
assert_greater(ev[modname], '0.0.1')
assert_greater('1000000.0', ev[modname]) # unlikely in our lifetimes
def test__version__():
# in released stage, version in the last CHANGELOG entry
# should correspond to the one in datalad
CHANGELOG_filename = op.join(
op.dirname(__file__), op.pardir, op.pardir, 'CHANGELOG.md')
if not op.exists(CHANGELOG_filename):
raise SkipTest("no %s found" % CHANGELOG_filename)
regex = re.compile(r'^# '
r'(?P<version>[0-9]+\.[0-9.abcrc~]+)\s+'
r'\((?P<date>.*)\)'
)
with open(CHANGELOG_filename, 'rb') as f:
for line in f:
line = line.rstrip()
if not line.startswith(b'# '):
# The first section header we hit, must be our changelog entry
continue
reg = regex.match(ensure_unicode(line))
if not reg: # first one at that level is the one
raise AssertionError(
"Following line must have matched our regex: %r" % line)
regd = reg.groupdict()
changelog_version = regd['version']
lv_changelog_version = Version(changelog_version)
# we might have a suffix - sanitize
san__version__ = __version__.rstrip('.dirty')
lv__version__ = Version(san__version__)
if '???' in regd['date'] and 'will be better than ever' in regd['codename']:
# we only have our template
# we can only assert that its version should be higher than
# the one we have now
assert_greater(lv_changelog_version, lv__version__)
else:
# should be a "release" record
assert_not_in('???', regd['date'])
ok_startswith(__version__, changelog_version)
if lv__version__ != lv_changelog_version:
# It was not tagged yet and Changelog has no new records
# (they are composed by auto upon release)
assert_greater(lv__version__, lv_changelog_version)
assert_in('+', san__version__) # we have build suffix
else:
# all is good, tagged etc
assert_equal(lv_changelog_version, lv__version__)
assert_equal(changelog_version, san__version__)
return
raise AssertionError(
"No log line matching our regex found in %s" % CHANGELOG_filename
)
def test_external_versions_basic():
ev = ExternalVersions()
our_module = 'datalad'
assert_equal(ev.versions, {})
assert_equal(ev[our_module], __version__)
# and it could be compared
assert_greater_equal(ev[our_module], __version__)
# We got some odd failure in this test not long are after switching to versionner
# https://github.com/datalad/datalad/issues/5785. Verify that we do get expected
# data types
our_version = ev[our_module].version
assert isinstance(
our_version,
(str, list)), f"Got {our_version!r} of type {type(our_version)}"
assert_greater(ev[our_module], '0.1')
assert_equal(list(ev.keys()), [our_module])
assert_true(our_module in ev)
assert_false('unknown' in ev)
# all are LooseVersions now
assert_true(isinstance(ev[our_module], LooseVersion))
version_str = __version__
assert_equal(ev.dumps(), "Versions: %s=%s" % (our_module, version_str))
# For non-existing one we get None
assert_equal(ev['custom__nonexisting'], None)
# and nothing gets added to _versions for nonexisting
assert_equal(set(ev.versions.keys()), {our_module})
# but if it is a module without version, we get it set to UNKNOWN
assert_equal(ev['os'], ev.UNKNOWN)
# And get a record on that inside
assert_equal(ev.versions.get('os'), ev.UNKNOWN)
# And that thing is "True", i.e. present
assert (ev['os'])
# but not comparable with anything besides itself (was above)
assert_raises(TypeError, cmp, ev['os'], '0')
assert_raises(TypeError, assert_greater, ev['os'], '0')
return
def test_try_lock_informatively(tempfile=None):
lock = InterProcessLock(tempfile + '.lck')
lock_path = ensure_unicode(
lock.path) # can be bytes, complicates string formattingetc
t0 = time()
with try_lock_informatively(lock, purpose="happy life") as acquired:
assert_true(lock.acquired)
assert_true(acquired)
assert_greater(
2,
time() -
t0) # should not take any notable time, we cannot be blocking
"""
# InterProcessLock is not re-entrant so nesting should not be used, will result
# in exception on release
with try_lock_informatively(lock, timeouts=[dt, dt*2], proceed_unlocked=True) as acquired:
assert_true(lock.acquired) # due to outer cm
assert_true(acquired) # lock is reentrant apparently
"""
# Let's try in a completely different subprocess
runner = WitlessRunner(env=dict(
os.environ, DATALAD_LOG_LEVEL='info', DATALAD_LOG_TARGET='stderr'))
script1 = Path(tempfile + "-script1.py")
script1_fmt = f"""
from fasteners import InterProcessLock
from time import time
from datalad.support.locking import try_lock_informatively
lock = InterProcessLock({lock_path!r})
with try_lock_informatively(lock, timeouts=[0.05, 0.15], proceed_unlocked={{proceed_unlocked}}) as acquired:
print("Lock acquired=%s" % acquired)
"""
script1.write_text(script1_fmt.format(proceed_unlocked=True))
t0 = time()
res = runner.run([sys.executable, str(script1)],
protocol=StdOutErrCapture)
assert_in('Lock acquired=False', res['stdout'])
assert_in(f'Failed to acquire lock at {lock_path} in 0.05',
res['stderr'])
assert_in(f'Failed to acquire lock at {lock_path} in 0.15',
res['stderr'])
assert_in('proceed without locking', res['stderr'])
assert_greater(time() - t0, 0.19999) # should wait for at least 0.2
try:
import psutil
# PID does not correspond
assert_in('Check following process: PID=', res['stderr'])
assert_in(f'CWD={os.getcwd()} CMDLINE=', res['stderr'])
except ImportError:
pass # psutil was not installed, cannot get list of files
except AssertionError:
# we must have had the other one then
assert_in('failed to determine one', res['stderr'])
if not on_osx:
# so far we had only OSX reporting failing to get PIDs information
# but if it is something else -- re-raise original exception
raise
# in 2nd case, lets try without proceeding unlocked
script1.write_text(script1_fmt.format(proceed_unlocked=False))
t0 = time()
with assert_raises(CommandError) as cme:
runner.run([sys.executable, str(script1)],
protocol=StdOutErrCapture)
assert_in(f"Failed to acquire lock at {lock_path} in 2 attempts.",
str(cme.value))
assert_in(f"RuntimeError", str(cme.value))
assert_false(
cme.value.stdout) # nothing there since print should not happen
assert_in(f'Failed to acquire lock at {lock_path} in 0.05',
cme.value.stderr)
assert_in(f'Failed to acquire lock at {lock_path} in 0.15',
cme.value.stderr)
assert_greater(time() - t0, 0.19999) # should wait for at least 0.2
# now that we left context, should work out just fine
res = runner.run([sys.executable, str(script1)], protocol=StdOutErrCapture)
assert_in('Lock acquired=True', res['stdout'])
assert_not_in(f'Failed to acquire lock', res['stderr'])
assert_not_in('PID', res['stderr'])
def test_wtf(topdir=None):
path = opj(topdir, OBSCURE_FILENAME)
# smoke test for now
with swallow_outputs() as cmo:
wtf(dataset=path, on_failure="ignore")
assert_not_in('## dataset', cmo.out)
assert_in('## configuration', cmo.out)
# Those sections get sensored out by default now
assert_not_in('user.name: ', cmo.out)
with chpwd(path):
with swallow_outputs() as cmo:
wtf()
assert_not_in('## dataset', cmo.out)
assert_in('## configuration', cmo.out)
# now with a dataset
ds = create(path)
with swallow_outputs() as cmo:
wtf(dataset=ds.path)
assert_in('## configuration', cmo.out)
assert_in('## dataset', cmo.out)
assert_in(u'path: {}'.format(ds.path), ensure_unicode(cmo.out))
assert_in('branches', cmo.out)
assert_in(DEFAULT_BRANCH + '@', cmo.out)
assert_in('git-annex@', cmo.out)
# and if we run with all sensitive
for sensitive in ('some', True):
with swallow_outputs() as cmo:
wtf(dataset=ds.path, sensitive=sensitive)
# we fake those for tests anyways, but we do show cfg in this mode
# and explicitly not showing them
assert_in('user.name: %s' % _HIDDEN, cmo.out)
with swallow_outputs() as cmo:
wtf(dataset=ds.path, sensitive='all')
assert_not_in(_HIDDEN, cmo.out) # all is shown
assert_in('user.name: ', cmo.out)
# Sections selection
#
# If we ask for no sections and there is no dataset
with chpwd(path):
with swallow_outputs() as cmo:
wtf(sections=[])
assert_not_in('## dataset', cmo.out)
for s in SECTION_CALLABLES:
assert_not_in('## %s' % s.lower(), cmo.out.lower())
# ask for a selected set
secs = ['git-annex', 'configuration']
with chpwd(path):
with swallow_outputs() as cmo:
wtf(sections=secs)
for s in SECTION_CALLABLES:
(assert_in if s in secs else assert_not_in)('## %s' %
s.lower(),
cmo.out.lower())
# order should match our desired one, not alphabetical
# but because of https://github.com/datalad/datalad/issues/3915
# alphanum is now desired
assert cmo.out.index('## git-annex') > cmo.out.index(
'## configuration')
# not achievable from cmdline is to pass an empty list of sections.
with chpwd(path):
with swallow_outputs() as cmo:
wtf(sections=[])
eq_(cmo.out.rstrip(), '# WTF')
# and we could decorate it nicely for embedding e.g. into github issues
with swallow_outputs() as cmo:
wtf(sections=['dependencies'], decor='html_details')
ok_startswith(cmo.out,
'<details><summary>DataLad %s WTF' % __version__)
assert_in('## dependencies', cmo.out)
# short flavor
with swallow_outputs() as cmo:
wtf(flavor='short')
assert_in("- datalad: version=%s" % __version__, cmo.out)
assert_in("- dependencies: ", cmo.out)
eq_(len(cmo.out.splitlines()),
4) # #WTF, datalad, dependencies, trailing new line
with swallow_outputs() as cmo:
wtf(flavor='short', sections='*')
assert_greater(len(cmo.out.splitlines()), 10) # many more
# check that wtf of an unavailable section yields impossible result (#6712)
res = wtf(sections=['murkie'], on_failure='ignore')
eq_(res[0]["status"], "impossible")
# should result only in '# WTF'
skip_if_no_module('pyperclip')
# verify that it works correctly in the env/platform
import pyperclip
with swallow_outputs() as cmo:
try:
pyperclip.copy("xxx")
pyperclip_works = pyperclip.paste().strip() == "xxx"
wtf(dataset=ds.path, clipboard=True)
except (AttributeError, pyperclip.PyperclipException) as exc:
# AttributeError could come from pyperclip if no DISPLAY
raise SkipTest(str(exc))
assert_in("WTF information of length", cmo.out)
assert_not_in('user.name', cmo.out)
if not pyperclip_works:
# Some times does not throw but just fails to work
raise SkipTest(
"Pyperclip seems to be not functioning here correctly")
assert_not_in('user.name', pyperclip.paste())
assert_in(_HIDDEN, pyperclip.paste()) # by default no sensitive info
assert_in("cmd:annex:", pyperclip.paste()) # but the content is there
def _test_list_tuple(thing):
version = ExternalVersions._deduce_version(thing)
assert_greater(version, '0.0.1')
assert_greater('0.2', version)
assert_equal('0.1', version)
assert_equal(version, '0.1')
def test_gracefull_death():
def assert_provides_and_raises(pc, exception, target=None):
"""Helper to get all results before exception is raised"""
results = []
with assert_raises(exception):
for r in pc:
results.append(r)
# results should be sorted since we do not guarantee order
results = sorted(results)
if target is not None:
assert_equal(results, target)
return results
def interrupted_producer():
yield 1
raise ValueError()
def consumer(i):
sleep(0.001)
yield i
assert_provides_and_raises(
ProducerConsumer(interrupted_producer(), consumer, jobs=3), ValueError,
[1])
def faulty_consumer(i):
sleep(0.001)
if i == 1:
raise ValueError()
return i
# so we do not get failed, but other parallel ones finish their job
results = assert_provides_and_raises(
ProducerConsumer(range(1000), faulty_consumer, jobs=5), ValueError)
# and analysis of futures to raise an exception can take some time etc, so
# we could get more, but for sure we should not get all 999 and not even a 100
if info_log_level:
assert_greater(100, len(results))
assert_equal(results[:4], [0, 2, 3, 4])
def producer():
for i in range(10):
sleep(0.0003)
yield i
raise ValueError()
# by default we do not stop upon producer failing
assert_provides_and_raises(ProducerConsumer(producer(), consumer, jobs=2),
ValueError, list(range(10)))
# if producer produces more than we can as quickly consume but then fails
# ATM we do not proceed to consume other items, but fail when we finish
# consuming until the time point when producer has failed
# by default we do not stop upon producer failing
results = assert_provides_and_raises(
ProducerConsumer(producer(),
consumer,
reraise_immediately=True,
jobs=2), ValueError)
# we will get some results, seems around 4 and they should be "sequential"
assert_equal(results, list(range(len(results))))
assert_greater_equal(len(results), 2)
# This test relies too much on threads scheduling to not hog up on handling
# consumers, but if it happens so - they might actually consume all results
# before producer decides to finally raise an exception. As such it remains
# flaky and thus not ran, but could be useful to test locally while
# changing that logic.
#
# if info_log_level and not (on_windows or on_osx):
# # consumers should not be able to consume all produced items.
# # production of 10 should take 3 unites, while consumers 10/2 (jobs)
# # 5 units, so some should not have a chance.
# assert_greater_equal(8, len(results))
# Simulate situation close to what we have when outside code consumes
# some yielded results and then "looses interest" (on_failure="error").
# In this case we should still exit gracefully (no GeneratorExit warnings),
# not over-produce, and also do not kill already running consumers
consumed = []
def inner():
def consumer(i):
sleep(0.01)
consumed.append(i)
return i
pc = iter(ProducerConsumer(range(1000), consumer, jobs=2))
yield next(pc)
yield next(pc)
assert_equal(sorted(inner()), [0, 1])
consumed = sorted(consumed)
assert_equal(consumed, list(range(len(consumed))))
assert_greater_equal(len(consumed),
4) # we should wait for that 2nd batch to finish
if info_log_level:
assert_greater_equal(20, len(consumed))