def test_with_result_progress_generator():
# Tests ability for the decorator to decorate a regular function
# or a generator function (then it returns a generator function)
@with_result_progress
def func(l):
return l
generated = []
@with_result_progress
def gen(l):
for i in l:
generated.append(i)
yield i
recs = [{'status': 'ok', 'unrelated': i} for i in range(2)]
# still works for a func and returns provided list
ok_(not inspect.isgeneratorfunction(func))
assert_equal(func(recs), recs)
# generator should still yield and next iteration should only happen
# when requested
ok_(inspect.isgeneratorfunction(gen))
g = gen(recs)
ok_generator(g)
assert_equal(generated, []) # nothing yet
assert_equal(next(g), recs[0])
assert_equal(generated, recs[:1])
assert_equal(next(g), recs[1])
assert_equal(generated, recs)
# just to make sure all good to redo
assert_equal(list(gen(recs)), recs)
def test_ExtractedArchive(path):
archive = op.join(path, fn_archive_obscure_ext)
earchive = ExtractedArchive(archive)
assert_false(op.exists(earchive.path))
# no longer the case -- just using hash for now
# assert_in(os.path.basename(archive), earchive.path)
fpath = op.join(fn_archive_obscure, # lead directory
fn_in_archive_obscure)
extracted = earchive.get_extracted_filename(fpath)
eq_(extracted, op.join(earchive.path, fpath))
assert_false(op.exists(extracted)) # not yet
extracted_ = earchive.get_extracted_file(fpath)
eq_(extracted, extracted_)
assert_true(op.exists(extracted)) # now it should
extracted_files = earchive.get_extracted_files()
ok_generator(extracted_files)
eq_(sorted(extracted_files),
sorted([
# ['bbc/3.txt', 'bbc/abc']
op.join(fn_archive_obscure, fn_in_archive_obscure),
op.join(fn_archive_obscure, '3.txt')
]))
earchive.clean()
if not os.environ.get('DATALAD_TESTS_TEMP_KEEP'):
assert_false(op.exists(earchive.path))
def test_ok_generator():
def func(a, b=1):
return a+b
def gen(a, b=1): # pragma: no cover
yield a+b
# not sure how to determine if xrange is a generator
assert_raises(AssertionError, ok_generator, range(2))
assert_raises(AssertionError, ok_generator, gen)
ok_generator(gen(1))
assert_raises(AssertionError, ok_generator, func)
assert_raises(AssertionError, ok_generator, func(1))
def test_func_to_node():
int_node = func_to_node(int) # node which requires nothing and nothing of output is used
assert int_node.__doc__
in_dict = {'in': 1}
ok_generator(int_node(in_dict))
# xrange is not considered to be a generator
def xrange_(n, offset=0):
for x in range(offset, offset + n):
yield x
xrange_node = func_to_node(xrange_, data_args='in', outputs='out')
assert_in('assigned to out', xrange_node.__doc__)
assert_false('Additional keyword arguments' in xrange_node.__doc__)
range_node_gen = xrange_node(in_dict)
ok_generator(range_node_gen)
assert_equal(list(range_node_gen), [{'in': 1, 'out': 0}])
# with additional kwargs
xrange_node = func_to_node(xrange_, data_args='in', outputs='out', kwargs={'offset': 10})
assert_in('assigned to out', xrange_node.__doc__)
assert_in('Additional keyword arguments', xrange_node.__doc__)
range_node_gen = xrange_node(in_dict)
ok_generator(range_node_gen)
assert_equal(list(range_node_gen), [{'in': 1, 'out': 10}])
# testing func_node
data = {'offset': 5, 'in': 1}
xrange_node = func_to_node(xrange_, data_args='in', data_kwargs=['offset'], outputs='out')
assert_in('assigned to out', xrange_node.__doc__)
assert_false('Additional keyword arguments' in xrange_node.__doc__)
gen = xrange_node(data)
ok_generator(gen)
assert_equal(list(gen), [{'offset': 5, 'out': 5, 'in': 1}])
# with multiple outputs
def split_(s, num):
yield s.split('/', num)
data = {'num': 3, 'in': 'datalad/crawler/nodes'}
split_node = func_to_node(split_, data_args='in', data_kwargs=['num'], outputs=['a', 'b', 'c'])
assert_in('assigned to a, b, c', split_node.__doc__)
assert_false('Additional keyword arguments' in split_node.__doc__)
split_node_gen = split_node(data)
assert_equal(list(split_node_gen), [{'a': 'datalad', 'c': 'nodes', 'b': 'crawler', 'num': 3, 'in': 'datalad/crawler/nodes'}])
def test_with_progress_generator():
# Well, we could also pass an iterable directly now and display
# progress iterative over it
g = with_progress(range(3))
ok_generator(g)
assert_equal(list(g), list(range(3)))
def test_func_to_node():
int_node = func_to_node(
int) # node which requires nothing and nothing of output is used
assert int_node.__doc__
in_dict = {'in': 1}
ok_generator(int_node(in_dict))
# xrange is not considered to be a generator
def xrange_(n, offset=0):
for x in range(offset, offset + n):
yield x
xrange_node = func_to_node(xrange_, data_args='in', outputs='out')
assert_in('assigned to out', xrange_node.__doc__)
assert_false('Additional keyword arguments' in xrange_node.__doc__)
range_node_gen = xrange_node(in_dict)
ok_generator(range_node_gen)
assert_equal(list(range_node_gen), [{'in': 1, 'out': 0}])
# with additional kwargs
xrange_node = func_to_node(xrange_,
data_args='in',
outputs='out',
kwargs={'offset': 10})
assert_in('assigned to out', xrange_node.__doc__)
assert_in('Additional keyword arguments', xrange_node.__doc__)
range_node_gen = xrange_node(in_dict)
ok_generator(range_node_gen)
assert_equal(list(range_node_gen), [{'in': 1, 'out': 10}])
# testing func_node
data = {'offset': 5, 'in': 1}
xrange_node = func_to_node(xrange_,
data_args='in',
data_kwargs=['offset'],
outputs='out')
assert_in('assigned to out', xrange_node.__doc__)
assert_false('Additional keyword arguments' in xrange_node.__doc__)
gen = xrange_node(data)
ok_generator(gen)
assert_equal(list(gen), [{'offset': 5, 'out': 5, 'in': 1}])
# with multiple outputs
def split_(s, num):
yield s.split('/', num)
data = {'num': 3, 'in': 'datalad/crawler/nodes'}
split_node = func_to_node(split_,
data_args='in',
data_kwargs=['num'],
outputs=['a', 'b', 'c'])
assert_in('assigned to a, b, c', split_node.__doc__)
assert_false('Additional keyword arguments' in split_node.__doc__)
split_node_gen = split_node(data)
assert_equal(list(split_node_gen), [{
'a': 'datalad',
'c': 'nodes',
'b': 'crawler',
'num': 3,
'in': 'datalad/crawler/nodes'
}])
