def test_with_result_progress_generator(): # Tests ability for the decorator to decorate a regular function # or a generator function (then it returns a generator function) @with_result_progress def func(l): return l generated = [] @with_result_progress def gen(l): for i in l: generated.append(i) yield i recs = [{'status': 'ok', 'unrelated': i} for i in range(2)] # still works for a func and returns provided list ok_(not inspect.isgeneratorfunction(func)) assert_equal(func(recs), recs) # generator should still yield and next iteration should only happen # when requested ok_(inspect.isgeneratorfunction(gen)) g = gen(recs) ok_generator(g) assert_equal(generated, []) # nothing yet assert_equal(next(g), recs[0]) assert_equal(generated, recs[:1]) assert_equal(next(g), recs[1]) assert_equal(generated, recs) # just to make sure all good to redo assert_equal(list(gen(recs)), recs)
def test_ExtractedArchive(path): archive = op.join(path, fn_archive_obscure_ext) earchive = ExtractedArchive(archive) assert_false(op.exists(earchive.path)) # no longer the case -- just using hash for now # assert_in(os.path.basename(archive), earchive.path) fpath = op.join(fn_archive_obscure, # lead directory fn_in_archive_obscure) extracted = earchive.get_extracted_filename(fpath) eq_(extracted, op.join(earchive.path, fpath)) assert_false(op.exists(extracted)) # not yet extracted_ = earchive.get_extracted_file(fpath) eq_(extracted, extracted_) assert_true(op.exists(extracted)) # now it should extracted_files = earchive.get_extracted_files() ok_generator(extracted_files) eq_(sorted(extracted_files), sorted([ # ['bbc/3.txt', 'bbc/abc'] op.join(fn_archive_obscure, fn_in_archive_obscure), op.join(fn_archive_obscure, '3.txt') ])) earchive.clean() if not os.environ.get('DATALAD_TESTS_TEMP_KEEP'): assert_false(op.exists(earchive.path))
def test_ok_generator(): def func(a, b=1): return a+b def gen(a, b=1): # pragma: no cover yield a+b # not sure how to determine if xrange is a generator assert_raises(AssertionError, ok_generator, range(2)) assert_raises(AssertionError, ok_generator, gen) ok_generator(gen(1)) assert_raises(AssertionError, ok_generator, func) assert_raises(AssertionError, ok_generator, func(1))
def test_func_to_node(): int_node = func_to_node(int) # node which requires nothing and nothing of output is used assert int_node.__doc__ in_dict = {'in': 1} ok_generator(int_node(in_dict)) # xrange is not considered to be a generator def xrange_(n, offset=0): for x in range(offset, offset + n): yield x xrange_node = func_to_node(xrange_, data_args='in', outputs='out') assert_in('assigned to out', xrange_node.__doc__) assert_false('Additional keyword arguments' in xrange_node.__doc__) range_node_gen = xrange_node(in_dict) ok_generator(range_node_gen) assert_equal(list(range_node_gen), [{'in': 1, 'out': 0}]) # with additional kwargs xrange_node = func_to_node(xrange_, data_args='in', outputs='out', kwargs={'offset': 10}) assert_in('assigned to out', xrange_node.__doc__) assert_in('Additional keyword arguments', xrange_node.__doc__) range_node_gen = xrange_node(in_dict) ok_generator(range_node_gen) assert_equal(list(range_node_gen), [{'in': 1, 'out': 10}]) # testing func_node data = {'offset': 5, 'in': 1} xrange_node = func_to_node(xrange_, data_args='in', data_kwargs=['offset'], outputs='out') assert_in('assigned to out', xrange_node.__doc__) assert_false('Additional keyword arguments' in xrange_node.__doc__) gen = xrange_node(data) ok_generator(gen) assert_equal(list(gen), [{'offset': 5, 'out': 5, 'in': 1}]) # with multiple outputs def split_(s, num): yield s.split('/', num) data = {'num': 3, 'in': 'datalad/crawler/nodes'} split_node = func_to_node(split_, data_args='in', data_kwargs=['num'], outputs=['a', 'b', 'c']) assert_in('assigned to a, b, c', split_node.__doc__) assert_false('Additional keyword arguments' in split_node.__doc__) split_node_gen = split_node(data) assert_equal(list(split_node_gen), [{'a': 'datalad', 'c': 'nodes', 'b': 'crawler', 'num': 3, 'in': 'datalad/crawler/nodes'}])
def test_with_progress_generator(): # Well, we could also pass an iterable directly now and display # progress iterative over it g = with_progress(range(3)) ok_generator(g) assert_equal(list(g), list(range(3)))
def test_func_to_node(): int_node = func_to_node( int) # node which requires nothing and nothing of output is used assert int_node.__doc__ in_dict = {'in': 1} ok_generator(int_node(in_dict)) # xrange is not considered to be a generator def xrange_(n, offset=0): for x in range(offset, offset + n): yield x xrange_node = func_to_node(xrange_, data_args='in', outputs='out') assert_in('assigned to out', xrange_node.__doc__) assert_false('Additional keyword arguments' in xrange_node.__doc__) range_node_gen = xrange_node(in_dict) ok_generator(range_node_gen) assert_equal(list(range_node_gen), [{'in': 1, 'out': 0}]) # with additional kwargs xrange_node = func_to_node(xrange_, data_args='in', outputs='out', kwargs={'offset': 10}) assert_in('assigned to out', xrange_node.__doc__) assert_in('Additional keyword arguments', xrange_node.__doc__) range_node_gen = xrange_node(in_dict) ok_generator(range_node_gen) assert_equal(list(range_node_gen), [{'in': 1, 'out': 10}]) # testing func_node data = {'offset': 5, 'in': 1} xrange_node = func_to_node(xrange_, data_args='in', data_kwargs=['offset'], outputs='out') assert_in('assigned to out', xrange_node.__doc__) assert_false('Additional keyword arguments' in xrange_node.__doc__) gen = xrange_node(data) ok_generator(gen) assert_equal(list(gen), [{'offset': 5, 'out': 5, 'in': 1}]) # with multiple outputs def split_(s, num): yield s.split('/', num) data = {'num': 3, 'in': 'datalad/crawler/nodes'} split_node = func_to_node(split_, data_args='in', data_kwargs=['num'], outputs=['a', 'b', 'c']) assert_in('assigned to a, b, c', split_node.__doc__) assert_false('Additional keyword arguments' in split_node.__doc__) split_node_gen = split_node(data) assert_equal(list(split_node_gen), [{ 'a': 'datalad', 'c': 'nodes', 'b': 'crawler', 'num': 3, 'in': 'datalad/crawler/nodes' }])