def test_sort_5(): table = (('foo', 'bar'), (2.3, 2), (1.2, 9), (2.3, 6), (3.2, 1), (1.2, 10)) expectation = (('foo', 'bar'), (1.2, 9), (1.2, 10), (2.3, 2), (2.3, 6), (3.2, 1)) # can use either field names or indices (from 1) to specify sort key result = sort(table, key=('foo', 'bar')) ieq(expectation, result) result = sort(table, key=(0, 1)) ieq(expectation, result) result = sort(table, key=('foo', 1)) ieq(expectation, result) result = sort(table, key=(0, 'bar')) ieq(expectation, result)
def __init__(self, left, right, lkey, rkey, presorted=False, missing=None, buffersize=None, tempdir=None, cache=True, lprefix=None, rprefix=None): if presorted: self.left = left self.right = right else: self.left = sort(left, lkey, buffersize=buffersize, tempdir=tempdir, cache=cache) self.right = sort(right, rkey, buffersize=buffersize, tempdir=tempdir, cache=cache) self.lkey = lkey self.rkey = rkey self.missing = missing self.lprefix = lprefix self.rprefix = rprefix
def test_sort_2(): table = (('foo', 'bar'), ('C', '2'), ('A', '9'), ('A', '6'), ('F', '1'), ('D', '10')) result = sort(table, key=('foo', 'bar')) expectation = (('foo', 'bar'), ('A', '6'), ('A', '9'), ('C', '2'), ('D', '10'), ('F', '1')) ieq(expectation, result) result = sort(table) # default is lexical sort expectation = (('foo', 'bar'), ('A', '6'), ('A', '9'), ('C', '2'), ('D', '10'), ('F', '1')) ieq(expectation, result)
def test_mergesort_3(): table1 = (('foo', 'bar'), ('A', 9), ('C', 2), ('D', 10), ('A', 6), ('F', 1)) table2 = (('foo', 'baz'), ('B', 3), ('D', 10), ('A', 10), ('F', 4)) # should be same as concatenate then sort (but more efficient, esp. when # presorted) expect = sort(cat(table1, table2), key='foo', reverse=True) actual = mergesort(table1, table2, key='foo', reverse=True) ieq(expect, actual) ieq(expect, actual) actual = mergesort(sort(table1, key='foo', reverse=True), sort(table2, key='foo', reverse=True), key='foo', reverse=True, presorted=True) ieq(expect, actual) ieq(expect, actual)
def test_fromhdf5sorted(): f = NamedTemporaryFile() # set up a new hdf5 table to work with h5file = tables.open_file(f.name, mode='w', title='Test file') h5file.create_group('/', 'testgroup', 'Test Group') h5table = h5file.create_table('/testgroup', 'testtable', FooBar, 'Test Table') # load some data into the table table1 = (('foo', 'bar'), (3, b'asdfgh'), (2, b'qwerty'), (1, b'zxcvbn')) for row in table1[1:]: for i, f in enumerate(table1[0]): h5table.row[f] = row[i] h5table.row.append() h5table.cols.foo.create_csindex() h5file.flush() # verify we can get the data back out table2 = fromhdf5sorted(h5table, sortby='foo') ieq(sort(table1, 'foo'), table2) ieq(sort(table1, 'foo'), table2) # clean up h5file.close()
def __init__(self, a, b, presorted=False, buffersize=None, tempdir=None, cache=True): if presorted: self.a = a self.b = b else: self.a = sort(a, buffersize=buffersize, tempdir=tempdir, cache=cache) self.b = sort(b, buffersize=buffersize, tempdir=tempdir, cache=cache)
def test_fromhdf5sorted(): f = NamedTemporaryFile() # set up a new hdf5 table to work with h5file = tables.open_file(f.name, mode='w', title='Test file') h5file.create_group('/', 'testgroup', 'Test Group') h5table = h5file.create_table('/testgroup', 'testtable', FooBar, 'Test Table') # load some data into the table table1 = (('foo', 'bar'), (3, b'asdfgh'), (2, b'qwerty'), (1, b'zxcvbn')) for row in table1[1:]: for i, f in enumerate(table1[0]): h5table.row[f] = row[i] h5table.row.append() h5table.cols.foo.create_csindex() h5file.flush() # verify we can get the data back out table2 = fromhdf5sorted(h5table, sortby='foo') ieq(sort(table1, 'foo'), table2) ieq(sort(table1, 'foo'), table2) # clean up h5file.close()
def __init__(self, left, right, lkey, rkey, presorted=False, buffersize=None, tempdir=None, cache=True): if presorted: self.left = left self.right = right else: self.left = sort(left, lkey, buffersize=buffersize, tempdir=tempdir, cache=cache) self.right = sort(right, rkey, buffersize=buffersize, tempdir=tempdir, cache=cache) self.lkey = lkey self.rkey = rkey
def test_sort_none(): table = (('foo', 'bar'), ('C', 2), ('A', 9), ('A', None), ('F', 1), ('D', 10)) result = sort(table, 'bar') expectation = (('foo', 'bar'), ('A', None), ('F', 1), ('C', 2), ('A', 9), ('D', 10)) ieq(expectation, result) dt = datetime.now().replace table = (('foo', 'bar'), ('C', dt(hour=5)), ('A', dt(hour=1)), ('A', None), ('F', dt(hour=9)), ('D', dt(hour=17))) result = sort(table, 'bar') expectation = (('foo', 'bar'), ('A', None), ('A', dt(hour=1)), ('C', dt(hour=5)), ('F', dt(hour=9)), ('D', dt(hour=17))) ieq(expectation, result)
def diff(a, b, presorted=False, buffersize=None, tempdir=None, cache=True): """ Find the difference between rows in two tables. Returns a pair of tables. E.g.:: >>> import petl as etl >>> a = [['foo', 'bar', 'baz'], ... ['A', 1, True], ... ['C', 7, False], ... ['B', 2, False], ... ['C', 9, True]] >>> b = [['x', 'y', 'z'], ... ['B', 2, False], ... ['A', 9, False], ... ['B', 3, True], ... ['C', 9, True]] >>> added, subtracted = etl.diff(a, b) >>> # rows in b not in a ... added +-----+---+-------+ | x | y | z | +=====+===+=======+ | 'A' | 9 | False | +-----+---+-------+ | 'B' | 3 | True | +-----+---+-------+ >>> # rows in a not in b ... subtracted +-----+-----+-------+ | foo | bar | baz | +=====+=====+=======+ | 'A' | 1 | True | +-----+-----+-------+ | 'C' | 7 | False | +-----+-----+-------+ Convenient shorthand for ``(complement(b, a), complement(a, b))``. See also :func:`petl.transform.setops.complement`. If `presorted` is True, it is assumed that the data are already sorted by the given key, and the `buffersize`, `tempdir` and `cache` arguments are ignored. Otherwise, the data are sorted, see also the discussion of the `buffersize`, `tempdir` and `cache` arguments under the :func:`petl.transform.sorts.sort` function. """ if not presorted: a = sort(a) b = sort(b) added = complement(b, a, presorted=True, buffersize=buffersize, tempdir=tempdir, cache=cache) subtracted = complement(a, b, presorted=True, buffersize=buffersize, tempdir=tempdir, cache=cache) return added, subtracted
def test_sort_buffered_tempdir(): table = (("foo", "bar"), ("C", 2), ("A", 9), ("A", 6), ("F", 1), ("D", 10)) # test sort forwards expectation = (("foo", "bar"), ("F", 1), ("C", 2), ("A", 6), ("A", 9), ("D", 10)) result = sort(table, "bar") ieq(expectation, result) result = sort(table, "bar", buffersize=2, tempdir="/tmp") ieq(expectation, result)
def __init__(self, a, b, presorted=False, buffersize=None, tempdir=None, cache=True): if presorted: self.a = a self.b = b else: self.a = sort(a, buffersize=buffersize, tempdir=tempdir, cache=cache) self.b = sort(b, buffersize=buffersize, tempdir=tempdir, cache=cache)
def test_sort_2(): table = (("foo", "bar"), ("C", "2"), ("A", "9"), ("A", "6"), ("F", "1"), ("D", "10")) result = sort(table, key=("foo", "bar")) expectation = (("foo", "bar"), ("A", "6"), ("A", "9"), ("C", "2"), ("D", "10"), ("F", "1")) ieq(expectation, result) result = sort(table) # default is lexical sort expectation = (("foo", "bar"), ("A", "6"), ("A", "9"), ("C", "2"), ("D", "10"), ("F", "1")) ieq(expectation, result)
def test_sort_buffered_tempdir(): table = (('foo', 'bar'), ('C', 2), ('A', 9), ('A', 6), ('F', 1), ('D', 10)) # test sort forwards expectation = (('foo', 'bar'), ('F', 1), ('C', 2), ('A', 6), ('A', 9), ('D', 10)) result = sort(table, 'bar') ieq(expectation, result) result = sort(table, 'bar', buffersize=2, tempdir='/tmp') ieq(expectation, result)
def __init__(self, left, right, lkey, rkey, presorted=False, buffersize=None, tempdir=None, cache=True): if presorted: self.left = left self.right = right else: self.left = sort(left, lkey, buffersize=buffersize, tempdir=tempdir, cache=cache) self.right = sort(right, rkey, buffersize=buffersize, tempdir=tempdir, cache=cache) self.lkey = lkey self.rkey = rkey
def test_sort_buffered_tempdir(): table = (("foo", "bar"), ("C", 2), ("A", 9), ("A", 6), ("F", 1), ("D", 10)) # test sort forwards expectation = (("foo", "bar"), ("F", 1), ("C", 2), ("A", 6), ("A", 9), ("D", 10)) result = sort(table, "bar") ieq(expectation, result) tempdir = "tmp" if not os.path.exists(tempdir): os.mkdir(tempdir) result = sort(table, "bar", buffersize=2, tempdir=tempdir) ieq(expectation, result)
def test_sort_buffered_tempdir(): table = (('foo', 'bar'), ('C', 2), ('A', 9), ('A', 6), ('F', 1), ('D', 10)) # test sort forwards expectation = (('foo', 'bar'), ('F', 1), ('C', 2), ('A', 6), ('A', 9), ('D', 10)) result = sort(table, 'bar') ieq(expectation, result) tempdir = 'tmp' if not os.path.exists(tempdir): os.mkdir(tempdir) result = sort(table, 'bar', buffersize=2, tempdir=tempdir) ieq(expectation, result)
def test_sort_2(): table = (('foo', 'bar'), ('C', '2'), ('A', '9'), ('A', '6'), ('F', '1'), ('D', '10')) result = sort(table, key=('foo', 'bar')) expectation = (('foo', 'bar'), ('A', '6'), ('A', '9'), ('C', '2'), ('D', '10'), ('F', '1')) ieq(expectation, result) result = sort(table) # default is lexical sort expectation = (('foo', 'bar'), ('A', '6'), ('A', '9'), ('C', '2'), ('D', '10'), ('F', '1')) ieq(expectation, result)
def test_sort_5(): table = (('foo', 'bar'), (2.3, 2), (1.2, 9), (2.3, 6), (3.2, 1), (1.2, 10)) expectation = (('foo', 'bar'), (1.2, 9), (1.2, 10), (2.3, 2), (2.3, 6), (3.2, 1)) # can use either field names or indices (from 1) to specify sort key result = sort(table, key=('foo', 'bar')) ieq(expectation, result) result = sort(table, key=(0, 1)) ieq(expectation, result) result = sort(table, key=('foo', 1)) ieq(expectation, result) result = sort(table, key=(0, 'bar')) ieq(expectation, result)
def test_sort_3(): table = (("foo", "bar"), ("C", "2"), ("A", "9"), ("A", "6"), ("F", "1"), ("D", "10")) result = sort(table, "bar") expectation = (("foo", "bar"), ("F", "1"), ("D", "10"), ("C", "2"), ("A", "6"), ("A", "9")) ieq(expectation, result)
def __init__(self, table, key=None, count=None, presorted=False, buffersize=None, tempdir=None, cache=True): if presorted: self.table = table else: self.table = sort(table, key=key, buffersize=buffersize, tempdir=tempdir, cache=cache) self.key = key self.count = count
def test_sort_4(): table = (("foo", "bar"), ("C", 2), ("A", 9), ("A", 6), ("F", 1), ("D", 10)) result = sort(table, "bar") expectation = (("foo", "bar"), ("F", 1), ("C", 2), ("A", 6), ("A", 9), ("D", 10)) ieq(expectation, result)
def __init__(self, source, key, aggregation=None, presorted=False, buffersize=None, tempdir=None, cache=True): if presorted: self.source = source else: self.source = sort(source, key, buffersize=buffersize, tempdir=tempdir, cache=cache) self.key = key if aggregation is None: self.aggregation = OrderedDict() elif isinstance(aggregation, (list, tuple)): self.aggregation = OrderedDict() for t in aggregation: self.aggregation[t[0]] = t[1:] elif isinstance(aggregation, dict): self.aggregation = aggregation else: raise ArgumentError( 'expected aggregation is None, list, tuple or dict, found %r' % aggregation)
def test_sort_missing_cell_numeric(): """ Sorting table with missing values raises IndexError #385 """ tbl = (('a', 'b'), ('4', ), ('2', '1'), ('1', )) expect = (('a', 'b'), ('1', ), ('2', '1'), ('4', )) tbl_sorted = sort(tbl) ieq(expect, tbl_sorted)
def test_sort_buffered_independent(): table = (('foo', 'bar'), ('C', 2), ('A', 9), ('A', 6), ('F', 1), ('D', 10)) expectation = (('foo', 'bar'), ('F', 1), ('C', 2), ('A', 6), ('A', 9), ('D', 10)) result = sort(table, 'bar', buffersize=4) nrows(result) # cause data to be cached # check that two row iterators are independent, i.e., consuming rows # from one does not affect the other it1 = iter(result) it2 = iter(result) eq_(expectation[0], it1.next()) eq_(expectation[1], it1.next()) eq_(expectation[0], it2.next()) eq_(expectation[1], it2.next()) eq_(expectation[2], it2.next()) eq_(expectation[2], it1.next())
def test_sort_missing_cell_text(): """ Sorting table with missing values raises IndexError #385 """ tbl = (('a', 'b', 'c'), ('C', ), ('A', '4', '5')) expect = (('a', 'b', 'c'), ('A', '4', '5'), ('C', )) tbl_sorted = sort(tbl) ieq(expect, tbl_sorted)
def __init__(self, left, right, lkey, rkey, presorted=False, missing=None, buffersize=None, tempdir=None, cache=True, lprefix=None, rprefix=None): if presorted: self.left = left self.right = right else: self.left = sort(left, lkey, buffersize=buffersize, tempdir=tempdir, cache=cache) self.right = sort(right, rkey, buffersize=buffersize, tempdir=tempdir, cache=cache) self.lkey = lkey self.rkey = rkey self.missing = missing self.lprefix = lprefix self.rprefix = rprefix
def test_sort_6(): table = (("foo", "bar"), (2.3, 2), (1.2, 9), (2.3, 6), (3.2, 1), (1.2, 10)) expectation = (("foo", "bar"), (3.2, 1), (2.3, 6), (2.3, 2), (1.2, 10), (1.2, 9)) result = sort(table, key=("foo", "bar"), reverse=True) ieq(expectation, result)
def __init__(self, source, key=None, presorted=False, buffersize=None, tempdir=None, cache=True): if presorted: self.source = source else: self.source = sort(source, key, buffersize=buffersize, tempdir=tempdir, cache=cache) self.key = key
def __init__(self, source, key=None, presorted=False, buffersize=None, tempdir=None, cache=True): if presorted: self.source = source else: self.source = sort(source, key, buffersize=buffersize, tempdir=tempdir, cache=cache) self.key = key # TODO property
def test_sort_4(): table = (('foo', 'bar'), ('C', 2), ('A', 9), ('A', 6), ('F', 1), ('D', 10)) result = sort(table, 'bar') expectation = (('foo', 'bar'), ('F', 1), ('C', 2), ('A', 6), ('A', 9), ('D', 10)) ieq(expectation, result)
def test_mergesort_2(): table1 = (("foo", "bar"), ("A", 9), ("C", 2), ("D", 10), ("A", 6), ("F", 1)) table2 = (("foo", "baz"), ("B", 3), ("D", 10), ("A", 10), ("F", 4)) # should be same as concatenate then sort (but more efficient, esp. when # presorted) expect = sort(cat(table1, table2), key="foo") actual = mergesort(table1, table2, key="foo") ieq(expect, actual) ieq(expect, actual) actual = mergesort(sort(table1, key="foo"), sort(table2, key="foo"), key="foo", presorted=True) ieq(expect, actual) ieq(expect, actual)
def __init__(self, table, key=None, count=None, presorted=False, buffersize=None, tempdir=None, cache=True): if presorted: self.table = table else: self.table = sort(table, key=key, buffersize=buffersize, tempdir=tempdir, cache=cache) self.key = key self.count = count
def test_sort_6(): table = (('foo', 'bar'), (2.3, 2), (1.2, 9), (2.3, 6), (3.2, 1), (1.2, 10)) expectation = (('foo', 'bar'), (3.2, 1), (2.3, 6), (2.3, 2), (1.2, 10), (1.2, 9)) result = sort(table, key=('foo', 'bar'), reverse=True) ieq(expectation, result)
def test_mergesort_1(): table1 = (('foo', 'bar'), ('A', 6), ('C', 2), ('D', 10), ('A', 9), ('F', 1)) table2 = (('foo', 'bar'), ('B', 3), ('D', 10), ('A', 10), ('F', 4)) # should be same as concatenate then sort (but more efficient, esp. when # presorted) expect = sort(cat(table1, table2)) actual = mergesort(table1, table2) ieq(expect, actual) ieq(expect, actual) actual = mergesort(sort(table1), sort(table2), presorted=True) ieq(expect, actual) ieq(expect, actual)
def __init__(self, table, key, aggregation=list, value=None, presorted=False, buffersize=None, tempdir=None, cache=True): if presorted: self.table = table else: self.table = sort(table, key, buffersize=buffersize, tempdir=tempdir, cache=cache) self.key = key self.aggregation = aggregation self.value = value
def __init__(self, source, key, reducer, fields=None, presorted=False, buffersize=None, tempdir=None, cache=True): if presorted: self.source = source else: self.source = sort(source, key, buffersize=buffersize, tempdir=tempdir, cache=cache) self.key = key self.fields = fields self.reducer = reducer
def test_sort_3(): table = (('foo', 'bar'), ('C', '2'), ('A', '9'), ('A', '6'), ('F', '1'), ('D', '10')) result = sort(table, 'bar') expectation = (('foo', 'bar'), ('F', '1'), ('D', '10'), ('C', '2'), ('A', '6'), ('A', '9')) ieq(expectation, result)
def __init__(self, source, f1, f2, f3, aggfun, missing=None, presorted=False, buffersize=None, tempdir=None, cache=True): if presorted: self.source = source else: self.source = sort(source, key=(f1, f2), buffersize=buffersize, tempdir=tempdir, cache=cache) self.f1, self.f2, self.f3 = f1, f2, f3 self.aggfun = aggfun self.missing = missing
def groupselectmax(table, key, value, presorted=False, buffersize=None, tempdir=None, cache=True): """Group by the `key` field then return the row with the minimum of the `value` field within each group. N.B., will only return one row for each group, even if multiple rows have the same (maximum) value.""" return groupselectfirst(sort(table, value, reverse=True), key, presorted=presorted, buffersize=buffersize, tempdir=tempdir, cache=cache)
def test_sort_none(): table = (('foo', 'bar'), ('C', 2), ('A', 9), ('A', None), ('F', 1), ('D', 10)) result = sort(table, 'bar') expectation = (('foo', 'bar'), ('A', None), ('F', 1), ('C', 2), ('A', 9), ('D', 10)) ieq(expectation, result) dt = datetime.now().replace table = (('foo', 'bar'), ('C', dt(hour=5)), ('A', dt(hour=1)), ('A', None), ('F', dt(hour=9)), ('D', dt(hour=17))) result = sort(table, 'bar') expectation = (('foo', 'bar'), ('A', None), ('A', dt(hour=1)), ('C', dt(hour=5)), ('F', dt(hour=9)), ('D', dt(hour=17))) ieq(expectation, result)
def __init__(self, source, f1, f2, f3, aggfun, missing=None, presorted=False, buffersize=None, tempdir=None, cache=True): if presorted: self.source = source else: self.source = sort(source, key=(f1, f2), buffersize=buffersize, tempdir=tempdir, cache=cache) self.f1, self.f2, self.f3 = f1, f2, f3 self.aggfun = aggfun self.missing = missing
def __init__(self, source, key, mapper, header=None, presorted=False, buffersize=None, tempdir=None, cache=True): if presorted: self.source = source else: self.source = sort(source, key, buffersize=buffersize, tempdir=tempdir, cache=cache) self.key = key self.header = header self.mapper = mapper
def test_mergesort_4(): table1 = (("foo", "bar", "baz"), (1, "A", True), (2, "B", None), (4, "C", True)) table2 = (("bar", "baz", "quux"), ("A", True, 42.0), ("B", False, 79.3), ("C", False, 12.4)) expect = sort(cat(table1, table2), key="bar") actual = mergesort(table1, table2, key="bar") ieq(expect, actual) ieq(expect, actual)
def collapsedintervals(table, start='start', stop='stop', key=None): """ Utility function to collapse intervals in a table. If no facet `key` is given, returns an iterator over `(start, stop)` tuples. If facet `key` is given, returns an iterator over `(key, start, stop)` tuples. """ if key is None: table = sort(table, key=start) for iv in _collapse(values(table, (start, stop))): yield iv else: table = sort(table, key=(key, start)) for k, g in rowgroupby(table, key=key, value=(start, stop)): for iv in _collapse(g): yield (k, ) + iv
def test_sort_buffered_tempdir(): table = (('foo', 'bar'), ('C', 2), ('A', 9), ('A', 6), ('F', 1), ('D', 10)) # test sort forwards expectation = (('foo', 'bar'), ('F', 1), ('C', 2), ('A', 6), ('A', 9), ('D', 10)) result = sort(table, 'bar') ieq(expectation, result) result = sort(table, 'bar', buffersize=2, tempdir='/tmp') ieq(expectation, result)
def __init__(self, source, key, missing=None, exclude=None, include=None, presorted=False, buffersize=None, tempdir=None, cache=True): if presorted: self.source = source else: self.source = sort(source, key, buffersize=buffersize, tempdir=tempdir, cache=cache) self.key = key self.missing = missing self.exclude = exclude self.include = include
def collapsedintervals(table, start='start', stop='stop', key=None): """ Utility function to collapse intervals in a table. If no facet `key` is given, returns an iterator over `(start, stop)` tuples. If facet `key` is given, returns an iterator over `(key, start, stop)` tuples. """ if key is None: table = sort(table, key=start) for iv in _collapse(values(table, (start, stop))): yield iv else: table = sort(table, key=(key, start)) for k, g in rowgroupby(table, key=key, value=(start, stop)): for iv in _collapse(g): yield (k,) + iv
def groupselectmax(table, key, value): """ Group by the `key` field then return the row with the minimum of the `value` field within each group. N.B., will only return one row for each group, even if multiple rows have the same (maximum) value. .. versionadded:: 0.14 """ return groupselectfirst(sort(table, value, reverse=True), key)
def __init__(self, source, key, missing=None, exclude=None, include=None, presorted=False, buffersize=None, tempdir=None, cache=True): if presorted: self.source = source else: self.source = sort(source, key, buffersize=buffersize, tempdir=tempdir, cache=cache) self.key = key self.missing = missing self.exclude = exclude self.include = include
def test_mergesort_4(): table1 = (('foo', 'bar', 'baz'), (1, 'A', True), (2, 'B', None), (4, 'C', True)) table2 = (('bar', 'baz', 'quux'), ('A', True, 42.0), ('B', False, 79.3), ('C', False, 12.4)) expect = sort(cat(table1, table2), key='bar') actual = mergesort(table1, table2, key='bar') ieq(expect, actual) ieq(expect, actual)
def test_sort_buffered(): table = (("foo", "bar"), ("C", 2), ("A", 9), ("A", 6), ("F", 1), ("D", 10)) # test sort forwards expectation = (("foo", "bar"), ("F", 1), ("C", 2), ("A", 6), ("A", 9), ("D", 10)) result = sort(table, "bar") ieq(expectation, result) result = sort(table, "bar", buffersize=2) # print list(result) ieq(expectation, result) # sort in reverse expectation = (("foo", "bar"), ("D", 10), ("A", 9), ("A", 6), ("C", 2), ("F", 1)) result = sort(table, "bar", reverse=True) ieq(expectation, result) result = sort(table, "bar", reverse=True, buffersize=2) ieq(expectation, result) # no key expectation = (("foo", "bar"), ("F", 1), ("D", 10), ("C", 2), ("A", 9), ("A", 6)) result = sort(table, reverse=True) ieq(expectation, result) result = sort(table, reverse=True, buffersize=2) ieq(expectation, result)
def test_sort_buffered(): table = (('foo', 'bar'), ('C', 2), ('A', 9), ('A', 6), ('F', 1), ('D', 10)) # test sort forwards expectation = (('foo', 'bar'), ('F', 1), ('C', 2), ('A', 6), ('A', 9), ('D', 10)) result = sort(table, 'bar') ieq(expectation, result) result = sort(table, 'bar', buffersize=2) ieq(expectation, result) # sort in reverse expectation = (('foo', 'bar'), ('D', 10), ('A', 9), ('A', 6), ('C', 2), ('F', 1)) result = sort(table, 'bar', reverse=True) ieq(expectation, result) result = sort(table, 'bar', reverse=True, buffersize=2) ieq(expectation, result) # no key expectation = (('foo', 'bar'), ('F', 1), ('D', 10), ('C', 2), ('A', 9), ('A', 6)) result = sort(table, reverse=True) ieq(expectation, result) result = sort(table, reverse=True, buffersize=2) ieq(expectation, result)
def groupselectmax(table, key, value, presorted=False, buffersize=None, tempdir=None, cache=True): """Group by the `key` field then return the row with the maximum of the `value` field within each group. N.B., will only return one row for each group, even if multiple rows have the same (maximum) value.""" return groupselectfirst(sort(table, value, reverse=True), key, presorted=presorted, buffersize=buffersize, tempdir=tempdir, cache=cache)
def test_sort_buffered_independent(): table = (('foo', 'bar'), ('C', 2), ('A', 9), ('A', 6), ('F', 1), ('D', 10)) expectation = (('foo', 'bar'), ('F', 1), ('C', 2), ('A', 6), ('A', 9), ('D', 10)) result = sort(table, 'bar', buffersize=4) nrows(result) # cause data to be cached # check that two row iterators are independent, i.e., consuming rows # from one does not affect the other it1 = iter(result) it2 = iter(result) eq_(expectation[0], next(it1)) eq_(expectation[1], next(it1)) eq_(expectation[0], next(it2)) eq_(expectation[1], next(it2)) eq_(expectation[2], next(it2)) eq_(expectation[2], next(it1))
def test_sort_buffered_cleanup(): table = (('foo', 'bar'), ('C', 2), ('A', 9), ('A', 6), ('F', 1), ('D', 10)) result = sort(table, 'bar', buffersize=2) debug('initially filecache should be empty') eq_(None, result._filecache) debug('pull rows through, should populate file cache') eq_(5, nrows(result)) eq_(3, len(result._filecache)) debug('check all files exist') filenames = _get_names(result._filecache) for fn in filenames: assert os.path.exists(fn), fn debug('delete object and garbage collect') del result gc.collect() debug('check all files have been deleted') for fn in filenames: assert not os.path.exists(fn), fn
def __init__(self, table, key, aggregation=list, value=None, presorted=False, buffersize=None, tempdir=None, cache=True): if presorted: self.table = table else: self.table = sort(table, key, buffersize=buffersize, tempdir=tempdir, cache=cache) self.key = key self.aggregation = aggregation self.value = value
def people_list(request, uuid): try: csvdownload = CSVDownload.objects.get(uuid=uuid) except CSVDownload.DoesNotExist: return HttpResponseNotFound("Not found.") fname = '{0}.csv'.format(csvdownload.uuid) full_fname = os.path.join(settings.CSV_DIR, fname) people = fromcsv(full_fname) sortby = request.GET.get('sortby', 'name') ordering = request.GET.get('ordering', 'asc') count_str = request.GET.get('count', '10') if sortby not in header(people): return HttpResponseBadRequest('Bad request.') if ordering not in ('asc', 'desc'): return HttpResponseBadRequest('Bad request.') try: count = int(count_str) except ValueError: return HttpResponseBadRequest('Bad request.') if count < 1: return HttpResponseBadRequest('Bad request.') people = sort(people, sortby, reverse=ordering == 'desc') people = head(people, count) return render( request, 'people_list.html', { 'csvdownload': csvdownload, 'headers': header(people), 'people': data(people), 'has_more': len(people) > count, 'queryparams': { 'sortby': sortby, 'ordering': ordering, 'count': str(count + 10) } })
def test_sort_buffered_cleanup_open_iterator(): table = (('foo', 'bar'), ('C', 2), ('A', 9), ('A', 6), ('F', 1), ('D', 10)) # check if cleanup is robust against open iterators result = sort(table, 'bar', buffersize=2) debug('pull rows through, should populate file cache') eq_(5, nrows(result)) eq_(3, len(result._filecache)) debug('check all files exist') filenames = _get_names(result._filecache) for fn in filenames: assert os.path.exists(fn), fn debug(filenames) debug('open an iterator') it = iter(result) next(it) next(it) debug('delete objects and garbage collect') del result del it gc.collect() for fn in filenames: assert not os.path.exists(fn), fn