def test_rowgroupby():
table = (('foo', 'bar', 'baz'),
('a', 1, True),
('b', 2, True),
('b', 3))
# simplest form
g = rowgroupby(table, 'foo')
key, vals = g.next()
vals = list(vals)
eq_('a', key)
eq_(1, len(vals))
eq_(('a', 1, True), vals[0])
key, vals = g.next()
vals = list(vals)
eq_('b', key)
eq_(2, len(vals))
eq_(('b', 2, True), vals[0])
eq_(('b', 3), vals[1])
# specify value
g = rowgroupby(table, 'foo', 'bar')
key, vals = g.next()
vals = list(vals)
eq_('a', key)
eq_(1, len(vals))
eq_(1, vals[0])
key, vals = g.next()
vals = list(vals)
eq_('b', key)
eq_(2, len(vals))
eq_(2, vals[0])
eq_(3, vals[1])
# callable key
g = rowgroupby(table, lambda r: r['foo'], lambda r: r['baz'])
key, vals = g.next()
vals = list(vals)
eq_('a', key)
eq_(1, len(vals))
eq_(True, vals[0])
key, vals = g.next()
vals = list(vals)
eq_('b', key)
eq_(2, len(vals))
eq_(True, vals[0])
eq_(None, vals[1]) # gets padded
def test_rowgroupby():
table = (('foo', 'bar', 'baz'),
('a', 1, True),
('b', 2, True),
('b', 3))
# simplest form
g = rowgroupby(table, 'foo')
key, vals = g.next()
vals = list(vals)
eq_('a', key)
eq_(1, len(vals))
eq_(('a', 1, True), vals[0])
key, vals = g.next()
vals = list(vals)
eq_('b', key)
eq_(2, len(vals))
eq_(('b', 2, True), vals[0])
eq_(('b', 3), vals[1])
# specify value
g = rowgroupby(table, 'foo', 'bar')
key, vals = g.next()
vals = list(vals)
eq_('a', key)
eq_(1, len(vals))
eq_(1, vals[0])
key, vals = g.next()
vals = list(vals)
eq_('b', key)
eq_(2, len(vals))
eq_(2, vals[0])
eq_(3, vals[1])
# callable key
g = rowgroupby(table, lambda r: r['foo'], lambda r: r['baz'])
key, vals = g.next()
vals = list(vals)
eq_('a', key)
eq_(1, len(vals))
eq_(True, vals[0])
key, vals = g.next()
vals = list(vals)
eq_('b', key)
eq_(2, len(vals))
eq_(True, vals[0])
eq_(None, vals[1]) # gets padded
def test_rowgroupby():
table = (("foo", "bar", "baz"), ("a", 1, True), ("b", 2, True), ("b", 3))
# simplest form
g = rowgroupby(table, "foo")
key, vals = g.next()
vals = list(vals)
eq_("a", key)
eq_(1, len(vals))
eq_(("a", 1, True), vals[0])
key, vals = g.next()
vals = list(vals)
eq_("b", key)
eq_(2, len(vals))
eq_(("b", 2, True), vals[0])
eq_(("b", 3), vals[1])
# specify value
g = rowgroupby(table, "foo", "bar")
key, vals = g.next()
vals = list(vals)
eq_("a", key)
eq_(1, len(vals))
eq_(1, vals[0])
key, vals = g.next()
vals = list(vals)
eq_("b", key)
eq_(2, len(vals))
eq_(2, vals[0])
eq_(3, vals[1])
# callable key
g = rowgroupby(table, lambda r: r["foo"], lambda r: r["baz"])
key, vals = g.next()
vals = list(vals)
eq_("a", key)
eq_(1, len(vals))
eq_(True, vals[0])
key, vals = g.next()
vals = list(vals)
eq_("b", key)
eq_(2, len(vals))
eq_(True, vals[0])
eq_(None, vals[1]) # gets padded
def produce_weather_data(count):
result = []
output_data = petl.fromcsv(config.weather_data_output_path, delimiter='|')
header = petl.header(output_data)
result.insert(0, header)
for key, group in petl.rowgroupby(output_data, key='Position'):
random_index = random.randint(0, config.available_records - 1)
group_list = list(group)
result.append(tuple(group_list[random_index]))
result = result[0:count]
merged_output = ['|'.join(data) for data in result]
merged_output = '\n'.join(merged_output)
return merged_output
def etl(self, *args, **kw):
table = petl.fromxlsx(self._src_path)
model = DEPTH_TO_WATER
self._update_model(model, self._vocab)
# group table by sys_loc_code
header = petl.header(table)
for name, records in petl.rowgroupby(petl.sort(table, 'sys_loc_code'),
'sys_loc_code'):
records = [dict(zip(header, record)) for record in records]
record = records[0]
location_id = self._post_location(record, model)
thing_id = self._post_thing(record, model, location_id)
print('---------------')
print(f'len records {len(records)}')
# self.add_package(record)
self.observation.set_records(records)
self.observation.etl(tids=self._make_tids(thing_id, record),
models=(model, ))
look(table)
isordered(table, key='foo')
isordered(table, key='foo', strict=True)
isordered(table, key='foo', reverse=True)
# rowgroupby
table = (('foo', 'bar', 'baz'),
('a', 1, True),
('b', 3, True),
('b', 2))
from petl import rowgroupby, look
look(table)
# group entire rows
for key, group in rowgroupby(table, 'foo'):
print key, list(group)
# group specific values
for key, group in rowgroupby(table, 'foo', 'bar'):
print key, list(group)
# fold
table1 = (('id', 'count'), (1, 3), (1, 5), (2, 4), (2, 8))
from petl import fold, look
look(table1)
import operator
table2 = fold(table1, 'id', operator.add, 'count', presorted=True)
look(table2)
list(d)
# records()
###############
import petl as etl
table = [['foo', 'bar'], ['a', 1], ['b', 2]]
d = etl.records(table)
d
list(d)
# rowgroupby()
##############
import petl as etl
table1 = [['foo', 'bar', 'baz'], ['a', 1, True], ['b', 3, True], ['b', 2]]
# group entire rows
for key, group in etl.rowgroupby(table1, 'foo'):
print(key, list(group))
# group specific values
for key, group in etl.rowgroupby(table1, 'foo', 'bar'):
print(key, list(group))
# empty()
#########
import petl as etl
table = (etl.empty().addcolumn('foo', ['A', 'B']).addcolumn('bar', [1, 2]))
table
look(table)
isordered(table, key='foo')
isordered(table, key='foo', strict=True)
isordered(table, key='foo', reverse=True)
# rowgroupby
table = (('foo', 'bar', 'baz'),
('a', 1, True),
('b', 3, True),
('b', 2))
from petl import rowgroupby, look
look(table)
# group entire rows
for key, group in rowgroupby(table, 'foo'):
print key, list(group)
# group specific values
for key, group in rowgroupby(table, 'foo', 'bar'):
print key, list(group)
# fold
table1 = (('id', 'count'), (1, 3), (1, 5), (2, 4), (2, 8))
from petl import fold, look
look(table1)
import operator
table2 = fold(table1, 'id', operator.add, 'count', presorted=True)
look(table2)
import petl as etl
table = [["foo", "bar"], ["a", 1], ["b", 2]]
d = etl.records(table)
d
list(d)
# rowgroupby()
##############
import petl as etl
table1 = [["foo", "bar", "baz"], ["a", 1, True], ["b", 3, True], ["b", 2]]
# group entire rows
for key, group in etl.rowgroupby(table1, "foo"):
print(key, list(group))
# group specific values
for key, group in etl.rowgroupby(table1, "foo", "bar"):
print(key, list(group))
# empty()
#########
import petl as etl
table = etl.empty().addcolumn("foo", ["A", "B"]).addcolumn("bar", [1, 2])
table
