def _ieq_row(re, ra, ir):
assert ra is not None, "Expected row #%d is None, but result row is not None" % ir
assert re is not None, "Expected row #%d is not None, but result row is None" % ir
ic = 0
for ve, va in izip_longest(re, ra, fillvalue=None):
if isinstance(ve, list):
for je, ja in izip_longest(ve, va, fillvalue=None):
_ieq_col(je, ja, re, ra, ir, ic)
elif not isinstance(ve, dict):
_ieq_col(ve, va, re, ra, ir, ic)
ic = ic + 1
def _eq_rows(expect, actual, cast=None):
'''test when values are equals for eacfh row and column'''
ie = iter(expect)
ia = iter(actual)
for re, ra in izip_longest(ie, ia, fillvalue=None):
if cast:
ra = cast(ra)
for ve, va in izip_longest(re, ra, fillvalue=None):
if isinstance(ve, list):
for je, ja in izip_longest(ve, va, fillvalue=None):
_eq2(je, ja, re, ra)
elif not isinstance(ve, dict):
_eq2(ve, va, re, ra)
def ieq(expect, actual, cast=None):
ie = iter(expect)
ia = iter(actual)
for e, a in izip_longest(ie, ia, fillvalue=None):
if cast:
a = cast(a)
eq_(e, a)
def ieq(expect, actual, cast=None):
ie = iter(expect)
ia = iter(actual)
for e, a in izip_longest(ie, ia, fillvalue=None):
if cast:
a = cast(a)
eq_(e, a)
def toxls(tbl, filename, sheet, encoding=None, style_compression=0,
styles=None):
"""
Write a table to a new Excel .xls file.
"""
import xlwt
if encoding is None:
encoding = locale.getpreferredencoding()
wb = xlwt.Workbook(encoding=encoding, style_compression=style_compression)
ws = wb.add_sheet(sheet)
if styles is None:
# simple version, don't worry about styles
for r, row in enumerate(tbl):
for c, v in enumerate(row):
ws.write(r, c, label=v)
else:
# handle styles
it = iter(tbl)
hdr = next(it)
flds = list(map(str, hdr))
for c, f in enumerate(flds):
ws.write(0, c, label=f)
if f not in styles or styles[f] is None:
styles[f] = xlwt.Style.default_style
# convert to list for easy zipping
styles = [styles[f] for f in flds]
for r, row in enumerate(it):
for c, (v, style) in enumerate(izip_longest(row, styles,
fillvalue=None)):
ws.write(r+1, c, label=v, style=style)
wb.save(filename)
def columns(table, missing=None):
"""
Construct a :class:`dict` mapping field names to lists of values. E.g.::
>>> import petl as etl
>>> table = [['foo', 'bar'], ['a', 1], ['b', 2], ['b', 3]]
>>> cols = etl.columns(table)
>>> cols['foo']
['a', 'b', 'b']
>>> cols['bar']
[1, 2, 3]
See also :func:`petl.util.materialise.facetcolumns`.
"""
cols = OrderedDict()
it = iter(table)
hdr = next(it)
flds = list(map(text_type, hdr))
for f in flds:
cols[f] = list()
for row in it:
for f, v in izip_longest(flds, row, fillvalue=missing):
if f in cols:
cols[f].append(v)
return cols
def ieq(expect, actual, cast=None):
'''test when values are equals for eacfh row and column'''
ie = iter(expect)
ia = iter(actual)
for re, ra in izip_longest(ie, ia, fillvalue=None):
if cast:
ra = cast(ra)
if re is None and ra is None:
continue
if type(re) in (int, float, bool, str):
eq_(re, ra)
continue
for ve, va in izip_longest(re, ra, fillvalue=None):
if isinstance(ve, list):
for je, ja in izip_longest(ve, va, fillvalue=None):
_eq_print(je, ja, re, ra)
elif not isinstance(ve, dict):
_eq_print(ve, va, re, ra)
def ieq(expect, actual, cast=None):
"""Test when values of a iterable are equals for each row and column"""
ie = iter(expect)
ia = iter(actual)
ir = 0
for re, ra in izip_longest(ie, ia, fillvalue=None):
if cast:
ra = cast(ra)
if re is None and ra is None:
continue
if type(re) in (int, float, bool, str):
eq_(re, ra)
continue
_ieq_row(re, ra, ir)
ir = ir + 1
def _write_row(f, flds, row, lineterminator, vrepr, tr_style, td_styles, truncate):
tr_css = _get_tr_css(row, tr_style)
if tr_css:
f.write(("<tr style='%s'>" % tr_css) + lineterminator)
else:
f.write("<tr>" + lineterminator)
for h, v in izip_longest(flds, row, fillvalue=None):
r = vrepr(v)
if truncate:
r = r[:truncate]
td_css = _get_td_css(h, v, td_styles)
if td_css:
f.write(("<td style='%s'>%s</td>" % (td_css, r)) + lineterminator)
else:
f.write(("<td>%s</td>" % r) + lineterminator)
f.write("</tr>" + lineterminator)
def _write_row(f, flds, row, lineterminator, vrepr, tr_style, td_styles,
truncate):
tr_css = _get_tr_css(row, tr_style)
if tr_css:
f.write(("<tr style='%s'>" % tr_css) + lineterminator)
else:
f.write("<tr>" + lineterminator)
for h, v in izip_longest(flds, row, fillvalue=None):
r = vrepr(v)
if truncate:
r = r[:truncate]
td_css = _get_td_css(h, v, td_styles)
if td_css:
f.write(("<td style='%s'>%s</td>" % (td_css, r)) + lineterminator)
else:
f.write(("<td>%s</td>" % r) + lineterminator)
f.write('</tr>' + lineterminator)
def iterannex(tables, missing):
its = [iter(t) for t in tables]
hdrs = [next(it) for it in its]
outhdr = tuple(chain(*hdrs))
yield outhdr
for rows in izip_longest(*its):
outrow = list()
for i, row in enumerate(rows):
lh = len(hdrs[i])
if row is None: # handle uneven length tables
row = [missing] * len(hdrs[i])
else:
lr = len(row)
if lr < lh: # handle short rows
row = list(row)
row.extend([missing] * (lh - lr))
elif lr > lh: # handle long rows
row = row[:lh]
outrow.extend(row)
yield tuple(outrow)
def iterannex(tables, missing):
its = [iter(t) for t in tables]
hdrs = [next(it) for it in its]
outhdr = tuple(chain(*hdrs))
yield outhdr
for rows in izip_longest(*its):
outrow = list()
for i, row in enumerate(rows):
lh = len(hdrs[i])
if row is None: # handle uneven length tables
row = [missing] * len(hdrs[i])
else:
lr = len(row)
if lr < lh: # handle short rows
row = list(row)
row.extend([missing] * (lh-lr))
elif lr > lh: # handle long rows
row = row[:lh]
outrow.extend(row)
yield tuple(outrow)
def toxls(tbl,
filename,
sheet,
encoding=None,
style_compression=0,
styles=None):
"""
Write a table to a new Excel .xls file.
"""
import xlwt
if encoding is None:
encoding = locale.getpreferredencoding()
wb = xlwt.Workbook(encoding=encoding, style_compression=style_compression)
ws = wb.add_sheet(sheet)
if styles is None:
# simple version, don't worry about styles
for r, row in enumerate(tbl):
for c, v in enumerate(row):
ws.write(r, c, label=v)
else:
# handle styles
it = iter(tbl)
hdr = next(it)
flds = list(map(str, hdr))
for c, f in enumerate(flds):
ws.write(0, c, label=f)
if f not in styles or styles[f] is None:
styles[f] = xlwt.Style.default_style
# convert to list for easy zipping
styles = [styles[f] for f in flds]
for r, row in enumerate(it):
for c, (v, style) in enumerate(
izip_longest(row, styles, fillvalue=None)):
ws.write(r + 1, c, label=v, style=style)
target = write_source_from_arg(filename)
with target.open('wb') as target2:
wb.save(target2)
def iteraddcolumn(table, field, col, index, missing):
it = iter(table)
hdr = next(it)
# determine position of new column
if index is None:
index = len(hdr)
# construct output header
outhdr = list(hdr)
outhdr.insert(index, field)
yield tuple(outhdr)
# construct output data
for row, val in izip_longest(it, col, fillvalue=missing):
# run out of rows?
if row == missing:
row = [missing] * len(hdr)
outrow = list(row)
outrow.insert(index, val)
yield tuple(outrow)
def iteraddcolumn(table, field, col, index, missing):
it = iter(table)
hdr = next(it)
# determine position of new column
if index is None:
index = len(hdr)
# construct output header
outhdr = list(hdr)
outhdr.insert(index, field)
yield tuple(outhdr)
# construct output data
for row, val in izip_longest(it, col, fillvalue=missing):
# run out of rows?
if row == missing:
row = [missing] * len(hdr)
outrow = list(row)
outrow.insert(index, val)
yield tuple(outrow)
def facetcolumns(table, key, missing=None):
"""
Like :func:`petl.util.materialise.columns` but stratified by values of the
given key field. E.g.::
>>> import petl as etl
>>> table = [['foo', 'bar', 'baz'],
... ['a', 1, True],
... ['b', 2, True],
... ['b', 3]]
>>> fc = etl.facetcolumns(table, 'foo')
>>> fc['a']
{'foo': ['a'], 'bar': [1], 'baz': [True]}
>>> fc['b']
{'foo': ['b', 'b'], 'bar': [2, 3], 'baz': [True, None]}
"""
fct = dict()
it = iter(table)
hdr = next(it)
flds = list(map(text_type, hdr))
indices = asindices(hdr, key)
assert len(indices) > 0, 'no key field selected'
getkey = operator.itemgetter(*indices)
for row in it:
kv = getkey(row)
if kv not in fct:
cols = dict()
for f in flds:
cols[f] = list()
fct[kv] = cols
else:
cols = fct[kv]
for f, v in izip_longest(flds, row, fillvalue=missing):
if f in cols:
cols[f].append(v)
return fct
def _update_field_defs_from(props, row, fields, previous, fill_missing):
for prop, val in izip_longest(props, row):
if prop is None:
break
dprev = previous.get(prop + '_prec')
fprev = previous.get(prop + '_prop')
fcurr = None
if isinstance(val, dict):
# get the fields from a recursive definition of record inside this field
tdef, dcurr, fcurr = _get_definition_from_record(prop, val, fprev, dprev, fill_missing)
else:
# get the field definition for building the schema
tdef, dcurr = _get_definition_from_type_of(prop, val, dprev)
if tdef is not None:
fields[prop] = {'name': prop, 'type': ['null', tdef]}
elif fill_missing:
fields[prop] = {'name': prop, 'type': ['null', 'string']}
if dcurr is not None:
previous[prop + '_prec'] = dcurr
if fcurr is not None:
previous[prop + '_prop'] = fcurr
def izip_longest(self, *args, **kwargs):
return izip_longest(self, *args, **kwargs)
def itercolumns(cols, header, missing):
if header is None:
header = ['f%s' % i for i in range(len(cols))]
yield tuple(header)
for row in izip_longest(*cols, **dict(fillvalue=missing)):
yield row
def izip_longest(self, *args, **kwargs):
return izip_longest(self, *args, **kwargs)
def itercolumns(cols, header, missing):
if header is None:
header = ['f%s' % i for i in range(len(cols))]
yield tuple(header)
for row in izip_longest(*cols, **dict(fillvalue=missing)):
yield row
