def _ieq_row(re, ra, ir): assert ra is not None, "Expected row #%d is None, but result row is not None" % ir assert re is not None, "Expected row #%d is not None, but result row is None" % ir ic = 0 for ve, va in izip_longest(re, ra, fillvalue=None): if isinstance(ve, list): for je, ja in izip_longest(ve, va, fillvalue=None): _ieq_col(je, ja, re, ra, ir, ic) elif not isinstance(ve, dict): _ieq_col(ve, va, re, ra, ir, ic) ic = ic + 1
def _eq_rows(expect, actual, cast=None): '''test when values are equals for eacfh row and column''' ie = iter(expect) ia = iter(actual) for re, ra in izip_longest(ie, ia, fillvalue=None): if cast: ra = cast(ra) for ve, va in izip_longest(re, ra, fillvalue=None): if isinstance(ve, list): for je, ja in izip_longest(ve, va, fillvalue=None): _eq2(je, ja, re, ra) elif not isinstance(ve, dict): _eq2(ve, va, re, ra)
def ieq(expect, actual, cast=None): ie = iter(expect) ia = iter(actual) for e, a in izip_longest(ie, ia, fillvalue=None): if cast: a = cast(a) eq_(e, a)
def toxls(tbl, filename, sheet, encoding=None, style_compression=0, styles=None): """ Write a table to a new Excel .xls file. """ import xlwt if encoding is None: encoding = locale.getpreferredencoding() wb = xlwt.Workbook(encoding=encoding, style_compression=style_compression) ws = wb.add_sheet(sheet) if styles is None: # simple version, don't worry about styles for r, row in enumerate(tbl): for c, v in enumerate(row): ws.write(r, c, label=v) else: # handle styles it = iter(tbl) hdr = next(it) flds = list(map(str, hdr)) for c, f in enumerate(flds): ws.write(0, c, label=f) if f not in styles or styles[f] is None: styles[f] = xlwt.Style.default_style # convert to list for easy zipping styles = [styles[f] for f in flds] for r, row in enumerate(it): for c, (v, style) in enumerate(izip_longest(row, styles, fillvalue=None)): ws.write(r+1, c, label=v, style=style) wb.save(filename)
def columns(table, missing=None): """ Construct a :class:`dict` mapping field names to lists of values. E.g.:: >>> import petl as etl >>> table = [['foo', 'bar'], ['a', 1], ['b', 2], ['b', 3]] >>> cols = etl.columns(table) >>> cols['foo'] ['a', 'b', 'b'] >>> cols['bar'] [1, 2, 3] See also :func:`petl.util.materialise.facetcolumns`. """ cols = OrderedDict() it = iter(table) hdr = next(it) flds = list(map(text_type, hdr)) for f in flds: cols[f] = list() for row in it: for f, v in izip_longest(flds, row, fillvalue=missing): if f in cols: cols[f].append(v) return cols
def ieq(expect, actual, cast=None): '''test when values are equals for eacfh row and column''' ie = iter(expect) ia = iter(actual) for re, ra in izip_longest(ie, ia, fillvalue=None): if cast: ra = cast(ra) if re is None and ra is None: continue if type(re) in (int, float, bool, str): eq_(re, ra) continue for ve, va in izip_longest(re, ra, fillvalue=None): if isinstance(ve, list): for je, ja in izip_longest(ve, va, fillvalue=None): _eq_print(je, ja, re, ra) elif not isinstance(ve, dict): _eq_print(ve, va, re, ra)
def ieq(expect, actual, cast=None): """Test when values of a iterable are equals for each row and column""" ie = iter(expect) ia = iter(actual) ir = 0 for re, ra in izip_longest(ie, ia, fillvalue=None): if cast: ra = cast(ra) if re is None and ra is None: continue if type(re) in (int, float, bool, str): eq_(re, ra) continue _ieq_row(re, ra, ir) ir = ir + 1
def _write_row(f, flds, row, lineterminator, vrepr, tr_style, td_styles, truncate): tr_css = _get_tr_css(row, tr_style) if tr_css: f.write(("<tr style='%s'>" % tr_css) + lineterminator) else: f.write("<tr>" + lineterminator) for h, v in izip_longest(flds, row, fillvalue=None): r = vrepr(v) if truncate: r = r[:truncate] td_css = _get_td_css(h, v, td_styles) if td_css: f.write(("<td style='%s'>%s</td>" % (td_css, r)) + lineterminator) else: f.write(("<td>%s</td>" % r) + lineterminator) f.write("</tr>" + lineterminator)
def _write_row(f, flds, row, lineterminator, vrepr, tr_style, td_styles, truncate): tr_css = _get_tr_css(row, tr_style) if tr_css: f.write(("<tr style='%s'>" % tr_css) + lineterminator) else: f.write("<tr>" + lineterminator) for h, v in izip_longest(flds, row, fillvalue=None): r = vrepr(v) if truncate: r = r[:truncate] td_css = _get_td_css(h, v, td_styles) if td_css: f.write(("<td style='%s'>%s</td>" % (td_css, r)) + lineterminator) else: f.write(("<td>%s</td>" % r) + lineterminator) f.write('</tr>' + lineterminator)
def iterannex(tables, missing): its = [iter(t) for t in tables] hdrs = [next(it) for it in its] outhdr = tuple(chain(*hdrs)) yield outhdr for rows in izip_longest(*its): outrow = list() for i, row in enumerate(rows): lh = len(hdrs[i]) if row is None: # handle uneven length tables row = [missing] * len(hdrs[i]) else: lr = len(row) if lr < lh: # handle short rows row = list(row) row.extend([missing] * (lh - lr)) elif lr > lh: # handle long rows row = row[:lh] outrow.extend(row) yield tuple(outrow)
def iterannex(tables, missing): its = [iter(t) for t in tables] hdrs = [next(it) for it in its] outhdr = tuple(chain(*hdrs)) yield outhdr for rows in izip_longest(*its): outrow = list() for i, row in enumerate(rows): lh = len(hdrs[i]) if row is None: # handle uneven length tables row = [missing] * len(hdrs[i]) else: lr = len(row) if lr < lh: # handle short rows row = list(row) row.extend([missing] * (lh-lr)) elif lr > lh: # handle long rows row = row[:lh] outrow.extend(row) yield tuple(outrow)
def toxls(tbl, filename, sheet, encoding=None, style_compression=0, styles=None): """ Write a table to a new Excel .xls file. """ import xlwt if encoding is None: encoding = locale.getpreferredencoding() wb = xlwt.Workbook(encoding=encoding, style_compression=style_compression) ws = wb.add_sheet(sheet) if styles is None: # simple version, don't worry about styles for r, row in enumerate(tbl): for c, v in enumerate(row): ws.write(r, c, label=v) else: # handle styles it = iter(tbl) hdr = next(it) flds = list(map(str, hdr)) for c, f in enumerate(flds): ws.write(0, c, label=f) if f not in styles or styles[f] is None: styles[f] = xlwt.Style.default_style # convert to list for easy zipping styles = [styles[f] for f in flds] for r, row in enumerate(it): for c, (v, style) in enumerate( izip_longest(row, styles, fillvalue=None)): ws.write(r + 1, c, label=v, style=style) target = write_source_from_arg(filename) with target.open('wb') as target2: wb.save(target2)
def iteraddcolumn(table, field, col, index, missing): it = iter(table) hdr = next(it) # determine position of new column if index is None: index = len(hdr) # construct output header outhdr = list(hdr) outhdr.insert(index, field) yield tuple(outhdr) # construct output data for row, val in izip_longest(it, col, fillvalue=missing): # run out of rows? if row == missing: row = [missing] * len(hdr) outrow = list(row) outrow.insert(index, val) yield tuple(outrow)
def facetcolumns(table, key, missing=None): """ Like :func:`petl.util.materialise.columns` but stratified by values of the given key field. E.g.:: >>> import petl as etl >>> table = [['foo', 'bar', 'baz'], ... ['a', 1, True], ... ['b', 2, True], ... ['b', 3]] >>> fc = etl.facetcolumns(table, 'foo') >>> fc['a'] {'foo': ['a'], 'bar': [1], 'baz': [True]} >>> fc['b'] {'foo': ['b', 'b'], 'bar': [2, 3], 'baz': [True, None]} """ fct = dict() it = iter(table) hdr = next(it) flds = list(map(text_type, hdr)) indices = asindices(hdr, key) assert len(indices) > 0, 'no key field selected' getkey = operator.itemgetter(*indices) for row in it: kv = getkey(row) if kv not in fct: cols = dict() for f in flds: cols[f] = list() fct[kv] = cols else: cols = fct[kv] for f, v in izip_longest(flds, row, fillvalue=missing): if f in cols: cols[f].append(v) return fct
def _update_field_defs_from(props, row, fields, previous, fill_missing): for prop, val in izip_longest(props, row): if prop is None: break dprev = previous.get(prop + '_prec') fprev = previous.get(prop + '_prop') fcurr = None if isinstance(val, dict): # get the fields from a recursive definition of record inside this field tdef, dcurr, fcurr = _get_definition_from_record(prop, val, fprev, dprev, fill_missing) else: # get the field definition for building the schema tdef, dcurr = _get_definition_from_type_of(prop, val, dprev) if tdef is not None: fields[prop] = {'name': prop, 'type': ['null', tdef]} elif fill_missing: fields[prop] = {'name': prop, 'type': ['null', 'string']} if dcurr is not None: previous[prop + '_prec'] = dcurr if fcurr is not None: previous[prop + '_prop'] = fcurr
def izip_longest(self, *args, **kwargs): return izip_longest(self, *args, **kwargs)
def itercolumns(cols, header, missing): if header is None: header = ['f%s' % i for i in range(len(cols))] yield tuple(header) for row in izip_longest(*cols, **dict(fillvalue=missing)): yield row