def _iterteetext(table, source, encoding, errors, template, prologue, epilogue):
# guard conditions
assert template is not None, 'template is required'
# prepare source
source = write_source_from_arg(source)
with source.open('wb') as buf:
# deal with text encoding
if PY2:
codec = getcodec(encoding)
f = codec.streamwriter(buf, errors=errors)
else:
f = io.TextIOWrapper(buf,
encoding=encoding,
errors=errors)
# write the data
try:
if prologue is not None:
f.write(prologue)
it = iter(table)
hdr = next(it)
yield tuple(hdr)
flds = list(map(text_type, hdr))
for row in it:
rec = asdict(flds, row)
s = template.format(**rec)
f.write(s)
yield row
if epilogue is not None:
f.write(epilogue)
f.flush()
finally:
if not PY2:
f.detach()
def _writetext(table, source, mode, encoding, errors, template, prologue,
epilogue):
# guard conditions
assert template is not None, 'template is required'
# prepare source
source = write_source_from_arg(source)
with source.open(mode) as buf:
# deal with text encoding
if PY2:
codec = getcodec(encoding)
f = codec.streamwriter(buf, errors=errors)
else:
f = io.TextIOWrapper(buf,
encoding=encoding,
errors=errors,
newline='')
# write the table
try:
if prologue is not None:
f.write(prologue)
it = iter(table)
hdr = next(it)
flds = list(map(text_type, hdr))
for row in it:
rec = asdict(flds, row)
s = template.format(**rec)
f.write(s)
if epilogue is not None:
f.write(epilogue)
f.flush()
finally:
if not PY2:
f.detach()
def dictlookupone(table, key, dictionary=None, strict=False):
"""
Load a dictionary with data from the given table, mapping to dicts,
assuming there is at most one row for each key. E.g.::
>>> import petl as etl
>>> table1 = [['foo', 'bar'],
... ['a', 1],
... ['b', 2],
... ['b', 3]]
>>> # if the specified key is not unique and strict=False (default),
... # the first value wins
... lkp = etl.dictlookupone(table1, 'foo')
>>> lkp['a']
{'foo': 'a', 'bar': 1}
>>> lkp['b']
{'foo': 'b', 'bar': 2}
>>> # if the specified key is not unique and strict=True, will raise
... # DuplicateKeyError
... try:
... lkp = etl.dictlookupone(table1, 'foo', strict=True)
... except etl.errors.DuplicateKeyError as e:
... print(e)
...
duplicate key: 'b'
>>> # compound keys are supported
... table2 = [['foo', 'bar', 'baz'],
... ['a', 1, True],
... ['b', 2, False],
... ['b', 3, True],
... ['b', 3, False]]
>>> lkp = etl.dictlookupone(table2, ('foo', 'bar'))
>>> lkp[('a', 1)]
{'foo': 'a', 'baz': True, 'bar': 1}
>>> lkp[('b', 2)]
{'foo': 'b', 'baz': False, 'bar': 2}
>>> lkp[('b', 3)]
{'foo': 'b', 'baz': True, 'bar': 3}
>>> # data can be loaded into an existing dictionary-like
... # object, including persistent dictionaries created via the
... # shelve module
... import shelve
>>> lkp = shelve.open('example.dat', flag='n')
>>> lkp = etl.dictlookupone(table1, 'foo', lkp)
>>> lkp.close()
>>> lkp = shelve.open('example.dat', flag='r')
>>> lkp['a']
{'foo': 'a', 'bar': 1}
>>> lkp['b']
{'foo': 'b', 'bar': 2}
"""
if dictionary is None:
dictionary = dict()
it = iter(table)
hdr = next(it)
flds = list(map(text_type, hdr))
keyindices = asindices(hdr, key)
assert len(keyindices) > 0, 'no key selected'
getkey = operator.itemgetter(*keyindices)
for row in it:
k = getkey(row)
if strict and k in dictionary:
raise DuplicateKeyError(k)
elif k not in dictionary:
d = asdict(flds, row)
dictionary[k] = d
return dictionary
def dictlookup(table, key, dictionary=None):
"""
Load a dictionary with data from the given table, mapping to dicts. E.g.::
>>> import petl as etl
>>> table1 = [['foo', 'bar'],
... ['a', 1],
... ['b', 2],
... ['b', 3]]
>>> lkp = etl.dictlookup(table1, 'foo')
>>> lkp['a']
[{'foo': 'a', 'bar': 1}]
>>> lkp['b']
[{'foo': 'b', 'bar': 2}, {'foo': 'b', 'bar': 3}]
>>> # compound keys are supported
... table2 = [['foo', 'bar', 'baz'],
... ['a', 1, True],
... ['b', 2, False],
... ['b', 3, True],
... ['b', 3, False]]
>>> lkp = etl.dictlookup(table2, ('foo', 'bar'))
>>> lkp[('a', 1)]
[{'foo': 'a', 'baz': True, 'bar': 1}]
>>> lkp[('b', 2)]
[{'foo': 'b', 'baz': False, 'bar': 2}]
>>> lkp[('b', 3)]
[{'foo': 'b', 'baz': True, 'bar': 3}, {'foo': 'b', 'baz': False, 'bar': 3}]
>>> # data can be loaded into an existing dictionary-like
... # object, including persistent dictionaries created via the
... # shelve module
... import shelve
>>> lkp = shelve.open('example.dat', flag='n')
>>> lkp = etl.dictlookup(table1, 'foo', lkp)
>>> lkp.close()
>>> lkp = shelve.open('example.dat', flag='r')
>>> lkp['a']
[{'foo': 'a', 'bar': 1}]
>>> lkp['b']
[{'foo': 'b', 'bar': 2}, {'foo': 'b', 'bar': 3}]
"""
if dictionary is None:
dictionary = dict()
it = iter(table)
hdr = next(it)
flds = list(map(text_type, hdr))
keyindices = asindices(hdr, key)
assert len(keyindices) > 0, 'no key selected'
getkey = operator.itemgetter(*keyindices)
for row in it:
k = getkey(row)
rec = asdict(flds, row)
if k in dictionary:
# work properly with shelve
l = dictionary[k]
l.append(rec)
dictionary[k] = l
else:
dictionary[k] = [rec]
return dictionary
def dictlookupone(table, key, dictionary=None, strict=False):
"""
Load a dictionary with data from the given table, mapping to dicts,
assuming there is at most one row for each key. E.g.::
>>> import petl as etl
>>> table1 = [['foo', 'bar'],
... ['a', 1],
... ['b', 2],
... ['b', 3]]
>>> # if the specified key is not unique and strict=False (default),
... # the first value wins
... lkp = etl.dictlookupone(table1, 'foo')
>>> lkp['a']
{'foo': 'a', 'bar': 1}
>>> lkp['b']
{'foo': 'b', 'bar': 2}
>>> # if the specified key is not unique and strict=True, will raise
... # DuplicateKeyError
... try:
... lkp = etl.dictlookupone(table1, 'foo', strict=True)
... except etl.errors.DuplicateKeyError as e:
... print(e)
...
duplicate key: 'b'
>>> # compound keys are supported
... table2 = [['foo', 'bar', 'baz'],
... ['a', 1, True],
... ['b', 2, False],
... ['b', 3, True],
... ['b', 3, False]]
>>> lkp = etl.dictlookupone(table2, ('foo', 'bar'))
>>> lkp[('a', 1)]
{'foo': 'a', 'baz': True, 'bar': 1}
>>> lkp[('b', 2)]
{'foo': 'b', 'baz': False, 'bar': 2}
>>> lkp[('b', 3)]
{'foo': 'b', 'baz': True, 'bar': 3}
>>> # data can be loaded into an existing dictionary-like
... # object, including persistent dictionaries created via the
... # shelve module
... import shelve
>>> lkp = shelve.open('example.dat', flag='n')
>>> lkp = etl.dictlookupone(table1, 'foo', lkp)
>>> lkp.close()
>>> lkp = shelve.open('example.dat', flag='r')
>>> lkp['a']
{'foo': 'a', 'bar': 1}
>>> lkp['b']
{'foo': 'b', 'bar': 2}
"""
if dictionary is None:
dictionary = dict()
it = iter(table)
hdr = next(it)
flds = list(map(text_type, hdr))
keyindices = asindices(hdr, key)
assert len(keyindices) > 0, 'no key selected'
getkey = operator.itemgetter(*keyindices)
for row in it:
k = getkey(row)
if strict and k in dictionary:
raise DuplicateKeyError(k)
elif k not in dictionary:
d = asdict(flds, row)
dictionary[k] = d
return dictionary
def dictlookup(table, key, dictionary=None):
"""
Load a dictionary with data from the given table, mapping to dicts. E.g.::
>>> import petl as etl
>>> table1 = [['foo', 'bar'],
... ['a', 1],
... ['b', 2],
... ['b', 3]]
>>> lkp = etl.dictlookup(table1, 'foo')
>>> lkp['a']
[{'foo': 'a', 'bar': 1}]
>>> lkp['b']
[{'foo': 'b', 'bar': 2}, {'foo': 'b', 'bar': 3}]
>>> # compound keys are supported
... table2 = [['foo', 'bar', 'baz'],
... ['a', 1, True],
... ['b', 2, False],
... ['b', 3, True],
... ['b', 3, False]]
>>> lkp = etl.dictlookup(table2, ('foo', 'bar'))
>>> lkp[('a', 1)]
[{'foo': 'a', 'baz': True, 'bar': 1}]
>>> lkp[('b', 2)]
[{'foo': 'b', 'baz': False, 'bar': 2}]
>>> lkp[('b', 3)]
[{'foo': 'b', 'baz': True, 'bar': 3}, {'foo': 'b', 'baz': False, 'bar': 3}]
>>> # data can be loaded into an existing dictionary-like
... # object, including persistent dictionaries created via the
... # shelve module
... import shelve
>>> lkp = shelve.open('example.dat', flag='n')
>>> lkp = etl.dictlookup(table1, 'foo', lkp)
>>> lkp.close()
>>> lkp = shelve.open('example.dat', flag='r')
>>> lkp['a']
[{'foo': 'a', 'bar': 1}]
>>> lkp['b']
[{'foo': 'b', 'bar': 2}, {'foo': 'b', 'bar': 3}]
"""
if dictionary is None:
dictionary = dict()
it = iter(table)
hdr = next(it)
flds = list(map(text_type, hdr))
keyindices = asindices(hdr, key)
assert len(keyindices) > 0, 'no key selected'
getkey = operator.itemgetter(*keyindices)
for row in it:
k = getkey(row)
rec = asdict(flds, row)
if k in dictionary:
# work properly with shelve
l = dictionary[k]
l.append(rec)
dictionary[k] = l
else:
dictionary[k] = [rec]
return dictionary