def Insert(iterable, index, items):
"""
iterable >> Insert(index, items)
Insert item(s) into lists/tuples in iterable.
>>> [(1, 2), (3, 4)] >> Insert(1, 'X') >> Collect()
[(1, 'X', 2), (3, 'X', 4)]
>>> items = ['a', 'b']
>>> [(1, 2), (3, 4)] >> Insert(2, items) >> Collect()
[(1, 2, 'a'), (3, 4, 'b')]
>>> items = [('a', 'b'), ('c', 'd')]
>>> [(1, 2), (3, 4)] >> Insert(1, items) >> Collect()
[(1, 'a', 'b', 2), (3, 'c', 'd', 4)]
>>> from nutsflow import Enumerate
>>> [(1, 2), (3, 4)] >> Insert(0, Enumerate()) >> Collect()
[(0, 1, 2), (1, 3, 4)]
:param iterable iterable iterable: Any iterable over tuples or lists
:param int index: Index at which position items are inserted.
:param iterable|object items: A single object or an iterable over objects.
:return: iterator where items are inserted into the iterable elements.
:rtype: iterator over tuples
"""
items = items if is_iterable(items) else itt.repeat(items)
for elem, item in zip(iterable, items):
elem = list(elem)
head, tail = elem[:index], elem[index:]
yield tuple(head + as_list(item) + tail)
def Append(iterable, items):
"""
iterable >> Append(items)
Append item(s) to lists/tuples in iterable.
>>> [(1, 2), (3, 4)] >> Append('X') >> Collect()
[(1, 2, 'X'), (3, 4, 'X')]
>>> items = ['a', 'b']
>>> [(1, 2), (3, 4)] >> Append(items) >> Collect()
[(1, 2, 'a'), (3, 4, 'b')]
>>> items = [('a', 'b'), ('c', 'd')]
>>> [(1, 2), (3, 4)] >> Append(items) >> Collect()
[(1, 2, 'a', 'b'), (3, 4, 'c', 'd')]
>>> from nutsflow import Enumerate
>>> [(1, 2), (3, 4)] >> Append(Enumerate()) >> Collect()
[(1, 2, 0), (3, 4, 1)]
:param iterable iterable iterable: Any iterable over tuples or lists
:param iterable|object items: A single object or an iterable over objects.
:return: iterator where items are appended to the iterable elements.
:rtype: iterator over tuples
"""
items = items if is_iterable(items) else itt.repeat(items)
for elem, item in zip(iterable, items):
yield tuple(elem) + as_tuple(item)
def __call__(self, x):
"""Return element x and potentially print its value"""
if not self.__should_print(x):
return x
self.cnt = 0 # reset counter
self.time = time.time() # reset timer
fmtfunc = self.fmtfunc
if hasattr(x, 'ndim'): # is it a numpy array?
x = x.tolist() if x.ndim else x.item()
if not fmtfunc:
text = x
elif isinstance(fmtfunc, str):
if isinstance(x, dict):
text = fmtfunc.format(**x)
else:
text = fmtfunc.format(*(x if is_iterable(x) else [x]))
elif hasattr(fmtfunc, '__call__'):
text = fmtfunc(x)
else:
raise ValueError('Invalid format ' + str(fmtfunc))
console(text, end=self.end)
return x
def __rrshift__(self, iterable):
"""Write elements of iterable to file"""
cols = self.columns
iterable = iter(iterable)
for _ in range(self.skipheader):
next(iterable)
for row in iterable:
row = row if is_iterable(row) else [row]
row = [row[i] for i in cols] if cols else row
row = [self.fmtfunc(r) for r in row]
self.writer.writerow(row)
if self.flush:
self.csvfile.flush()
def __init__(self,
filepath,
colnames=None,
fmtfunc=None,
rowname='Row',
**kwargs):
self.fmtfunc = (lambda x: x) if fmtfunc is None else fmtfunc
self.is_functions = is_iterable(self.fmtfunc)
self.csvfile = open(filepath, 'r')
stripped = (r.strip() for r in self.csvfile)
self.reader = csv.reader(stripped, **kwargs)
header = next(self.reader)
colnames = header if colnames is None else colnames
self.cols = [header.index(n) for n in colnames]
self.Row = namedtuple(rowname, colnames)
def FlattenCol(iterable, cols):
"""
iterable >> FlattenCol(cols)
Flattens the specified columns of the tuples/iterables within the iterable.
Only one level is flattened.
(1 3) (5 7)
(2 4) (6 8) >> FlattenCol((0,1) >> (1 3) (2 4) (5 7) (6 8)
If a column contains a single element (instead of an iterable) it is
wrapped into a repeater. This allows to flatten columns that are iterable
together with non-iterable columns, e.g.
(1 3) (6 7)
(2 ) ( 8) >> FlattenCols((0,1) >> (1 3) (2 3) (6 7) (6 8)
>>> from nutsflow import Collect
>>> data = [([1, 2], [3, 4]), ([5, 6], [7, 8])]
>>> data >> FlattenCol(0) >> Collect()
[(1,), (2,), (5,), (6,)]
>>> data >> FlattenCol((0, 1)) >> Collect()
[(1, 3), (2, 4), (5, 7), (6, 8)]
>>> data >> FlattenCol((1, 0)) >> Collect()
[(3, 1), (4, 2), (7, 5), (8, 6)]
>>> data >> FlattenCol((1, 1, 0)) >> Collect()
[(3, 3, 1), (4, 4, 2), (7, 7, 5), (8, 8, 6)]
>>> data = [([1, 2], 3), (6, [7, 8])]
>>> data >> FlattenCol((0, 1)) >> Collect()
[(1, 3), (2, 3), (6, 7), (6, 8)]
:param iterable iterable: Any iterable.
:params int|tuple columns: Column index or indices
:return: Flattened columns of iterable
:rtype: generator
"""
cols = as_tuple(cols)
get = lambda e: e if is_iterable(e) else itt.repeat(e)
for es in iterable:
for e in zip(*[get(es[c]) for c in cols]):
yield e
def Format(x, fmt):
"""
iterable >> Format(fmt)
Return input as formatted string. For format definition see:
https://docs.python.org/2/library/string.html
>>> from nutsflow import Collect
>>> [1, 2, 3] >> Format('num:{}') >> Collect()
['num:1', 'num:2', 'num:3']
>>> [(1, 2), (3, 4)] >> Format('{0}:{1}') >> Collect()
['1:2', '3:4']
:param iterable iterable: Any iterable
:param string fmt: Formatting string, e.g. '{:02d}'
:return: Returns inputs as strings formatted as specified
:rtype: str
"""
return fmt.format(*(x if is_iterable(x) else [x]))
def test_is_iterable():
assert is_iterable([1, 2])
assert not is_iterable('12')
def __init__(self,
filepath,
columns=None,
skipheader=0,
fmtfunc=None,
**kwargs):
"""
ReadCSV(filepath, columns, skipheader, fmtfunc, **kwargs)
Read data in Comma Separated Format (CSV) from file.
See also CSVWriter.
Can also read Tab Separated Format (TSV) be providing the
corresponding delimiter. Note that in the docstring below
delimiter is '\\t' but in code it should be '\t'.
>>> from nutsflow import Collect
>>> filepath = 'tests/data/data.csv'
>>> with ReadCSV(filepath, skipheader=1) as reader:
... reader >> Collect()
[('1', '2', '3'), ('4', '5', '6')]
>>> with ReadCSV(filepath, skipheader=1, fmtfunc=int) as reader:
... reader >> Collect()
[(1, 2, 3), (4, 5, 6)]
>>> fmtfuncs=(int, str, float)
>>> with ReadCSV(filepath, skipheader=1, fmtfunc=fmtfuncs) as reader:
... reader >> Collect()
[(1, '2', 3.0), (4, '5', 6.0)]
>>> with ReadCSV(filepath, (2, 1), 1, int) as reader:
... reader >> Collect()
[(3, 2), (6, 5)]
>>> with ReadCSV(filepath, (2, 1), 1, (str,int)) as reader:
... reader >> Collect()
[('3', 2), ('6', 5)]
>>> with ReadCSV(filepath, 2, 1, int) as reader:
... reader >> Collect()
[3, 6]
>>> filepath = 'tests/data/data.tsv'
>>> with ReadCSV(filepath, skipheader=1, fmtfunc=int,
... delimiter='\\t') as reader:
... reader >> Collect()
[(1, 2, 3), (4, 5, 6)]
:param string filepath: Path to file in CSV format.
:param tuple columns: Indices of the columns to read.
If None all columns are read.
:param int skipheader: Number of header lines to skip.
:param tuple|function fmtfunc: Function or functions to apply to the
column elements of each row.
:param kwargs kwargs: Keyword arguments for Python's CSV reader.
See https://docs.python.org/2/library/csv.html
"""
self.csvfile = open(filepath, 'r')
self.columns = columns if columns is None else as_tuple(columns)
self.fmtfunc = (lambda x: x) if fmtfunc is None else fmtfunc
self.is_functions = is_iterable(self.fmtfunc)
for _ in range(skipheader):
next(self.csvfile)
itf.take(self.csvfile, skipheader)
stripped = (r.strip() for r in self.csvfile)
self.reader = csv.reader(stripped, **kwargs)