def test_ftp(self):
babe = Babe()
a = babe.pull(filename='tests/test.csv', name='Test')
a.push(filename='test.csv', protocol='ftp', user=self.user, password=self.password, host='localhost', port=self.port, protocol_early_check= False)
b = babe.pull(filename='test.csv', name='Test', protocol='ftp', user=self.user, password=self.password, host='localhost', port=self.port)
buf = StringIO()
b.push(stream=buf, format='csv')
self.assertEquals(buf.getvalue(), test_csv_content)
def test_multi(self):
a = Babe()
a = a.pull(stream=StringIO(self.s),
format='csv').pull(stream=StringIO(self.s), format='csv')
buf = StringIO()
a.push(stream=buf, format='csv')
self.assertEquals(buf.getvalue(), self.s + self.s)
def test_multi2(self):
a = Babe()
a = a.pull(stream=StringIO(self.s), format='csv').pull(string=self.s, format='csv')
a = a.merge_substreams()
buf = StringIO()
a.push(stream=buf, format='csv')
self.assertEquals(buf.getvalue(), self.s2)
def test_ftp(self):
babe = Babe()
a = babe.pull('tests/test.csv', name='Test')
a.push(filename='test.csv',
protocol='ftp',
host='localhost',
port=self.port)
def test_pull_process(self):
babe = Babe()
a = babe.pull(command=['/bin/ls', '-1', '.'],
source='ls',
fields=['filename'],
format="csv",
encoding='utf8')
a.push(filename='tests/ls.csv')
def test_multi2(self):
a = Babe()
a = a.pull(stream=StringIO(self.s), format='csv').pull(string=self.s,
format='csv')
a = a.merge_substreams()
buf = StringIO()
a.push(stream=buf, format='csv')
self.assertEquals(buf.getvalue(), self.s2)
def test_mail(self):
a = Babe().pull(stream=StringIO(self.s1),
source="Table 1",
format='csv')
a = a.pull(stream=StringIO(self.s2), source="Table 2", format='csv')
a.mail(subject="Test",
recipients="*****@*****.**",
in_body=True)
def test_sort(self):
babe = Babe()
s = "\n".join(["k,v"] + ["%u,%u" % (i, -i) for i in xrange(0, 10001)])
a = babe.pull(string=s, name="test", format="csv")
a = a.typedetect()
a = a.sort(field="v")
a = a.head(n=1)
self.assertEquals(a.to_string(), "k,v\n10000,-10000\n")
def test_zip(self):
babe = Babe()
a = babe.pull(stream=StringIO(self.s), format="csv")
a.push(filename='tests/test.zip')
b = Babe().pull(filename='tests/test.zip')
buf = StringIO()
b.push(stream=buf)
self.assertEquals(buf.getvalue(), self.s)
def test_sort(self):
babe = Babe()
s = '\n'.join(['k,v'] + ['%u,%u' % (i, -i) for i in xrange(0, 10001)])
a = babe.pull(string=s, name='test', format='csv')
a = a.typedetect()
a = a.sort(field='v')
a = a.head(n=1)
self.assertEquals(a.to_string(), 'k,v\n10000,-10000\n')
def test_zip(self):
babe = Babe()
a = babe.pull(stream=StringIO(self.s), format="csv")
a.push(filename='tests/test.zip')
b = Babe().pull(filename='tests/test.zip')
buf = StringIO()
b.push(stream=buf)
self.assertEquals(buf.getvalue(), self.s)
def test_sort(self):
babe = Babe()
s = '\n'.join(['k,v'] + [ '%u,%u' % (i,-i) for i in xrange(0,10001)])
a = babe.pull(string=s, name='test', format='csv')
a = a.typedetect()
a = a.sort(field='v')
a = a.head(n=1)
self.assertEquals(a.to_string(), 'k,v\n10000,-10000\n')
def test_pull_push(self):
babe = Babe()
a = babe.pull('tests/test.csv', name='Test').typedetect()
a = a.map('foo', lambda x: -x).multimap({
'bar': lambda x: x + 1,
'f': lambda f: f / 2
}).sort('foo')
a = a.groupkey('foo', int.__add__, 0, keepOriginal=True)
a.push(filename='tests/test2.csv')
def test_csv_read_write(self):
s = """foo\tbar\tf\td
1\t2\t3.2\t2010/10/02
3\t4\t1.2\t2011/02/02
"""
babe = Babe()
b = babe.pull(string=s, format='csv', name='Test', delimiter='\t')
b.push(filename='tests/files/test2.csv', delimiter='\t')
with open('tests/files/test2.csv') as f:
self.assertEquals(f.read(), s)
def test_csv_read_write_2_default_delimiter_to_string_bug(self):
s = """foo,bar,f,d
1,2,3.2,2010/10/02
3,4,1.2,2011/02/02
"""
babe = Babe()
b = babe.pull(string=s, format='csv', name='Test')
b.push(filename='tests/files/test4.csv')
with open('tests/files/test4.csv') as f:
self.assertEquals(f.read(), s)
def test_sortdiskbased(self):
babe = Babe()
s = '\n'.join(['k,v'] + [ '%u,%u' % (i,-i) for i in xrange(0,100001)])
a = babe.pull(stream=StringIO(s), name='test', format='csv')
a = a.typedetect()
a = a.sort_diskbased(field='v', nsize=10000)
a = a.head(n=1)
buf = StringIO()
a = a.push(stream=buf, format='csv')
self.assertEquals(buf.getvalue(), 'k,v\n100000,-100000\n')
def test_mail(self):
a = Babe().pull(string=self.s1,
source="Table 1",
format='csv')
a = a.pull(string=self.s2,
source="Table 2",
format='csv')
a.mail(subject="Test",
recipients="*****@*****.**",
in_body=True)
def test_sortdiskbased(self):
babe = Babe()
s = '\n'.join(['k,v'] + ['%u,%u' % (i, -i) for i in xrange(0, 100001)])
a = babe.pull(stream=StringIO(s), name='test', format='csv')
a = a.typedetect()
a = a.sort_diskbased(field='v', nsize=10000)
a = a.head(n=1)
buf = StringIO()
a = a.push(stream=buf, format='csv')
self.assertEquals(buf.getvalue(), 'k,v\n100000,-100000\n')
def test_ftp(self):
babe = Babe()
a = babe.pull(filename='tests/files/test.csv',
name='Test')
a.push(filename='test.csv',
protocol='ftp',
user=self.user,
password=self.password,
host='localhost',
port=self.port,
protocol_early_check=False)
b = babe.pull(filename='test.csv',
name='Test',
protocol='ftp',
user=self.user,
password=self.password,
host='localhost',
port=self.port)
self.assertEquals(b.to_string(), test_csv_content)
def test_ftp(self):
babe = Babe()
a = babe.pull(filename='tests/test.csv', name='Test')
a.push(filename='test.csv',
protocol='ftp',
user=self.user,
password=self.password,
host='localhost',
port=self.port,
protocol_early_check=False)
b = babe.pull(filename='test.csv',
name='Test',
protocol='ftp',
user=self.user,
password=self.password,
host='localhost',
port=self.port)
buf = StringIO()
b.push(stream=buf, format='csv')
self.assertEquals(buf.getvalue(), test_csv_content)
def test_ftpzip(self):
babe = Babe()
a = babe.pull(filename='tests/test.csv', name='Test')
a.push(filename='test.csv',
compress='test.zip',
protocol='ftp',
user=self.user,
password=self.password,
host='localhost',
port=self.port,
protocol_early_check=False)
def test_ftpzip(self):
babe = Babe()
a = babe.pull(filename='tests/files/test.csv',
name='Test')
a.push(filename='test.csv',
compress='test.zip',
protocol='ftp',
user=self.user,
password=self.password,
host='localhost',
port=self.port,
protocol_early_check=False)
def test_log(self):
buf = StringIO()
buf2 = StringIO()
babe = Babe()
a = babe.pull(filename='tests/test.csv', source='Test')
a = a.log(logfile=buf)
a.push(stream=buf2, format='csv')
s = """foo,bar,f,d
1,2,3.2,2010/10/02
3,4,1.2,2011/02/02
"""
self.assertEqual(s, buf.getvalue())
self.assertEqual(s, buf2.getvalue())
def test_split(self):
babe = Babe()
s = """a,b
1,3:4
2,7
"""
a = babe.pull(string=s,format='csv',name='Test')
a = a.split(field='b',separator=':')
self.assertEquals(a.to_string(), """a,b
1,3
1,4
2,7
""")
def test_split(self):
babe = Babe()
s = """a,b
1,3:4
2,7
"""
a = babe.pull(string=s, format='csv', name='Test')
a = a.split(field='b', separator=':')
self.assertEquals(a.to_string(), """a,b
1,3
1,4
2,7
""")
def test_log(self):
buf = StringIO()
buf2 = StringIO()
babe = Babe()
a = babe.pull(filename='tests/test.csv', source='Test')
a = a.log(logfile=buf)
a.push(stream=buf2, format='csv')
s = """foo,bar,f,d
1,2,3.2,2010/10/02
3,4,1.2,2011/02/02
"""
self.assertEqual(s, buf.getvalue())
self.assertEqual(s, buf2.getvalue())
def test_log(self):
buf = StringIO()
buf2 = StringIO()
babe = Babe()
a = babe.pull('tests/test.csv', name='Test')
a = a.log(stream=buf)
a.push(stream=buf2, format='csv')
s = """foo bar f d
1 2 3.2 2010/10/02
3 4 1.2 2011/02/02
"""
self.assertEqual(s, buf.getvalue())
self.assertEqual(s, buf2.getvalue())
def test_log(self):
buf = StringIO()
buf2 = StringIO()
babe = Babe()
a = babe.pull("tests/test.csv", name="Test")
a = a.log(stream=buf)
a.push(stream=buf2, format="csv")
s = """foo bar f d
1 2 3.2 2010/10/02
3 4 1.2 2011/02/02
"""
self.assertEqual(s, buf.getvalue())
self.assertEqual(s, buf2.getvalue())
def test_split(self):
babe = Babe()
buf = StringIO("""a,b
1,3:4
2,7
""")
a = babe.pull(stream=buf,format='csv',name='Test')
a = a.split(field='b',separator=':')
buf2 = StringIO()
a.push(stream=buf2, format='csv')
self.assertEquals(buf2.getvalue(), """a,b
1,3
1,4
2,7
""")
def test_split(self):
babe = Babe()
buf = StringIO("""a,b
1,3:4
2,7
""")
a = babe.pull(stream=buf, format='csv', name='Test')
a = a.split(field='b', separator=':')
buf2 = StringIO()
a.push(stream=buf2, format='csv')
self.assertEquals(buf2.getvalue(), """a,b
1,3
1,4
2,7
""")
def test_csv_escape(self):
s = """a\tb\tc
1\tab\t{\\"hello, buzz\\"}
2\tcd\t
"""
class Dialect(csv.Dialect):
lineterminator = '\n'
delimiter = ','
doublequote = False
escapechar = '\\'
quoting = csv.QUOTE_MINIMAL
quotechar = '|'
b = Babe()
b = b.pull(string=s, format='csv', name='Test')
b.push(filename='tests/files/test3.csv', dialect=Dialect)
def test_multi(self):
a = Babe()
a = a.pull(stream=StringIO(self.s), format='csv').pull(stream=StringIO(self.s), format='csv')
buf = StringIO()
a.push(stream=buf, format='csv')
self.assertEquals(buf.getvalue(), self.s+self.s)
def test_zipread(self):
babe = Babe()
a = babe.pull(filename='tests/test_read.zip', name="Test")
buf = StringIO()
a.push(stream=buf, format='csv')
self.assertEquals(buf.getvalue(), test_csv_content)
def test_pull_process(self):
babe = Babe()
a = babe.pull(command=['/bin/ls', '-1', '.'], source='ls', fields=['filename'], format="csv", encoding='utf8')
a.push(filename='tests/ls.csv')
def test_ftpzip(self):
babe = Babe()
a = babe.pull("tests/test.csv", name="Test")
a.push(filename="test.csv", compress="test.zip", protocol="ftp", host="localhost", port=self.port)
def test_writeutf16(self):
babe = Babe()
a = babe.pull("tests/test.csv", name="Test")
a.push(filename="tests/test_utf16.csv", encoding="utf_16")
def test_excel_read_write(self):
babe = Babe()
b = babe.pull(filename='tests/test.xlsx', name='Test2').typedetect()
b = b.mapTo(lambda row: row._replace(Foo=-row.Foo))
b.push(filename='tests/test2.xlsx')
def test_zip(self):
babe = Babe()
a = babe.pull('tests/test.csv', name='Test')
a.push(filename='test.csv', compress='tests/test.zip')
def test_cleanup(self):
babe = Babe()
a = babe.pull('tests/test_badencoded.csv',
utf8_cleanup=True,
name='Test')
a.push(filename='tests/test_badencoded_out.csv')
def test_writeutf16(self):
babe = Babe()
a = babe.pull('tests/test.csv', name='Test')
a.push(filename='tests/test_utf16.csv', encoding='utf_16')
def test_cleanup2(self):
# Test no cleanup
babe = Babe()
a = babe.pull(filename='tests/test_badencoded.csv', name='Test')
a.push(filename='tests/test_badencoded_out2.csv')
def test_ftp(self):
babe = Babe()
a = babe.pull(filename='tests/test.csv', name='Test')
a.push(filename='test.csv', protocol='ftp', user=self.user, password=self.password, host='localhost', port=self.port, protocol_early_check= False)
b = babe.pull(filename='test.csv', name='Test', protocol='ftp', user=self.user, password=self.password, host='localhost', port=self.port)
self.assertEquals(b.to_string(), test_csv_content)
def test_excel_read_write(self):
babe = Babe()
b = babe.pull("tests/test.xlsx", name="Test2").typedetect()
b = b.map("Foo", lambda x: -x)
b.push(filename="tests/test2.xlsx")
def test_zipread(self):
babe = Babe()
a = babe.pull(filename='tests/files/test_read.zip', name="Test")
self.assertEquals(a.to_string(), test_csv_content)
def test_zipread(self):
babe = Babe()
a = babe.pull(filename='tests/test_read.zip', name="Test")
buf = StringIO()
a.push(stream=buf, format='csv')
self.assertEquals(buf.getvalue(), test_csv_content)
def test_zipread(self):
babe = Babe()
a = babe.pull(filename='tests/test_read.zip', name="Test")
self.assertEquals(a.to_string(), test_csv_content)
def test_zip(self):
babe = Babe()
a = babe.pull("tests/test.csv", name="Test")
a.push(filename="test.csv", compress="tests/test.zip")
def test_cleanup(self):
babe = Babe()
a = babe.pull("tests/test_badencoded.csv", utf8_cleanup=True, name="Test")
a.push(filename="tests/test_badencoded_out.csv")
def test_excel_read_write(self):
babe = Babe()
b = babe.pull(filename='tests/test.xlsx', name='Test2').typedetect()
b = b.mapTo(lambda row: row._replace(Foo=-row.Foo))
b.push(filename='tests/test2.xlsx')
def test_cleanup(self):
babe = Babe()
a = babe.pull(filename='tests/files/test_badencoded.csv', utf8_cleanup=True, name='Test')
a.push(filename='tests/files/test_badencoded_out.csv')
def test_excel_read_write(self):
babe = Babe()
b = babe.pull('tests/test.xlsx', name='Test2').typedetect()
b = b.map('Foo', lambda x: -x)
b.push(filename='tests/test2.xlsx')
def test_cleanup2(self):
# Test no cleanup
babe = Babe()
a = babe.pull(filename='tests/files/test_badencoded.csv', name='Test')
a.push(filename='tests/files/test_badencoded_out2.csv')
def test_writeutf16(self):
babe = Babe()
a = babe.pull(filename='tests/files/test.csv', name='Test')
a.push(filename='tests/files/test_utf16.csv', encoding='utf_16')
def test_pull_push(self):
babe = Babe()
a = babe.pull("tests/test.csv", name="Test").typedetect()
a = a.map("foo", lambda x: -x).multimap({"bar": lambda x: x + 1, "f": lambda f: f / 2}).sort("foo")
a = a.groupkey("foo", int.__add__, 0, keepOriginal=True)
a.push(filename="tests/test2.csv")
