def test_partition(self):
a = Babe().pull(string=self.s, format='csv')
a = a.partition(field='date')
d = {}
a.push(stream_dict=d, format="csv")
self.assertEquals(d['2012-04-04'].getvalue(), 'date,name,value\n2012-04-04,John,1\n2012-04-04,Luke,2\n')
self.assertEquals(d['2012-04-05'].getvalue(), 'date,name,value\n2012-04-05,John,1\n')
def test_partition(self):
a = Babe().pull(string=self.s, format="csv")
a = a.partition(field="date")
d = {}
a.push(stream_dict=d, format="csv")
self.assertEquals(d["2012-04-04"].getvalue(), "date,name,value\n2012-04-04,John,1\n2012-04-04,Luke,2\n")
self.assertEquals(d["2012-04-05"].getvalue(), "date,name,value\n2012-04-05,John,1\n")
def test_partition_s3(self):
a = Babe().pull(stream=StringIO(self.s), format='csv')
a = a.partition(field='date')
a.push(protocol="s3",
bucket="florian-test",
format="csv",
filename_template='foobar/$date.csv.gz')
def test_partition(self):
a = Babe().pull(stream=StringIO(self.s), format='csv')
a = a.partition(field='date')
d = {}
a.push(stream_dict=d, format="csv")
self.assertEquals(
d['2012-04-04'].getvalue(),
'date,name,value\n2012-04-04,John,1\n2012-04-04,Luke,2\n')
self.assertEquals(d['2012-04-05'].getvalue(),
'date,name,value\n2012-04-05,John,1\n')
def test_pushsqlite_partition(self):
a = Babe().pull(string=self.s, format='csv')
a = a.typedetect()
a.push_sql(table='test_table', database_kind='sqlite', database='test.sqlite', drop_table = True, create_table=True)
a = Babe().pull(string=self.s2, format='csv')
a = a.typedetect()
a = a.partition(field='id')
a.push_sql(table='test_table', database_kind='sqlite', database='test.sqlite', delete_partition=True)
b = Babe().pull_sql(database_kind='sqlite', database='test.sqlite', table='test_table')
b = b.sort(field="id")
self.assertEquals(b.to_string(), self.sr)
def test_pushsqlite_partition(self):
a = Babe().pull(stream=StringIO(self.s), format='csv')
a = a.typedetect()
a.push_sql(table='test_table', database_kind='sqlite', database='test.sqlite', drop_table = True, create_table=True)
a = Babe().pull(stream=StringIO(self.s2), format='csv')
a = a.typedetect()
a = a.partition(field='id')
a.push_sql(table='test_table', database_kind='sqlite', database='test.sqlite', delete_partition=True)
b = Babe().pull_sql(database_kind='sqlite', database='test.sqlite', table='test_table')
b = b.sort(field="id")
buf = StringIO()
b.push(stream=buf, format='csv', delimiter=',')
self.assertEquals(buf.getvalue(), self.sr)
def test_pushsqlite_partition(self):
a = Babe().pull(string=self.s, format='csv')
a = a.typedetect()
a.push_sql(table='test_table',
database_kind='sqlite',
database='test.sqlite',
drop_table=True,
create_table=True)
a = Babe().pull(string=self.s2, format='csv')
a = a.typedetect()
a = a.partition(field='id')
a.push_sql(table='test_table',
database_kind='sqlite',
database='test.sqlite',
delete_partition=True)
b = Babe().pull_sql(database_kind='sqlite',
database='test.sqlite',
table='test_table')
b = b.sort(field="id")
self.assertEquals(b.to_string(), self.sr)
def test_pushsqlite_partition(self):
a = Babe().pull(stream=StringIO(self.s), format='csv')
a = a.typedetect()
a.push_sql(table='test_table',
database_kind='sqlite',
database='test.sqlite',
drop_table=True,
create_table=True)
a = Babe().pull(stream=StringIO(self.s2), format='csv')
a = a.typedetect()
a = a.partition(field='id')
a.push_sql(table='test_table',
database_kind='sqlite',
database='test.sqlite',
delete_partition=True)
b = Babe().pull_sql(database_kind='sqlite',
database='test.sqlite',
table='test_table')
b = b.sort(field="id")
buf = StringIO()
b.push(stream=buf, format='csv', delimiter=',')
self.assertEquals(buf.getvalue(), self.sr)
def test_partition_s3(self):
a = Babe().pull(string=self.s, format="csv")
a = a.partition(field="date")
a.push(protocol="s3", bucket="florian-test", format="csv", filename_template="foobar/$date.csv.gz")
def test_partition_s3(self):
a = Babe().pull(stream=StringIO(self.s), format='csv')
a = a.partition(field = 'date')
a.push(protocol="s3", bucket="florian-test", format="csv", filename_template='foobar/$date.csv.gz')
