def test_partition(self): a = Babe().pull(string=self.s, format='csv') a = a.partition(field='date') d = {} a.push(stream_dict=d, format="csv") self.assertEquals(d['2012-04-04'].getvalue(), 'date,name,value\n2012-04-04,John,1\n2012-04-04,Luke,2\n') self.assertEquals(d['2012-04-05'].getvalue(), 'date,name,value\n2012-04-05,John,1\n')
def test_partition(self): a = Babe().pull(string=self.s, format="csv") a = a.partition(field="date") d = {} a.push(stream_dict=d, format="csv") self.assertEquals(d["2012-04-04"].getvalue(), "date,name,value\n2012-04-04,John,1\n2012-04-04,Luke,2\n") self.assertEquals(d["2012-04-05"].getvalue(), "date,name,value\n2012-04-05,John,1\n")
def test_partition_s3(self): a = Babe().pull(stream=StringIO(self.s), format='csv') a = a.partition(field='date') a.push(protocol="s3", bucket="florian-test", format="csv", filename_template='foobar/$date.csv.gz')
def test_partition(self): a = Babe().pull(stream=StringIO(self.s), format='csv') a = a.partition(field='date') d = {} a.push(stream_dict=d, format="csv") self.assertEquals( d['2012-04-04'].getvalue(), 'date,name,value\n2012-04-04,John,1\n2012-04-04,Luke,2\n') self.assertEquals(d['2012-04-05'].getvalue(), 'date,name,value\n2012-04-05,John,1\n')
def test_pushsqlite_partition(self): a = Babe().pull(string=self.s, format='csv') a = a.typedetect() a.push_sql(table='test_table', database_kind='sqlite', database='test.sqlite', drop_table = True, create_table=True) a = Babe().pull(string=self.s2, format='csv') a = a.typedetect() a = a.partition(field='id') a.push_sql(table='test_table', database_kind='sqlite', database='test.sqlite', delete_partition=True) b = Babe().pull_sql(database_kind='sqlite', database='test.sqlite', table='test_table') b = b.sort(field="id") self.assertEquals(b.to_string(), self.sr)
def test_pushsqlite_partition(self): a = Babe().pull(stream=StringIO(self.s), format='csv') a = a.typedetect() a.push_sql(table='test_table', database_kind='sqlite', database='test.sqlite', drop_table = True, create_table=True) a = Babe().pull(stream=StringIO(self.s2), format='csv') a = a.typedetect() a = a.partition(field='id') a.push_sql(table='test_table', database_kind='sqlite', database='test.sqlite', delete_partition=True) b = Babe().pull_sql(database_kind='sqlite', database='test.sqlite', table='test_table') b = b.sort(field="id") buf = StringIO() b.push(stream=buf, format='csv', delimiter=',') self.assertEquals(buf.getvalue(), self.sr)
def test_pushsqlite_partition(self): a = Babe().pull(string=self.s, format='csv') a = a.typedetect() a.push_sql(table='test_table', database_kind='sqlite', database='test.sqlite', drop_table=True, create_table=True) a = Babe().pull(string=self.s2, format='csv') a = a.typedetect() a = a.partition(field='id') a.push_sql(table='test_table', database_kind='sqlite', database='test.sqlite', delete_partition=True) b = Babe().pull_sql(database_kind='sqlite', database='test.sqlite', table='test_table') b = b.sort(field="id") self.assertEquals(b.to_string(), self.sr)
def test_pushsqlite_partition(self): a = Babe().pull(stream=StringIO(self.s), format='csv') a = a.typedetect() a.push_sql(table='test_table', database_kind='sqlite', database='test.sqlite', drop_table=True, create_table=True) a = Babe().pull(stream=StringIO(self.s2), format='csv') a = a.typedetect() a = a.partition(field='id') a.push_sql(table='test_table', database_kind='sqlite', database='test.sqlite', delete_partition=True) b = Babe().pull_sql(database_kind='sqlite', database='test.sqlite', table='test_table') b = b.sort(field="id") buf = StringIO() b.push(stream=buf, format='csv', delimiter=',') self.assertEquals(buf.getvalue(), self.sr)
def test_partition_s3(self): a = Babe().pull(string=self.s, format="csv") a = a.partition(field="date") a.push(protocol="s3", bucket="florian-test", format="csv", filename_template="foobar/$date.csv.gz")
def test_partition_s3(self): a = Babe().pull(stream=StringIO(self.s), format='csv') a = a.partition(field = 'date') a.push(protocol="s3", bucket="florian-test", format="csv", filename_template='foobar/$date.csv.gz')