def test_replace(self): a = Babe().pull(filename='tests/test.csv', name='Test').typedetect() a = a.mapTo(lambda row : [row.foo+1, row.bar*2], fields=['a','b']) buf = StringIO() a.push(stream=buf, format='csv') s = """a,b\n2,4\n4,8\n""" self.assertEquals(buf.getvalue(), s)
def test_replace(self): a = Babe().pull(filename='tests/test.csv', name='Test').typedetect() a = a.mapTo(lambda row: [row.foo + 1, row.bar * 2], fields=['a', 'b']) buf = StringIO() a.push(stream=buf, format='csv') s = """a,b\n2,4\n4,8\n""" self.assertEquals(buf.getvalue(), s)
def test_insert(self): a = Babe().pull(filename='tests/test.csv', name='Test').typedetect() a = a.mapTo(lambda row: row.foo + 1, insert_fields=['fooplus']) s = """foo,bar,f,d,fooplus 1,2,3.2,2010/10/02,2 3,4,1.2,2011/02/02,4 """ self.assertEquals(a.to_string(), s)
def test_tuple(self): a = Babe().pull(filename='tests/test.csv', name='Test').typedetect() a = a.mapTo(lambda obj: obj._replace(foo=obj.foo + 1)) s = """foo,bar,f,d 2,2,3.2,2010/10/02 4,4,1.2,2011/02/02 """ self.assertEquals(a.to_string(), s)
def test_insert(self): a = Babe().pull(filename='tests/test.csv', name='Test').typedetect() a = a.mapTo(lambda row : row.foo+1, insert_fields=['fooplus']) s = """foo,bar,f,d,fooplus 1,2,3.2,2010/10/02,2 3,4,1.2,2011/02/02,4 """ self.assertEquals(a.to_string(), s)
def test_insert(self): a = Babe().pull(filename='tests/test.csv', name='Test').typedetect() a = a.mapTo(lambda row: row.foo + 1, insert_fields=['fooplus']) buf = StringIO() a.push(stream=buf, format='csv') s = """foo,bar,f,d,fooplus 1,2,3.2,2010/10/02,2 3,4,1.2,2011/02/02,4 """ self.assertEquals(buf.getvalue(), s)
def test_tuple(self): a = Babe().pull(filename='tests/test.csv', name='Test').typedetect() a = a.mapTo(lambda obj: obj._replace(foo=obj.foo + 1)) buf = StringIO() a.push(stream=buf, format='csv') s = """foo,bar,f,d 2,2,3.2,2010/10/02 4,4,1.2,2011/02/02 """ self.assertEquals(buf.getvalue(), s)
def test_insert(self): a = Babe().pull(filename='tests/test.csv', name='Test').typedetect() a = a.mapTo(lambda row : row.foo+1, insert_fields=['fooplus']) buf = StringIO() a.push(stream=buf, format='csv') s = """foo,bar,f,d,fooplus 1,2,3.2,2010/10/02,2 3,4,1.2,2011/02/02,4 """ self.assertEquals(buf.getvalue(), s)
def test_tuple(self): a = Babe().pull(filename='tests/test.csv', name='Test').typedetect() a = a.mapTo(lambda obj : obj._replace(foo=obj.foo + 1)) buf = StringIO() a.push(stream=buf, format='csv') s = """foo,bar,f,d 2,2,3.2,2010/10/02 4,4,1.2,2011/02/02 """ self.assertEquals(buf.getvalue(), s)
def test_replace(self): a = Babe().pull(filename='tests/test.csv', name='Test').typedetect() a = a.mapTo(lambda row: [row.foo + 1, row.bar * 2], fields=['a', 'b']) s = """a,b\n2,4\n4,8\n""" self.assertEquals(a.to_string(), s)
def test_replace(self): a = Babe().pull(filename='tests/test.csv', name='Test').typedetect() a = a.mapTo(lambda row : [row.foo+1, row.bar*2], fields=['a','b']) s = """a,b\n2,4\n4,8\n""" self.assertEquals(a.to_string(), s)
def test_gs_load_from_kontagent(self): # export 1 full day bucket = 'bertrandtest' game = 'wordox' day = '20151021' hour = '14' table_name = '{}_{}'.format(game, day) filename = '{}.csv'.format(table_name + hour) result = time.strptime(day + ' ' + hour, '%Y%m%d %H') start_time = datetime(result.tm_year, result.tm_mon, result.tm_mday, result.tm_hour) end_time = start_time + timedelta(hours=1) a = Babe() a = a.pull_kontagent(start_time=start_time, sample_mode=False, end_time=end_time, KT_APPID='869fb4a24faa4c61b702ea137cbe16ad', discard_names=["PointSend"]) a = a.mapTo(decode_data, insert_fields=["decoded_data"]) a = a.filterColumns(keep_fields=v1) a = a.filter(lambda row: uid_type_check(row) is True) a.push(filename=filename, format='csv', delimiter='\t', quotechar='|', encoding='utf8', bucket=bucket, protocol='gs') a.push_bigquery(filename=filename, bucket=bucket, project_id='bigquery-testing-1098', dataset_id='ladata', table_name=table_name, schema=[ { "name": "date", "type": "STRING", "mode": "REQUIRED" }, { "name": "hour", "type": "INTEGER", "mode": "REQUIRED" }, { "name": "time", "type": "TIMESTAMP", "mode": "REQUIRED" }, { "name": "name", "type": "STRING", "mode": "REQUIRED" }, { "name": "uid", "type": "INTEGER" }, { "name": "st1", "type": "STRING" }, { "name": "st2", "type": "STRING" }, { "name": "st3", "type": "STRING" }, { "name": "channel_type", "type": "STRING" }, { "name": "value", "type": "INTEGER" }, { "name": "level", "type": "INTEGER" }, { "name": "recipients", "type": "STRING" }, { "name": "tracking_data", "type": "STRING" }, { "name": "data", "type": "STRING" } ], job_id='{}_{}'.format(start_time, end_time), num_retries=5)