def test_csvbititerator_customtimefunc(self):
class UtcTzinfo(datetime.tzinfo):
ZERO = datetime.timedelta(0)
def utcoffset(self, dt):
return UtcTzinfo.ZERO
def dst(self, dt):
return UtcTzinfo.ZERO
def tzname(self, dt):
return "UTC"
def timefunc_utcstr(timeval):
dt = datetime.datetime.strptime(timeval, '%Y-%m-%dT%H:%M:%S')
dt = dt.replace(tzinfo=UtcTzinfo())
return calendar.timegm(dt.timetuple())
reader = csv_bit_reader(StringIO(u"""
1,10,1991-09-02T06:33:20
5,20,1991-09-02T06:35:00
3,41,1991-09-02T06:36:25
10,10485760,1991-09-02T06:36:25
"""),
timefunc=timefunc_utcstr)
rows = list(reader)
self.assertEqual(len(rows), 4)
self.assertEqual(rows[0], (1, 10, 683793200))
self.assertEqual(rows[1], (5, 20, 683793300))
self.assertEqual(rows[2], (3, 41, 683793385))
self.assertEqual(rows[3], (10, 10485760, 683793385))
def test_invalid_input(self):
invalid_inputs = [
# less than 2 columns
u"155",
# invalid row ID
u"a5,155",
# invalid column ID
u"155,a5",
# invalid timestamp
u"155,255,a5",
]
for text in invalid_inputs:
reader = csv_bit_reader(StringIO(text))
self.assertRaises(PilosaError, list, reader)
def test_csv_import(self):
client = self.get_client()
text = u"""
10, 7
10, 5
2, 3
7, 1
"""
reader = csv_bit_reader(StringIO(text))
frame = self.db.frame("importframe")
client.ensure_frame(frame)
client.import_frame(frame, reader)
bq = self.db.batch_query(
frame.bitmap(2),
frame.bitmap(7),
frame.bitmap(10),
)
response = client.query(bq)
target = [3, 1, 5]
self.assertEqual(3, len(response.results))
self.assertEqual(target, [result.bitmap.bits[0] for result in response.results])
def test_csvbititerator(self):
reader = csv_bit_reader(
StringIO(u"""
1,10,683793200
5,20,683793300
3,41,683793385
10,10485760,683793385
"""))
slice_bit_groups = list(batch_bits(reader, 2))
self.assertEqual(3, len(slice_bit_groups))
slice1, batch1 = slice_bit_groups[0]
self.assertEqual(slice1, 0)
self.assertEqual(2, len(list(batch1)))
slice2, batch2 = slice_bit_groups[1]
self.assertEqual(slice2, 0)
self.assertEqual(1, len(list(batch2)))
slice3, batch3 = slice_bit_groups[2]
self.assertEqual(slice3, 10)
self.assertEqual(1, len(list(batch3)))