def serialize_and_get(value):
pointless.serialize(value, 'deleteme.map')
try:
return pointless.Pointless('deleteme.map').GetRoot()
finally:
os.unlink('deleteme.map')
def testSerialize(self): fname = 'test_serialize.map' for v in SimpleSerializeTestCases(): pointless.serialize(v, fname) root = pointless.Pointless(fname).GetRoot() del root
def testThreadOpenClose(self): n_files = 1 n_threads = 32 n_iter = 1000 V = [None] V += range(10000) fnames = ['out.%i.map' % (i,) for i in xrange(n_files)] for fname in fnames: pointless.serialize(V, fname) p_list = [pointless.Pointless(fname) for i in xrange(n_files)] s_list = [threading.Event() for i in xrange(n_threads)] def thread_func(e): c = 0 for i in xrange(n_iter): j = random.randint(0, len(p_list) - 1) c += len(p_list[j].GetRoot()) if random.randint(0, 10) == 0: p_list[j] = pointless.Pointless(fnames[j]) e.set() for i in xrange(n_threads): thread.start_new_thread(thread_func, (s_list[i],)) for i in xrange(n_threads): s_list[i].wait()
def serialize_and_get(value):
pointless.serialize(value, 'deleteme.map')
try:
return pointless.Pointless('deleteme.map').GetRoot()
finally:
os.unlink('deleteme.map')
def testMap(self):
maps = [
{ 1: 0 },
{ 2: 'asdf' },
{ 'asdf':2 },
{
1: 2,
2: 3,
4: 4,
'asdf': 123,
(1,2): [4,5,'hello world']
}
]
maps[3][(1,2)].append(maps)
fname = 'test_map.map'
for test_i, m in enumerate(maps):
pointless.serialize(m, fname)
root_a = pointless.Pointless(fname).GetRoot()
for k in m.iterkeys():
self.assert_(k in root_a)
v_m = m[k]
v_a = root_a[k]
self.assert_(pointless.pointless_cmp(v_m, v_a) == 0)
del root_a
def testSerialize(self):
fname = 'test_serialize.map'
for v in SimpleSerializeTestCases():
pointless.serialize(v, fname)
root = pointless.Pointless(fname).GetRoot()
del root
def VectorSlices(fname, py_vector):
pointless.serialize(py_vector, fname)
po_vector = pointless.Pointless(fname).GetRoot()
# first, the original/pointless vector
yield (py_vector, po_vector)
# then further slices of those two
for i in six.moves.range(100):
i_0 = random.randint(-100, len(py_vector) * 2)
i_1 = random.randint(-100, len(py_vector) * 2)
s_py_vector = py_vector[i_0:i_1]
s_po_vector = po_vector[i_0:i_1]
yield s_py_vector, s_po_vector
# and further slices of those
for j in six.moves.range(100):
ii_0 = random.randint(-100, len(py_vector) // 2)
ii_1 = random.randint(-100, len(py_vector) // 2)
ss_py_vector = s_py_vector[ii_0:ii_1]
ss_po_vector = s_po_vector[ii_0:ii_1]
yield ss_py_vector, ss_po_vector
def VectorSlices(fname, py_vector): pointless.serialize(py_vector, fname) po_vector = pointless.Pointless(fname).GetRoot() # first, the original/pointless vector yield (py_vector, po_vector) # then further slices of those two for i in xrange(100): i_0 = random.randint(-100, len(py_vector) * 2) i_1 = random.randint(-100, len(py_vector) * 2) s_py_vector = py_vector[i_0:i_1] s_po_vector = po_vector[i_0:i_1] yield s_py_vector, s_po_vector # and further slices of those for j in xrange(100): ii_0 = random.randint(-100, len(py_vector) // 2) ii_1 = random.randint(-100, len(py_vector) // 2) ss_py_vector = s_py_vector[ii_0:ii_1] ss_po_vector = s_po_vector[ii_0:ii_1] yield ss_py_vector, ss_po_vector
def testThreadOpenClose(self): n_files = 1 n_threads = 32 n_iter = 1000 V = [None] V += range(10000) fnames = ['out.%i.map' % (i,) for i in six.moves.range(n_files)] for fname in fnames: pointless.serialize(V, fname) p_list = [pointless.Pointless(fname) for i in six.moves.range(n_files)] s_list = [threading.Event() for i in six.moves.range(n_threads)] def thread_func(e): c = 0 for i in six.moves.range(n_iter): j = random.randint(0, len(p_list) - 1) c += len(p_list[j].GetRoot()) if random.randint(0, 10) == 0: p_list[j] = pointless.Pointless(fnames[j]) e.set() for i in six.moves.range(n_threads): thread.start_new_thread(thread_func, (s_list[i],)) for i in six.moves.range(n_threads): s_list[i].wait()
def testCompression(self):
# None, or compressed vector type -> values in vector
vm = [
('i8', range(-128, 128)),
('u8', range(0, 256)),
('i16', range(-600, 600)),
('u16', range(0, 1200)),
('i32', range(-1, 33000)),
('u32', range(0, 66000)),
('f', [0.0, 1.0]),
(None, []),
(None, ['asdf', 'foo'])
]
fname = 'typecode.map'
for tc, v_a in vm:
pointless.serialize(v_a, fname)
v_b = pointless.Pointless(fname).GetRoot()
if tc == None:
self.assertRaises(ValueError, operator.attrgetter('typecode'), v_b)
else:
self.assert_(v_b.typecode == tc)
del v_b
def testMap(self):
maps = [{
1: 0
}, {
2: 'asdf'
}, {
'asdf': 2
}, {
1: 2,
2: 3,
4: 4,
'asdf': 123,
(1, 2): [4, 5, 'hello world']
}]
maps[-1][(1, 2)].append(maps)
fname = 'test_map.map'
for test_i, m in enumerate(maps):
pointless.serialize(m, fname)
root_a = pointless.Pointless(fname).GetRoot()
for k in six.iterkeys(m):
self.assert_(k in root_a)
v_m = m[k]
v_a = root_a[k]
if not self._contains_dict_or_set(v_m):
self.assert_(pointless.pointless_cmp(v_m, v_a) == 0)
del root_a
def testCompression(self):
# None, or compressed vector type -> values in vector
vm = [
('i8', list(range(-128, 128))),
('u8', list(range(0, 256))),
('i16', list(range(-600, 600))),
('u16', list(range(0, 1200))),
('i32', list(range(-1, 33000))),
('u32', list(range(0, 66000))),
('f', [0.0, 1.0]),
(None, []),
(None, ['asdf', 'foo'])
]
fname = 'typecode.map'
for tc, v_a in vm:
pointless.serialize(v_a, fname)
v_b = pointless.Pointless(fname).GetRoot()
if tc == None:
self.assertRaises(ValueError, operator.attrgetter('typecode'), v_b)
else:
self.assert_(v_b.typecode == tc)
del v_b
def testString(self):
fname = 'test_string.map'
v = ['string', 'value', u'string', six.unichr(1000)]
pointless.serialize(v, fname)
p = pointless.Pointless(fname)
v_ = p.GetRoot()
str(v)
str(v_)
def testString(self): fname = 'test_string.map' v = ['string', 'value', u'string', unichr(1000)] pointless.serialize(v, fname) p = pointless.Pointless(fname) v_ = p.GetRoot() str(v) str(v_)
def generate_data():
random.seed(0)
k_ = []
v_ = []
for i in xrange(1000000):
k = random_value(8)
v = random_value(8)
k_.append(k)
v_.append(v)
pointless.serialize([k_, v_], 'data.map')
def testMapUnicode(self):
v = {'abcdef': 1, u'abc': 2}
fname = 'test_uncode_str_lookup.map'
pointless.serialize(v, fname)
vv = pointless.Pointless(fname).GetRoot()
for a, b in six.iteritems(v):
self.assert_(a in vv)
bb = vv[a]
self.assertEquals(b, bb)
def generate_data(): random.seed(0) k_ = [] v_ = [] for i in xrange(1000000): k = random_value(8) v = random_value(8) k_.append(k) v_.append(v) pointless.serialize([k_, v_], 'data.map')
def testMapUnicode(self):
v = {
'abcdef': 1,
u'abc': 2
}
fname = 'test_uncode_str_lookup.map'
pointless.serialize(v, fname)
vv = pointless.Pointless(fname).GetRoot()
for a, b in v.iteritems():
self.assert_(a in vv)
bb = vv[a]
self.assertEquals(b, bb)
def testEvilCases(self):
for value, comparable in self.bad_case_iter():
pointless.serialize(value, 'file.map')
svalue = pointless.Pointless('file.map').GetRoot()
if comparable:
self.assertEquals(pointless.pointless_cmp(value, svalue), 0)
s_a = str(value)
s_b = str(svalue)
# stringifaction of dicts/set does not have a consistent ordering
if comparable:
if not self._contains_dict_or_set(value):
self.assertEquals(s_a, s_b)
def testPrint(self): fname = 'test_print.map' # for each simple test case for v in itertools.chain(SimpleSerializeTestCases(), AllBitvectorTestCases()): # serialize it pointless.serialize(v, fname) # load it root = pointless.Pointless(fname).GetRoot() # do a Python version of print s_python = cStringIO.StringIO() PythonPrint(root, s_python) # and the Pointless version s_root = cStringIO.StringIO() print >> s_root, root, self.assertEqual(s_root.getvalue(), s_python.getvalue()) del root
def testPrint(self):
fname = 'test_print.map'
# for each simple test case
for v in itertools.chain(SimpleSerializeTestCases(),
AllBitvectorTestCases()):
# serialize it
pointless.serialize(v, fname)
# load it
root = pointless.Pointless(fname).GetRoot()
# do a Python version of print
s_python = six.StringIO()
PythonPrint(root, s_python)
# and the Pointless version
s_root = six.StringIO()
s_root.write(str(root))
self.assertEqual(s_root.getvalue(), s_python.getvalue())
del root
def testMixedVectors(self):
v = [
[1.0, 2.0, 3 ], # 001
[1.0, 2 , 3.0], # 010
[1.0, 2 , 3 ], # 011
[1 , 2.0, 3.0], # 100
[1 , 2.0, 3 ], # 101
[1 , 2 , 3.0] # 110
]
for v_a in v:
pointless.serialize(v_a, 'mixed.map')
v_b = pointless.Pointless('mixed.map').GetRoot()
self.assertEquals(len(v_a), 3)
self.assertEquals(len(v_b), 3)
for a, b in zip(v_a, v_b):
if isinstance(a, six.integer_types) and isinstance(b, six.integer_types):
self.assertEquals(a, b)
else:
self.assertApproximates(a, b, 0.001)
del v_b
def testMixedVectors(self):
v = [
[1.0, 2.0, 3 ], # 001
[1.0, 2 , 3.0], # 010
[1.0, 2 , 3 ], # 011
[1 , 2.0, 3.0], # 100
[1 , 2.0, 3 ], # 101
[1 , 2 , 3.0] # 110
]
for v_a in v:
pointless.serialize(v_a, 'mixed.map')
v_b = pointless.Pointless('mixed.map').GetRoot()
self.assertEquals(len(v_a), 3)
self.assertEquals(len(v_b), 3)
for a, b in zip(v_a, v_b):
if isinstance(a, (int, long)) and isinstance(b, (int, long)):
self.assertEquals(a, b)
else:
self.assertApproximates(a, b, 0.001)
del v_b
def testSet(self): fname_a = 'test_set_contains_a.map' fname_b = 'test_set_contains_b.map' fname_c = 'test_set_contains_c.map' values = [ [], [[]], [[], []], [[[], False], False, False, False], [0], ['hello world'], [0, u'hello'], ['hello world', (1, 2, 3)], [False, True, 1.0, -1.0, 0.0, 1, 200], [0, 1, 2, 3, 4], ['a', 'b', 'c', ['d', 'e', 1.0]], [None, [None]], [range(10), range(100), range(1000), [range(10), ['asdf'] * 10]], [[0]], [[-1]], [[-1.0, 0.0]], [[0, False]], [[0, True]], [range(10, -100, -1), [[range(10)]]] ] # for each set V of values for v in values: # create set A out of it w = self._list_to_tuple(v) pointless.serialize(set(w), fname_a) root_a = pointless.Pointless(fname_a).GetRoot() # for each element in V for i in v: ii = self._list_to_tuple(i) # serialize it to value B pointless.serialize(i, fname_b) root_b = pointless.Pointless(fname_b).GetRoot() # serialize tuple'd value of it to C pointless.serialize(ii, fname_c) root_c = pointless.Pointless(fname_c).GetRoot() # element <- A, C <- A, B <- A, self.assert_(ii in root_a) self.assert_(root_c in root_a) self.assert_(root_b in root_a) del root_b del root_c del root_a
def testSet(self):
fname_a = 'test_set_contains_a.map'
fname_b = 'test_set_contains_b.map'
fname_c = 'test_set_contains_c.map'
values = [[], [[]], [[], []], [[[], False], False, False, False], [0],
['hello world'], [0, u'hello'], ['hello world', (1, 2, 3)],
[False, True, 1.0, -1.0, 0.0, 1, 200], [0, 1, 2, 3, 4],
['a', 'b', 'c', ['d', 'e', 1.0]], [None, [None]],
[
list(range(10)),
list(range(100)),
list(range(1000)), [list(range(10)), ['asdf'] * 10]
], [[0]], [[-1]], [[-1.0, 0.0]], [[0, False]], [[0, True]],
[list(range(10, -100, -1)), [[list(range(10))]]]]
# for each set V of values
for v in values:
# create set A out of it
w = self._list_to_tuple(v)
pointless.serialize(set(w), fname_a)
root_a = pointless.Pointless(fname_a).GetRoot()
# for each element in V
for i in v:
ii = self._list_to_tuple(i)
# serialize it to value B
pointless.serialize(i, fname_b)
root_b = pointless.Pointless(fname_b).GetRoot()
# serialize tuple'd value of it to C
pointless.serialize(ii, fname_c)
root_c = pointless.Pointless(fname_c).GetRoot()
# element <- A, C <- A, B <- A,
self.assert_(ii in root_a)
self.assert_(root_c in root_a)
self.assert_(root_b in root_a)
del root_b
del root_c
del root_a