Ejemplo n.º 1
0
def serialize_and_get(value):
	pointless.serialize(value, 'deleteme.map')

	try:
		return pointless.Pointless('deleteme.map').GetRoot()
	finally:
		os.unlink('deleteme.map')
Ejemplo n.º 2
0
	def testSerialize(self):
		fname = 'test_serialize.map'

		for v in SimpleSerializeTestCases():
			pointless.serialize(v, fname)
			root = pointless.Pointless(fname).GetRoot()
			del root
Ejemplo n.º 3
0
	def testThreadOpenClose(self):
		n_files = 1
		n_threads = 32
		n_iter = 1000

		V = [None]
		V += range(10000)
		fnames = ['out.%i.map' % (i,) for i in xrange(n_files)]

		for fname in fnames:
			pointless.serialize(V, fname)

		p_list = [pointless.Pointless(fname) for i in xrange(n_files)]
		s_list = [threading.Event() for i in xrange(n_threads)]

		def thread_func(e):
			c = 0

			for i in xrange(n_iter):
				j = random.randint(0, len(p_list) - 1)
				c += len(p_list[j].GetRoot())

				if random.randint(0, 10) == 0:
					p_list[j] = pointless.Pointless(fnames[j])

			e.set()

		for i in xrange(n_threads):
			thread.start_new_thread(thread_func, (s_list[i],))

		for i in xrange(n_threads):
			s_list[i].wait()
Ejemplo n.º 4
0
def serialize_and_get(value):
    pointless.serialize(value, 'deleteme.map')

    try:
        return pointless.Pointless('deleteme.map').GetRoot()
    finally:
        os.unlink('deleteme.map')
Ejemplo n.º 5
0
	def testMap(self):
		maps = [
			{ 1: 0 },
			{ 2: 'asdf' },
			{ 'asdf':2 },
			{
				1: 2,
				2: 3,
				4: 4,
				'asdf': 123, 
				(1,2): [4,5,'hello world']
			}
		]

		maps[3][(1,2)].append(maps)

		fname = 'test_map.map'

		for test_i, m in enumerate(maps):
			pointless.serialize(m, fname)
			root_a = pointless.Pointless(fname).GetRoot()

			for k in m.iterkeys():
				self.assert_(k in root_a)

				v_m = m[k]
				v_a = root_a[k]

				self.assert_(pointless.pointless_cmp(v_m, v_a) == 0)

			del root_a
Ejemplo n.º 6
0
    def testSerialize(self):
        fname = 'test_serialize.map'

        for v in SimpleSerializeTestCases():
            pointless.serialize(v, fname)
            root = pointless.Pointless(fname).GetRoot()
            del root
Ejemplo n.º 7
0
def VectorSlices(fname, py_vector):
    pointless.serialize(py_vector, fname)
    po_vector = pointless.Pointless(fname).GetRoot()

    # first, the original/pointless vector
    yield (py_vector, po_vector)

    # then further slices of those two
    for i in six.moves.range(100):
        i_0 = random.randint(-100, len(py_vector) * 2)
        i_1 = random.randint(-100, len(py_vector) * 2)

        s_py_vector = py_vector[i_0:i_1]
        s_po_vector = po_vector[i_0:i_1]

        yield s_py_vector, s_po_vector

        # and further slices of those
        for j in six.moves.range(100):
            ii_0 = random.randint(-100, len(py_vector) // 2)
            ii_1 = random.randint(-100, len(py_vector) // 2)

            ss_py_vector = s_py_vector[ii_0:ii_1]
            ss_po_vector = s_po_vector[ii_0:ii_1]

            yield ss_py_vector, ss_po_vector
Ejemplo n.º 8
0
def VectorSlices(fname, py_vector):
	pointless.serialize(py_vector, fname)
	po_vector = pointless.Pointless(fname).GetRoot()

	# first, the original/pointless vector
	yield (py_vector, po_vector)

	# then further slices of those two
	for i in xrange(100):
		i_0 = random.randint(-100, len(py_vector) * 2)
		i_1 = random.randint(-100, len(py_vector) * 2)

		s_py_vector = py_vector[i_0:i_1]
		s_po_vector = po_vector[i_0:i_1]

		yield s_py_vector, s_po_vector

		# and further slices of those
		for j in xrange(100):
			ii_0 = random.randint(-100, len(py_vector) // 2)
			ii_1 = random.randint(-100, len(py_vector) // 2)

			ss_py_vector = s_py_vector[ii_0:ii_1]
			ss_po_vector = s_po_vector[ii_0:ii_1]

			yield ss_py_vector, ss_po_vector
Ejemplo n.º 9
0
	def testThreadOpenClose(self):
		n_files = 1
		n_threads = 32
		n_iter = 1000

		V = [None]
		V += range(10000)
		fnames = ['out.%i.map' % (i,) for i in six.moves.range(n_files)]

		for fname in fnames:
			pointless.serialize(V, fname)

		p_list = [pointless.Pointless(fname) for i in six.moves.range(n_files)]
		s_list = [threading.Event() for i in six.moves.range(n_threads)]

		def thread_func(e):
			c = 0

			for i in six.moves.range(n_iter):
				j = random.randint(0, len(p_list) - 1)
				c += len(p_list[j].GetRoot())

				if random.randint(0, 10) == 0:
					p_list[j] = pointless.Pointless(fnames[j])

			e.set()

		for i in six.moves.range(n_threads):
			thread.start_new_thread(thread_func, (s_list[i],))

		for i in six.moves.range(n_threads):
			s_list[i].wait()
Ejemplo n.º 10
0
	def testCompression(self):
		# None, or compressed vector type -> values in vector
		vm = [
			('i8',  range(-128, 128)),
			('u8',  range(0, 256)),
			('i16', range(-600, 600)),
			('u16', range(0, 1200)),
			('i32', range(-1, 33000)),
			('u32', range(0, 66000)),
			('f',   [0.0, 1.0]),
			(None,  []),
			(None,  ['asdf', 'foo'])
		]

		fname = 'typecode.map'

		for tc, v_a in vm:
			pointless.serialize(v_a, fname)
			v_b = pointless.Pointless(fname).GetRoot()

			if tc == None:
				self.assertRaises(ValueError, operator.attrgetter('typecode'), v_b)
			else:
				self.assert_(v_b.typecode == tc)

		del v_b
Ejemplo n.º 11
0
    def testMap(self):
        maps = [{
            1: 0
        }, {
            2: 'asdf'
        }, {
            'asdf': 2
        }, {
            1: 2,
            2: 3,
            4: 4,
            'asdf': 123,
            (1, 2): [4, 5, 'hello world']
        }]

        maps[-1][(1, 2)].append(maps)

        fname = 'test_map.map'

        for test_i, m in enumerate(maps):
            pointless.serialize(m, fname)
            root_a = pointless.Pointless(fname).GetRoot()

            for k in six.iterkeys(m):
                self.assert_(k in root_a)

                v_m = m[k]
                v_a = root_a[k]

                if not self._contains_dict_or_set(v_m):
                    self.assert_(pointless.pointless_cmp(v_m, v_a) == 0)

            del root_a
Ejemplo n.º 12
0
	def testCompression(self):
		# None, or compressed vector type -> values in vector
		vm = [
			('i8',  list(range(-128, 128))),
			('u8',  list(range(0, 256))),
			('i16', list(range(-600, 600))),
			('u16', list(range(0, 1200))),
			('i32', list(range(-1, 33000))),
			('u32', list(range(0, 66000))),
			('f',   [0.0, 1.0]),
			(None,  []),
			(None,  ['asdf', 'foo'])
		]

		fname = 'typecode.map'

		for tc, v_a in vm:
			pointless.serialize(v_a, fname)
			v_b = pointless.Pointless(fname).GetRoot()

			if tc == None:
				self.assertRaises(ValueError, operator.attrgetter('typecode'), v_b)
			else:
				self.assert_(v_b.typecode == tc)

		del v_b
Ejemplo n.º 13
0
 def testString(self):
     fname = 'test_string.map'
     v = ['string', 'value', u'string', six.unichr(1000)]
     pointless.serialize(v, fname)
     p = pointless.Pointless(fname)
     v_ = p.GetRoot()
     str(v)
     str(v_)
Ejemplo n.º 14
0
	def testString(self):
		fname = 'test_string.map'
		v = ['string', 'value', u'string', unichr(1000)]
		pointless.serialize(v, fname)
		p = pointless.Pointless(fname)
		v_ = p.GetRoot()
		str(v)
		str(v_)
Ejemplo n.º 15
0
def generate_data():
    random.seed(0)
    k_ = []
    v_ = []

    for i in xrange(1000000):
        k = random_value(8)
        v = random_value(8)
        k_.append(k)
        v_.append(v)

    pointless.serialize([k_, v_], 'data.map')
Ejemplo n.º 16
0
    def testMapUnicode(self):
        v = {'abcdef': 1, u'abc': 2}

        fname = 'test_uncode_str_lookup.map'

        pointless.serialize(v, fname)
        vv = pointless.Pointless(fname).GetRoot()

        for a, b in six.iteritems(v):
            self.assert_(a in vv)
            bb = vv[a]
            self.assertEquals(b, bb)
Ejemplo n.º 17
0
def generate_data():
	random.seed(0)
	k_ = []
	v_ = []

	for i in xrange(1000000):
		k = random_value(8)
		v = random_value(8)
		k_.append(k)
		v_.append(v)

	pointless.serialize([k_, v_], 'data.map')
Ejemplo n.º 18
0
	def testMapUnicode(self):
		v = {
			'abcdef': 1,
			u'abc': 2
		}

		fname = 'test_uncode_str_lookup.map'

		pointless.serialize(v, fname)
		vv = pointless.Pointless(fname).GetRoot()

		for a, b in v.iteritems():
			self.assert_(a in vv)
			bb = vv[a]
			self.assertEquals(b, bb)
Ejemplo n.º 19
0
    def testEvilCases(self):
        for value, comparable in self.bad_case_iter():
            pointless.serialize(value, 'file.map')
            svalue = pointless.Pointless('file.map').GetRoot()

            if comparable:
                self.assertEquals(pointless.pointless_cmp(value, svalue), 0)

            s_a = str(value)
            s_b = str(svalue)

            # stringifaction of dicts/set does not have a consistent ordering
            if comparable:
                if not self._contains_dict_or_set(value):
                    self.assertEquals(s_a, s_b)
Ejemplo n.º 20
0
	def testPrint(self):
		fname = 'test_print.map'

		# for each simple test case
		for v in itertools.chain(SimpleSerializeTestCases(), AllBitvectorTestCases()):
			# serialize it
			pointless.serialize(v, fname)

			# load it
			root = pointless.Pointless(fname).GetRoot()

			# do a Python version of print
			s_python = cStringIO.StringIO()
			PythonPrint(root, s_python)

			# and the Pointless version
			s_root = cStringIO.StringIO()
			print >> s_root, root,

			self.assertEqual(s_root.getvalue(), s_python.getvalue())

			del root
Ejemplo n.º 21
0
    def testPrint(self):
        fname = 'test_print.map'

        # for each simple test case
        for v in itertools.chain(SimpleSerializeTestCases(),
                                 AllBitvectorTestCases()):
            # serialize it
            pointless.serialize(v, fname)

            # load it
            root = pointless.Pointless(fname).GetRoot()

            # do a Python version of print
            s_python = six.StringIO()
            PythonPrint(root, s_python)

            # and the Pointless version
            s_root = six.StringIO()
            s_root.write(str(root))

            self.assertEqual(s_root.getvalue(), s_python.getvalue())

            del root
Ejemplo n.º 22
0
	def testMixedVectors(self):
		v = [
			[1.0, 2.0, 3  ],  # 001
			[1.0, 2  , 3.0],  # 010
			[1.0, 2  , 3  ],  # 011
			[1  , 2.0, 3.0],  # 100
			[1  , 2.0, 3  ],  # 101
			[1  , 2  , 3.0]   # 110
		]

		for v_a in v:
			pointless.serialize(v_a, 'mixed.map')
			v_b = pointless.Pointless('mixed.map').GetRoot()

			self.assertEquals(len(v_a), 3)
			self.assertEquals(len(v_b), 3)

			for a, b in zip(v_a, v_b):
				if isinstance(a, six.integer_types) and isinstance(b, six.integer_types):
					self.assertEquals(a, b)
				else:
					self.assertApproximates(a, b, 0.001)

			del v_b
Ejemplo n.º 23
0
	def testMixedVectors(self):
		v = [
			[1.0, 2.0, 3  ],  # 001
			[1.0, 2  , 3.0],  # 010
			[1.0, 2  , 3  ],  # 011
			[1  , 2.0, 3.0],  # 100
			[1  , 2.0, 3  ],  # 101
			[1  , 2  , 3.0]   # 110
		]

		for v_a in v:
			pointless.serialize(v_a, 'mixed.map')
			v_b = pointless.Pointless('mixed.map').GetRoot()

			self.assertEquals(len(v_a), 3)
			self.assertEquals(len(v_b), 3)

			for a, b in zip(v_a, v_b):
				if isinstance(a, (int, long)) and isinstance(b, (int, long)):
					self.assertEquals(a, b)
				else:
					self.assertApproximates(a, b, 0.001)

			del v_b
Ejemplo n.º 24
0
	def testSet(self):
		fname_a = 'test_set_contains_a.map'
		fname_b = 'test_set_contains_b.map'
		fname_c = 'test_set_contains_c.map'

		values = [
			[], [[]], [[], []],
			[[[], False], False, False, False],
			[0], ['hello world'],
			[0, u'hello'], ['hello world', (1, 2, 3)],
			[False, True, 1.0, -1.0, 0.0, 1, 200],
			[0, 1, 2, 3, 4],
			['a', 'b', 'c', ['d', 'e', 1.0]],
			[None, [None]],
			[range(10), range(100), range(1000), [range(10), ['asdf'] * 10]],
			[[0]], [[-1]], [[-1.0, 0.0]], [[0, False]], [[0, True]],
			[range(10, -100, -1), [[range(10)]]]
		]

		# for each set V of values
		for v in values:
			# create set A out of it
			w = self._list_to_tuple(v)
			pointless.serialize(set(w), fname_a)
			root_a = pointless.Pointless(fname_a).GetRoot()

			# for each element in V
			for i in v:
				ii = self._list_to_tuple(i)

				# serialize it to value B
				pointless.serialize(i, fname_b)
				root_b = pointless.Pointless(fname_b).GetRoot()

				# serialize tuple'd value of it to C
				pointless.serialize(ii, fname_c)
				root_c = pointless.Pointless(fname_c).GetRoot()

				# element <- A, C <- A, B <- A, 
				self.assert_(ii in root_a)
				self.assert_(root_c in root_a)
				self.assert_(root_b in root_a)

				del root_b
				del root_c

			del root_a
Ejemplo n.º 25
0
    def testSet(self):
        fname_a = 'test_set_contains_a.map'
        fname_b = 'test_set_contains_b.map'
        fname_c = 'test_set_contains_c.map'

        values = [[], [[]], [[], []], [[[], False], False, False, False], [0],
                  ['hello world'], [0, u'hello'], ['hello world', (1, 2, 3)],
                  [False, True, 1.0, -1.0, 0.0, 1, 200], [0, 1, 2, 3, 4],
                  ['a', 'b', 'c', ['d', 'e', 1.0]], [None, [None]],
                  [
                      list(range(10)),
                      list(range(100)),
                      list(range(1000)), [list(range(10)), ['asdf'] * 10]
                  ], [[0]], [[-1]], [[-1.0, 0.0]], [[0, False]], [[0, True]],
                  [list(range(10, -100, -1)), [[list(range(10))]]]]

        # for each set V of values
        for v in values:
            # create set A out of it
            w = self._list_to_tuple(v)
            pointless.serialize(set(w), fname_a)
            root_a = pointless.Pointless(fname_a).GetRoot()

            # for each element in V
            for i in v:
                ii = self._list_to_tuple(i)

                # serialize it to value B
                pointless.serialize(i, fname_b)
                root_b = pointless.Pointless(fname_b).GetRoot()

                # serialize tuple'd value of it to C
                pointless.serialize(ii, fname_c)
                root_c = pointless.Pointless(fname_c).GetRoot()

                # element <- A, C <- A, B <- A,
                self.assert_(ii in root_a)
                self.assert_(root_c in root_a)
                self.assert_(root_b in root_a)

                del root_b
                del root_c

            del root_a