def setUp(self):
path = '/tmp/indexer/test_event_generic.ix'
try:
os.unlink(path)
except OSError:
pass
fs = FileStorage(path)
self.index = SocialEventList(storage=fs)
try:
uuid_generator.register('user', 1)
except Exception as e:
pass
try:
uuid_generator.register('post', 2)
except Exception as e:
pass
class TestSocialEventList(unittest.TestCase):
def setUp(self):
path = '/tmp/indexer/test_event_generic.ix'
try:
os.unlink(path)
except OSError:
pass
fs = FileStorage(path)
self.index = SocialEventList(storage=fs)
try:
uuid_generator.register('user', 1)
except Exception as e:
pass
try:
uuid_generator.register('post', 2)
except Exception as e:
pass
def test_padding(self):
block_data = uuid_generator.create('user'), uuid_generator.create('post'), 1, 0, 0, '\0'
block = self.index.block_class(*block_data)
paddings = \
[ block._fields_by_name['user_id'].padding
, block._fields_by_name['post_id'].padding
, block._fields_by_name['id'].padding
, block._fields_by_name['prev_p'].padding
, block._fields_by_name['next_p'].padding
, block._fields_by_name['skip'].padding
]
self.assertListEqual([0, 16, 32, 36, 40, 44], paddings)
self.assertEqual(45, len(block))
def test_base(self):
block = uuid_generator.create('user'), uuid_generator.create('post'), 0
block_id = self.index.append(*block)
find_block = self.index.get_block(block_id)
self.assertEqual(find_block.user_id, block[0])
self.assertEqual(find_block.post_id, block[1])
def create_uuids(self, num=1000, node='user', bytes=False):
result = []
for i in xrange(num):
result.append(bytes and uuid_generator.create(node).bytes or uuid_generator.create(node))
return result
def create_index(self, block_num=1000, user_num=10):
users = dict()
for user_id in self.create_uuids(user_num, bytes=True):
users[user_id] = 0
for i in xrange(block_num):
choosed_user = random.choice(users.keys())
block = choosed_user, uuid_generator.create('post').bytes, users[choosed_user]
block_id = self.index.append(*block, bytes=True)
users[choosed_user] = block_id
return users
def test_by_back_ref(self):
entries = 1000
users = 10
self.create_index(entries, users)
item = item_initial = self.index.end_iterator(1).next()
scans = 0
while item.prev_p:
scans += 1
item = self.index.get_block(item.prev_p)
self.assertEqual(item_initial.user_id, item.user_id)
self.assertNotEqual(item_initial.id, item.id)
self.assertLess(scans, 1000)
def _test_by_back_ref_performance_1(self):
entries = 10000
users = 10
self.create_index(entries, users)
def do():
item = self.index.end_iterator(1).next()
scans = 0
start_time = time.time()
while item.prev_p:
scans += 1
item = self.index.get_block(item.prev_p)
print 'scans: %d' % scans
print '%.6f' % (time.time() - start_time)
p = profile.Profile()
p.runcall(do)
p.print_stats()
def test_consistence(self):
entries = 1000
users = 10
self.create_index(entries, users)
check_user = None
item_prev_prev_p = None
for item in self.index.end_iterator(1):
# if item.prev_p:
# print item
# print item.prev_p
if not check_user:
check_user = item.user_id
if item.user_id == check_user:
# print item.id, item.user_id, item.prev_p
if item_prev_prev_p is not None:
self.assertEqual(item_prev_prev_p, item.id)
item_prev_prev_p = item.prev_p
self.assertNotEqual(item_prev_prev_p, None)
def test_by_search_list_algo(self):
u1 = uuid_generator.create('user').bytes
u2 = uuid_generator.create('user').bytes
u3 = uuid_generator.create('user').bytes
p1 = uuid_generator.create('post').bytes
p2 = uuid_generator.create('post').bytes
p3 = uuid_generator.create('post').bytes
p4 = uuid_generator.create('post').bytes
p5 = uuid_generator.create('post').bytes
p6 = uuid_generator.create('post').bytes
p7 = uuid_generator.create('post').bytes
p8 = uuid_generator.create('post').bytes
p9 = uuid_generator.create('post').bytes
self.index.append(u1, p1, 0, bytes=True)
self.index.append(u2, p2, 0, bytes=True)
self.index.append(u3, p3, 0, bytes=True)
self.index.append(u1, p4, 1, bytes=True)
self.index.append(u2, p5, 2, bytes=True)
self.index.append(u3, p6, 3, bytes=True)
self.index.append(u3, p7, 6, bytes=True)
self.index.append(u2, p8, 5, bytes=True)
self.index.append(u2, p9, 8, bytes=True)
user_last_post = \
[ [u1, 4]
, [u2, 9]
, [u3, 7]
]
user_last_post.sort(key=lambda v: v[1], reverse=True)
user_last_post_len = len(user_last_post)
result = list()
while user_last_post_len:
b = self.index.get_block(user_last_post[0][1])
result.append(b.id)
if not b.prev_p:
del user_last_post[0]
user_last_post_len -= 1
continue
user_last_post[0][1] = b.prev_p
if user_last_post_len > 1 and b.prev_p < user_last_post[1][1]:
user_last_post.sort(key=lambda v: v[1], reverse=True)
self.assertListEqual([9, 8, 7, 6, 5, 4, 3, 2, 1], result)
def test_sorting_dict(self):
d = dict()
keys = range(10000)
while keys:
choice = random.choice(keys)
d[choice] = uuid_generator.create('user')
keys.remove(choice)
self.assertListEqual(d.keys(), sorted(range(10000)))
def test_by_search_sortedblist_algo(self):
u1 = uuid_generator.create('user').bytes
u2 = uuid_generator.create('user').bytes
u3 = uuid_generator.create('user').bytes
p1 = uuid_generator.create('post').bytes
p2 = uuid_generator.create('post').bytes
p3 = uuid_generator.create('post').bytes
p4 = uuid_generator.create('post').bytes
p5 = uuid_generator.create('post').bytes
p6 = uuid_generator.create('post').bytes
p7 = uuid_generator.create('post').bytes
p8 = uuid_generator.create('post').bytes
p9 = uuid_generator.create('post').bytes
self.index.append(u1, p1, 0, bytes=True)
self.index.append(u2, p2, 0, bytes=True)
self.index.append(u3, p3, 0, bytes=True)
self.index.append(u1, p4, 1, bytes=True)
self.index.append(u2, p5, 2, bytes=True)
self.index.append(u3, p6, 3, bytes=True)
self.index.append(u3, p7, 6, bytes=True)
self.index.append(u2, p8, 5, bytes=True)
self.index.append(u2, p9, 8, bytes=True)
user_last_post = blist.sortedlist([9, 7, 4])
result = list()
while user_last_post:
b = self.index.get_block(user_last_post.pop())
result.append(b.id)
b.prev_p and user_last_post.add(b.prev_p)
self.assertListEqual([9, 8, 7, 6, 5, 4, 3, 2, 1], result)
def test_perfomance_sorted_blist(self):
users = self.create_index(1000000, 100000)
friends = blist.sortedlist()
for i in xrange(500):
friends.add(users.pop(random.choice(users.keys())))
result = list()
i = 0
skip, limit = 1000, 10
start_time = time.time()
while friends and i < (skip + limit):
b = self.index.get_block(friends.pop())
if i > skip:
result.append(b.id)
b.prev_p and friends.add(b.prev_p)
i += 1
print '%.6f' % (time.time() - start_time)
print result
