def setUp(self):
path = '/tmp/indexer/test_event.ix'
try:
os.unlink(path)
except OSError:
pass
fs = FileStorage(path)
class BlockModel(BaseBlockModel):
_fields = \
[ ('created', LongLongField())
, ('user_id', UuidField())
, ('post_id', UuidField())
]
self.block_class = BlockModel
self.index = InfiniteIndex(self.block_class, fs)
try:
uuid_generator.register('user', 1)
except Exception as e:
pass
try:
uuid_generator.register('post', 2)
except Exception as e:
pass
class TestEventBase(unittest.TestCase):
def setUp(self):
path = '/tmp/indexer/test_event.ix'
try:
os.unlink(path)
except OSError:
pass
fs = FileStorage(path)
class BlockModel(BaseBlockModel):
_fields = \
[ ('created', LongLongField())
, ('user_id', UuidField())
, ('post_id', UuidField())
]
self.block_class = BlockModel
self.index = InfiniteIndex(self.block_class, fs)
try:
uuid_generator.register('user', 1)
except Exception as e:
pass
try:
uuid_generator.register('post', 2)
except Exception as e:
pass
def create_uuids(self, num=1000, node='user', bytes=False):
result = []
for i in xrange(num):
result.append(bytes and uuid_generator.create(node).bytes or uuid_generator.create(node))
return result
def create_index(self, users=10000, posts=100000):
users = self.create_uuids(num=users, node='user', bytes=True)
index = self.index
created_initial = int(round(time.time() * 1000))
for i in xrange(posts):
choosed_user = random.choice(users)
index.append(self.block_class(created_initial + i, choosed_user, uuid_generator.create('post').bytes, _bytes=True))
index.storage.commit()
return users, created_initial, created_initial + posts
def create_index2(self, users=10000, posts=10 ** 6):
Block = namedtuple('Block', 'created user_id post_id')
users = self.create_uuids(num=users, node='user', bytes=True)
result = []
created_initial = int(round(time.time() * 1000))
for i in xrange(posts):
choosed_user = random.choice(users)
result.append( Block( created=created_initial + 1
, user_id=choosed_user
, post_id=uuid_generator.create('post').bytes
) )
return users, result
def _test_test_memory(self):
users, index = self.create_index2()
users_to_find = []
for i in xrange(500):
users_to_find.append(random.choice(users))
def search():
count = 0
scanned = 0
start_time = time.time()
for i, item in enumerate(reversed(index)):
scanned += 1
if item.user_id in users_to_find:
memory_usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024
print scanned, '%dM' % memory_usage, item
count += 1
if count > 9:
break
print '%.6f' % (time.time() - start_time)
# p = profile.Profile()
# p.runcall(search)
# p.print_stats()
search()
def _test_base(self):
users, created_initial, created_destination = self.create_index()
#assert size
#self.assertEqual(401024, os.fstat(self.index.storage.handler.fileno()).st_size)
#get 10 users to find posts
users_to_find = []
for i in xrange(500):
users_to_find.append(random.choice(users))
i = 0
start_time = time.time()
for b in self.index.end_iterator(10000):
if i > 10:
break
user_id = b.user_id
if user_id in users_to_find:
print user_id
i += 1
print '%.6f' % (time.time() - start_time)
def test_test1(self):
users, created_initial, created_destination = self.create_index()
#get 10 users to find posts
users_to_find = []
for i in xrange(100):
users_to_find.append(random.choice(users))
def search():
for scanned, b in self.index.storage.find_from_end(self.index.block_size, 8, 16, users_to_find, 10):
print scanned, self.index.block_class.from_bytes(b).values[0]
return scanned
start_time = time.time()
print 'scan: %d' % search()
print 'search time: %.6f' % (time.time() - start_time)
# p = profile.Profile()
# p.runcall(search)
# p.print_stats()
