class TestHashStorage(unittest.TestCase): def setUp(self): self.memory = MemoryStorage() self.redis_object = Redis() self.redis_storage = RedisStorage(self.redis_object) def test_hash_memory_storage_none_config(self): conf = self.memory.load_hash_configuration('nonexistentHash') self.assertIsNone(conf) def test_hash_memory_storage_rbp(self): hash1 = RandomBinaryProjections('testRBPHash', 10) hash1.reset(100) self.memory.store_hash_configuration(hash1) hash2 = RandomBinaryProjections(None, None) hash2.apply_config(self.memory.load_hash_configuration('testRBPHash')) self.assertEqual(hash1.dim, hash2.dim) self.assertEqual(hash1.hash_name, hash2.hash_name) self.assertEqual(hash1.projection_count, hash2.projection_count) for i in range(hash1.normals.shape[0]): for j in range(hash1.normals.shape[1]): self.assertEqual(hash1.normals[i, j], hash2.normals[i, j]) def test_hash_memory_storage_rdp(self): hash1 = RandomDiscretizedProjections('testRDPHash', 10, 0.1) hash1.reset(100) self.memory.store_hash_configuration(hash1) hash2 = RandomDiscretizedProjections(None, None, None) hash2.apply_config(self.memory.load_hash_configuration('testRDPHash')) self.assertEqual(hash1.dim, hash2.dim) self.assertEqual(hash1.hash_name, hash2.hash_name) self.assertEqual(hash1.bin_width, hash2.bin_width) self.assertEqual(hash1.projection_count, hash2.projection_count) for i in range(hash1.normals.shape[0]): for j in range(hash1.normals.shape[1]): self.assertEqual(hash1.normals[i, j], hash2.normals[i, j]) def test_hash_memory_storage_pcabp(self): train_vectors = numpy.random.randn(10, 100) hash1 = PCABinaryProjections('testPCABPHash', 4, train_vectors) self.memory.store_hash_configuration(hash1) hash2 = PCABinaryProjections(None, None, None) hash2.apply_config(self.memory.load_hash_configuration('testPCABPHash')) self.assertEqual(hash1.dim, hash2.dim) self.assertEqual(hash1.hash_name, hash2.hash_name) self.assertEqual(hash1.projection_count, hash2.projection_count) for i in range(hash1.components.shape[0]): for j in range(hash1.components.shape[1]): self.assertEqual(hash1.components[i, j], hash2.components[i, j]) def test_hash_memory_storage_pcadp(self): train_vectors = numpy.random.randn(10, 100) hash1 = PCADiscretizedProjections('testPCADPHash', 4, train_vectors, 0.1) self.memory.store_hash_configuration(hash1) hash2 = PCADiscretizedProjections(None, None, None, None) hash2.apply_config(self.memory.load_hash_configuration('testPCADPHash')) self.assertEqual(hash1.dim, hash2.dim) self.assertEqual(hash1.hash_name, hash2.hash_name) self.assertEqual(hash1.bin_width, hash2.bin_width) self.assertEqual(hash1.projection_count, hash2.projection_count) for i in range(hash1.components.shape[0]): for j in range(hash1.components.shape[1]): self.assertEqual(hash1.components[i, j], hash2.components[i, j]) def test_hash_redis_storage_none_config(self): conf = self.redis_storage.load_hash_configuration('nonexistentHash') self.assertIsNone(conf) def test_hash_redis_storage_rbp(self): hash1 = RandomBinaryProjections('testRBPHash', 10) hash1.reset(100) self.redis_storage.store_hash_configuration(hash1) hash2 = RandomBinaryProjections(None, None) hash2.apply_config(self.redis_storage.load_hash_configuration('testRBPHash')) self.assertEqual(hash1.dim, hash2.dim) self.assertEqual(hash1.hash_name, hash2.hash_name) self.assertEqual(hash1.projection_count, hash2.projection_count) for i in range(hash1.normals.shape[0]): for j in range(hash1.normals.shape[1]): self.assertEqual(hash1.normals[i, j], hash2.normals[i, j]) def test_hash_redis_storage_rdp(self): hash1 = RandomDiscretizedProjections('testRDPHash', 10, 0.1) hash1.reset(100) self.redis_storage.store_hash_configuration(hash1) hash2 = RandomDiscretizedProjections(None, None, None) hash2.apply_config(self.redis_storage.load_hash_configuration('testRDPHash')) self.assertEqual(hash1.dim, hash2.dim) self.assertEqual(hash1.hash_name, hash2.hash_name) self.assertEqual(hash1.bin_width, hash2.bin_width) self.assertEqual(hash1.projection_count, hash2.projection_count) for i in range(hash1.normals.shape[0]): for j in range(hash1.normals.shape[1]): self.assertEqual(hash1.normals[i, j], hash2.normals[i, j]) def test_hash_redis_storage_pcabp(self): train_vectors = numpy.random.randn(10, 100) hash1 = PCABinaryProjections('testPCABPHash', 4, train_vectors) self.redis_storage.store_hash_configuration(hash1) hash2 = PCABinaryProjections(None, None, None) hash2.apply_config(self.redis_storage.load_hash_configuration('testPCABPHash')) self.assertEqual(hash1.dim, hash2.dim) self.assertEqual(hash1.hash_name, hash2.hash_name) self.assertEqual(hash1.projection_count, hash2.projection_count) for i in range(hash1.components.shape[0]): for j in range(hash1.components.shape[1]): self.assertEqual(hash1.components[i, j], hash2.components[i, j]) def test_hash_redis_storage_pcadp(self): train_vectors = numpy.random.randn(10, 100) hash1 = PCADiscretizedProjections('testPCADPHash', 4, train_vectors, 0.1) self.redis_storage.store_hash_configuration(hash1) hash2 = PCADiscretizedProjections(None, None, None, None) hash2.apply_config(self.redis_storage.load_hash_configuration('testPCADPHash')) self.assertEqual(hash1.dim, hash2.dim) self.assertEqual(hash1.hash_name, hash2.hash_name) self.assertEqual(hash1.bin_width, hash2.bin_width) self.assertEqual(hash1.projection_count, hash2.projection_count) for i in range(hash1.components.shape[0]): for j in range(hash1.components.shape[1]): self.assertEqual(hash1.components[i, j], hash2.components[i, j])
class TestRandomBinaryProjectionTree(unittest.TestCase): def setUp(self): self.memory = MemoryStorage() self.redis_object = Redis(host='localhost', port=6379, db=0) self.redis_storage = RedisStorage(self.redis_object) def test_retrieval(self): # We want 12 projections, 20 results at least rbpt = RandomBinaryProjectionTree('testHash', 12, 20) # Create engine for 100 dimensional feature space, do not forget to set # nearest filter to 20, because default is 10 self.engine = Engine(100, lshashes=[rbpt], vector_filters=[NearestFilter(20)]) # First insert 200000 random vectors #print 'Indexing...' for k in range(200000): x = numpy.random.randn(100) x_data = 'data' self.engine.store_vector(x, x_data) # Now do random queries and check result set size #print 'Querying...' for k in range(10): x = numpy.random.randn(100) n = self.engine.neighbours(x) #print "Candidate count = %d" % self.engine.candidate_count(x) #print "Result size = %d" % len(n) self.assertEqual(len(n), 20) def test_storage_memory(self): # We want 10 projections, 20 results at least rbpt = RandomBinaryProjectionTree('testHash', 10, 20) # Create engine for 100 dimensional feature space self.engine = Engine(100, lshashes=[rbpt], vector_filters=[NearestFilter(20)]) # First insert 2000 random vectors for k in range(2000): x = numpy.random.randn(100) x_data = 'data' self.engine.store_vector(x, x_data) self.memory.store_hash_configuration(rbpt) rbpt2 = RandomBinaryProjectionTree(None, None, None) rbpt2.apply_config(self.memory.load_hash_configuration('testHash')) self.assertEqual(rbpt.dim, rbpt2.dim) self.assertEqual(rbpt.hash_name, rbpt2.hash_name) self.assertEqual(rbpt.projection_count, rbpt2.projection_count) for i in range(rbpt.normals.shape[0]): for j in range(rbpt.normals.shape[1]): self.assertEqual(rbpt.normals[i, j], rbpt2.normals[i, j]) # Now do random queries and check result set size for k in range(10): x = numpy.random.randn(100) keys1 = rbpt.hash_vector(x, querying=True) keys2 = rbpt2.hash_vector(x, querying=True) self.assertEqual(len(keys1), len(keys2)) for k in range(len(keys1)): self.assertEqual(keys1[k], keys2[k]) def test_storage_redis(self): # We want 10 projections, 20 results at least rbpt = RandomBinaryProjectionTree('testHash', 10, 20) # Create engine for 100 dimensional feature space self.engine = Engine(100, lshashes=[rbpt], vector_filters=[NearestFilter(20)]) # First insert 2000 random vectors for k in range(2000): x = numpy.random.randn(100) x_data = 'data' self.engine.store_vector(x, x_data) self.redis_storage.store_hash_configuration(rbpt) rbpt2 = RandomBinaryProjectionTree(None, None, None) rbpt2.apply_config(self.redis_storage.load_hash_configuration('testHash')) self.assertEqual(rbpt.dim, rbpt2.dim) self.assertEqual(rbpt.hash_name, rbpt2.hash_name) self.assertEqual(rbpt.projection_count, rbpt2.projection_count) for i in range(rbpt.normals.shape[0]): for j in range(rbpt.normals.shape[1]): self.assertEqual(rbpt.normals[i, j], rbpt2.normals[i, j]) # Now do random queries and check result set size for k in range(10): x = numpy.random.randn(100) keys1 = rbpt.hash_vector(x, querying=True) keys2 = rbpt2.hash_vector(x, querying=True) self.assertEqual(len(keys1), len(keys2)) for k in range(len(keys1)): self.assertEqual(keys1[k], keys2[k])
class TestRandomBinaryProjectionTree(unittest.TestCase): def setUp(self): self.memory = MemoryStorage() self.redis_object = Redis() self.redis_storage = RedisStorage(self.redis_object) numpy.random.seed(16) def test_retrieval(self): # We want 12 projections, 20 results at least rbpt = RandomBinaryProjectionTree('testHash', 12, 20) # Create engine for 100 dimensional feature space, do not forget to set # nearest filter to 20, because default is 10 self.engine = Engine(100, lshashes=[rbpt], vector_filters=[NearestFilter(20)]) # First insert 200000 random vectors for k in range(200000): x = numpy.random.randn(100) x_data = 'data {}'.format(k) self.engine.store_vector(x, x_data) # Now do random queries and check result set size for k in range(10): x = numpy.random.randn(100) n = self.engine.neighbours(x) self.assertEqual(len(n), 20) def test_storage_memory(self): # We want 10 projections, 20 results at least rbpt = RandomBinaryProjectionTree('testHash', 10, 20) # Create engine for 100 dimensional feature space self.engine = Engine(100, lshashes=[rbpt], vector_filters=[NearestFilter(20)]) # First insert 2000 random vectors for k in range(2000): x = numpy.random.randn(100) x_data = 'data' self.engine.store_vector(x, x_data) self.memory.store_hash_configuration(rbpt) rbpt2 = RandomBinaryProjectionTree(None, None, None) rbpt2.apply_config(self.memory.load_hash_configuration('testHash')) self.assertEqual(rbpt.dim, rbpt2.dim) self.assertEqual(rbpt.hash_name, rbpt2.hash_name) self.assertEqual(rbpt.projection_count, rbpt2.projection_count) for i in range(rbpt.normals.shape[0]): for j in range(rbpt.normals.shape[1]): self.assertEqual(rbpt.normals[i, j], rbpt2.normals[i, j]) # Now do random queries and check result set size for k in range(10): x = numpy.random.randn(100) keys1 = rbpt.hash_vector(x, querying=True) keys2 = rbpt2.hash_vector(x, querying=True) self.assertEqual(len(keys1), len(keys2)) for k in range(len(keys1)): self.assertEqual(keys1[k], keys2[k]) def test_storage_redis(self): # We want 10 projections, 20 results at least rbpt = RandomBinaryProjectionTree('testHash', 10, 20) # Create engine for 100 dimensional feature space self.engine = Engine(100, lshashes=[rbpt], vector_filters=[NearestFilter(20)]) # First insert 2000 random vectors for k in range(2000): x = numpy.random.randn(100) x_data = 'data' self.engine.store_vector(x, x_data) self.redis_storage.store_hash_configuration(rbpt) rbpt2 = RandomBinaryProjectionTree(None, None, None) rbpt2.apply_config( self.redis_storage.load_hash_configuration('testHash')) self.assertEqual(rbpt.dim, rbpt2.dim) self.assertEqual(rbpt.hash_name, rbpt2.hash_name) self.assertEqual(rbpt.projection_count, rbpt2.projection_count) for i in range(rbpt.normals.shape[0]): for j in range(rbpt.normals.shape[1]): self.assertEqual(rbpt.normals[i, j], rbpt2.normals[i, j]) # Now do random queries and check result set size for k in range(10): x = numpy.random.randn(100) keys1 = rbpt.hash_vector(x, querying=True) keys2 = rbpt2.hash_vector(x, querying=True) self.assertEqual(len(keys1), len(keys2)) for k in range(len(keys1)): self.assertEqual(keys1[k], keys2[k])
class TestHashStorage(unittest.TestCase): def setUp(self): self.memory = MemoryStorage() self.redis_object = Redis(host='localhost', port=6379, db=0) self.redis_storage = RedisStorage(self.redis_object) def test_hash_memory_storage_rbp(self): hash1 = RandomBinaryProjections('testRBPHash', 10) hash1.reset(100) self.memory.store_hash_configuration(hash1) hash2 = RandomBinaryProjections(None, None) hash2.apply_config(self.memory.load_hash_configuration('testRBPHash')) self.assertEqual(hash1.dim, hash2.dim) self.assertEqual(hash1.hash_name, hash2.hash_name) self.assertEqual(hash1.projection_count, hash2.projection_count) for i in range(hash1.normals.shape[0]): for j in range(hash1.normals.shape[1]): self.assertEqual(hash1.normals[i, j], hash2.normals[i, j]) def test_hash_memory_storage_rdp(self): hash1 = RandomDiscretizedProjections('testRDPHash', 10, 0.1) hash1.reset(100) self.memory.store_hash_configuration(hash1) hash2 = RandomDiscretizedProjections(None, None, None) hash2.apply_config(self.memory.load_hash_configuration('testRDPHash')) self.assertEqual(hash1.dim, hash2.dim) self.assertEqual(hash1.hash_name, hash2.hash_name) self.assertEqual(hash1.bin_width, hash2.bin_width) self.assertEqual(hash1.projection_count, hash2.projection_count) for i in range(hash1.normals.shape[0]): for j in range(hash1.normals.shape[1]): self.assertEqual(hash1.normals[i, j], hash2.normals[i, j]) def test_hash_memory_storage_pcabp(self): train_vectors = numpy.random.randn(10, 100) hash1 = PCABinaryProjections('testPCABPHash', 4, train_vectors) self.memory.store_hash_configuration(hash1) hash2 = PCABinaryProjections(None, None, None) hash2.apply_config(self.memory.load_hash_configuration('testPCABPHash')) self.assertEqual(hash1.dim, hash2.dim) self.assertEqual(hash1.hash_name, hash2.hash_name) self.assertEqual(hash1.projection_count, hash2.projection_count) for i in range(hash1.components.shape[0]): for j in range(hash1.components.shape[1]): self.assertEqual(hash1.components[i, j], hash2.components[i, j]) def test_hash_memory_storage_pcadp(self): train_vectors = numpy.random.randn(10, 100) hash1 = PCADiscretizedProjections('testPCADPHash', 4, train_vectors, 0.1) self.memory.store_hash_configuration(hash1) hash2 = PCADiscretizedProjections(None, None, None, None) hash2.apply_config(self.memory.load_hash_configuration('testPCADPHash')) self.assertEqual(hash1.dim, hash2.dim) self.assertEqual(hash1.hash_name, hash2.hash_name) self.assertEqual(hash1.bin_width, hash2.bin_width) self.assertEqual(hash1.projection_count, hash2.projection_count) for i in range(hash1.components.shape[0]): for j in range(hash1.components.shape[1]): self.assertEqual(hash1.components[i, j], hash2.components[i, j]) def test_hash_redis_storage_rbp(self): hash1 = RandomBinaryProjections('testRBPHash', 10) hash1.reset(100) self.redis_storage.store_hash_configuration(hash1) hash2 = RandomBinaryProjections(None, None) hash2.apply_config(self.redis_storage.load_hash_configuration('testRBPHash')) self.assertEqual(hash1.dim, hash2.dim) self.assertEqual(hash1.hash_name, hash2.hash_name) self.assertEqual(hash1.projection_count, hash2.projection_count) for i in range(hash1.normals.shape[0]): for j in range(hash1.normals.shape[1]): self.assertEqual(hash1.normals[i, j], hash2.normals[i, j]) def test_hash_redis_storage_rdp(self): hash1 = RandomDiscretizedProjections('testRDPHash', 10, 0.1) hash1.reset(100) self.redis_storage.store_hash_configuration(hash1) hash2 = RandomDiscretizedProjections(None, None, None) hash2.apply_config(self.redis_storage.load_hash_configuration('testRDPHash')) self.assertEqual(hash1.dim, hash2.dim) self.assertEqual(hash1.hash_name, hash2.hash_name) self.assertEqual(hash1.bin_width, hash2.bin_width) self.assertEqual(hash1.projection_count, hash2.projection_count) for i in range(hash1.normals.shape[0]): for j in range(hash1.normals.shape[1]): self.assertEqual(hash1.normals[i, j], hash2.normals[i, j]) def test_hash_redis_storage_pcabp(self): train_vectors = numpy.random.randn(10, 100) hash1 = PCABinaryProjections('testPCABPHash', 4, train_vectors) self.redis_storage.store_hash_configuration(hash1) hash2 = PCABinaryProjections(None, None, None) hash2.apply_config(self.redis_storage.load_hash_configuration('testPCABPHash')) self.assertEqual(hash1.dim, hash2.dim) self.assertEqual(hash1.hash_name, hash2.hash_name) self.assertEqual(hash1.projection_count, hash2.projection_count) for i in range(hash1.components.shape[0]): for j in range(hash1.components.shape[1]): self.assertEqual(hash1.components[i, j], hash2.components[i, j]) def test_hash_redis_storage_pcadp(self): train_vectors = numpy.random.randn(10, 100) hash1 = PCADiscretizedProjections('testPCADPHash', 4, train_vectors, 0.1) self.redis_storage.store_hash_configuration(hash1) hash2 = PCADiscretizedProjections(None, None, None, None) hash2.apply_config(self.redis_storage.load_hash_configuration('testPCADPHash')) self.assertEqual(hash1.dim, hash2.dim) self.assertEqual(hash1.hash_name, hash2.hash_name) self.assertEqual(hash1.bin_width, hash2.bin_width) self.assertEqual(hash1.projection_count, hash2.projection_count) for i in range(hash1.components.shape[0]): for j in range(hash1.components.shape[1]): self.assertEqual(hash1.components[i, j], hash2.components[i, j])