def load_hashmap(self):
# Create redis storage adapter
# need to start redis service
redis_object = Redis(host='localhost', port=6379, db=14)
redis_storage = RedisStorage(redis_object)
try:
config = redis_storage.load_hash_configuration('test')
lshash = RandomBinaryProjections(None, None)
lshash.apply_config(config)
except:
# Config is not existing, create hash from scratch, with 10 projections
lshash = RandomBinaryProjections('test', 10)
nearest = NearestFilter(self.nn)
# self.engine = Engine(feature_size, lshashes=[], vector_filters=[])
self.engine = Engine(self.feature_size,
lshashes=[lshash],
vector_filters=[nearest],
storage=redis_storage,
distance=CosineDistance())
# Do some stuff like indexing or querying with the engine...
# Finally store hash configuration in redis for later use
redis_storage.store_hash_configuration(lshash)
def loadHashmap(self, feature_size=129, result_n=1000): #这里参数没有用到
'''
feature_size: hash空间维数大小
result_n :返回多少个最近邻
'''
# Create redis storage adapter
redis_object = Redis(host='localhost', port=6379, db=0)
redis_storage = RedisStorage(redis_object)
try:
# Get hash config from redis
config = redis_storage.load_hash_configuration('test')
# Config is existing, create hash with None parameters
lshash = RandomBinaryProjections(None, None)
# Apply configuration loaded from redis
lshash.apply_config(config)
except:
# Config is not existing, create hash from scratch, with 10 projections
lshash = RandomBinaryProjections('test', 0)
# Create engine for feature space of 100 dimensions and use our hash.
# This will set the dimension of the lshash only the first time, not when
# using the configuration loaded from redis. Use redis storage to store
# buckets.
nearest = NearestFilter(result_n)
#self.engine = Engine(feature_size, lshashes=[], vector_filters=[])
self.engine = Engine(feature_size, lshashes=[lshash], vector_filters=[nearest], storage=redis_storage, distance=EuclideanDistance())
# Do some stuff like indexing or querying with the engine...
# Finally store hash configuration in redis for later use
redis_storage.store_hash_configuration(lshash)
def __init__(self):
redis_object = redis.Redis(host='localhost', port=6379, db=0)
redis_storage = RedisStorage(redis_object)
# Get hash config from redis
config = redis_storage.load_hash_configuration('MyHash')
if config is None:
# Config is not existing, create hash from scratch, with 5 projections
self.lshash = RandomBinaryProjections('MyHash', 5)
else:
# Config is existing, create hash with None parameters
self.lshash = RandomBinaryProjections(None, None)
# Apply configuration loaded from redis
self.lshash.apply_config(config)
# print("HERE")
# Create engine for feature space of 100 dimensions and use our hash.
# This will set the dimension of the lshash only the first time, not when
# using the configuration loaded from redis. Use redis storage to store
# buckets.
self.engine = Engine(4, lshashes=[self.lshash], storage=redis_storage)
redis_storage.store_hash_configuration(self.lshash)
class NearestNeighbourFinder(object):
"""
Using an already-projected corpus in the form of a VectorCorpus, allow easy queries to find nearest
neighbour event chains in the corpus, given a new event chain.
"""
def __init__(self, model_type, model_name, hash, corpus_path, with_events=False):
self.hash = hash
self.corpus_path = corpus_path
self.model_type = model_type
self.model_name = model_name
self.with_events = with_events
self.model = None
self.search_engine = None
def init_engine(self, redis_port=6379):
# Need to load the model to get information about it
self.model = NarrativeChainModel.load_by_type(self.model_type, self.model_name)
vector_size = self.model.vector_size
# Point the Redis server to the model's database
db_filename = "vectors.rdb"
model_dir = self.model.get_model_directory(self.model_name)
# Prepare an engine for reading vectors from
try:
redis = Redis(host='localhost', port=redis_port, db=0)
except ConnectionError, e:
raise RuntimeError("could not connect to redis server on port %s. Is it running? (%s)" % (redis_port, e))
# Set the storage location to be in the model's directory/file
redis.config_set("dbfilename", "vectors.rdb")
redis.config_set("dir", model_dir)
redis_storage = RedisStorage(redis)
self.search_engine = Engine(vector_size, lshashes=[self.hash], storage=redis_storage,
fetch_vector_filters=[UniqueVectorFilter()], vector_filters=[])
def setUp(self):
self.memory = MemoryStorage()
self.redis_object = Redis(host='localhost', port=6379, db=0)
self.redis_storage = RedisStorage(self.redis_object)
def setUp(self):
self.storage = RedisStorage(Redis())
super(RedisStorageTest, self).setUp()
def setUp(self):
self.memory = MemoryStorage()
self.redis_object = Redis()
self.redis_storage = RedisStorage(self.redis_object)
numpy.random.seed(16)
transforms.ToPILImage(),
transforms.Resize(64),
transforms.CenterCrop(64),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
])
dimension = 512
r = redis.Redis(
host='redis',
port=6379,
# charset='utf-8',
# decode_responses=True,
)
redis_storage = RedisStorage(r)
# Get hash config from redis
config = redis_storage.load_hash_configuration('MyHash')
if config is None:
# Config is not existing, create hash from scratch, with 10 projections
lshash = RandomBinaryProjections('MyHash', 50)
else:
# Config is existing, create hash with None parameters
lshash = RandomBinaryProjections(None, None)
# Apply configuration loaded from redis
lshash.apply_config(config)
# Create engine for feature space of 100 dimensions and use our hash.
# This will set the dimension of the lshash only the first time, not when
# using the configuration loaded from redis. Use redis storage to store
# buckets.
import web
from nearpy import Engine
from nearpy.hashes import RandomBinaryProjections
from redis import Redis
from nearpy.storage import RedisStorage
# Dimension of our vector space
dimension = 68
# Create a random binary hash with 10 bits
rbp = RandomBinaryProjections('rbp', 10)
# Create engine with pipeline configuration
redis_storage = RedisStorage(Redis(host='localhost', port=6379, db=0))
engine = Engine(dimension, lshashes=[rbp], storage=redis_storage)
db = web.database(dbn='sqlite', db='sorch.db')
def insertVector(id, vector):
engine.store_vector(vector, id)
def getNN(vector):
return engine.neighbours(vector)
def insertMetadata(id, url, artist, title, length):
db.insert(id=id, url=url, artist=artist, title=title, length=length)
import time
from nearpy.storage import RedisStorage
import files
import pandas as pd
import numpy as np
from tqdm import tqdm
from nearpy import Engine
from nearpy.hashes import RandomBinaryProjections
from redis import Redis
redis_object = Redis(host='localhost', port=6379, db=0)
redis_storage = RedisStorage(redis_object)
if __name__ == '__main__':
batch_size = 128
data = pd.read_csv(files.small_images_classes_features)
config = redis_storage.load_hash_configuration('MyHash')
dimension = 4096
if config is None:
# Config is not existing, create hash from scratch, with 10 projections
lshash = RandomBinaryProjections('MyHash', 10)
else:
# Config is existing, create hash with None parameters
lshash = RandomBinaryProjections(None, None)
# Apply configuration loaded from redis
lshash.apply_config(config)
def setUp(self):
self.memory = MemoryStorage()
self.redis_object = Redis()
self.redis_storage = RedisStorage(self.redis_object)
def redisStorage(host='localhost'):
return RedisStorage(Redis(host=host, port=6379, db=0))
