def main():
print "Starting the setOperations stress test"
plist,prefixes,VNH_2_IP,prefixes_announced,part_2_prefix=initializeData()
print part_2_prefix.keys()
r0=Redis(db=0)
for participant in part_2_prefix:
r0.set(participant,part_2_prefix[participant])
r0.bgsave()
print "Starting the set operation with following parameters:"
print "# participants:",len(prefixes_announced['pg1'].keys()),", # prefixes:",len(plist)
start_setops=time.time()
#part_2_prefix_updated=prefix_decompose(part_2_prefix)
print "Execution Time: ",time.time()-start_setops,' seconds'
def main():
print "Starting the setOperations stress test"
plist, prefixes, VNH_2_IP, prefixes_announced, part_2_prefix = initializeData(
)
print part_2_prefix.keys()
r0 = Redis(db=0)
for participant in part_2_prefix:
r0.set(participant, part_2_prefix[participant])
r0.bgsave()
print "Starting the set operation with following parameters:"
print "# participants:", len(
prefixes_announced['pg1'].keys()), ", # prefixes:", len(plist)
start_setops = time.time()
#part_2_prefix_updated=prefix_decompose(part_2_prefix)
print "Execution Time: ", time.time() - start_setops, ' seconds'
class Connection(object):
"""A lightweight connector to Redis database server."""
def __init__(self, host, port, db):
self.host = host
self.port = port
self._db = db
self.redis = Redis(host=self.host, port=self.port, db=self._db)
self.check_connection()
def check_connection(self):
try:
self.redis.randomkey()
logging.info("Connected to Redis[db:%s] on %s:%s" % (self._db, self.host, self.port), exc_info=False)
except:
logging.error("Cannot connect to Redis[db:%s] on %s:%s" % (self._db, self.host, self.port), exc_info=False)
def close(self):
""" Close connection """
self.redis.connection.disconnect()
logging.info("Connection to Redis on %s:%s closed." % (self.host, self.port), exc_info=False)
def __del__(self):
self.close()
def switch_to(self, to_db):
""" Switches the connection from a redis db to another """
""" Closes the connection and establishes it again with a different redis db """
self.redis.connection.disconnect()
self.redis = Redis(host=self.host, port=self.port, db=to_db)
self._db = to_db
self.check_connection()
# Here's a set of function that I think you might need to use frequently.
# You can reach other Redis commands directly from reds.[command] from your Connection subclass.
def get(self, key):
return self.redis.get(key)
def set(self, key, val):
return self.redis.set(key, val)
def save(self):
""" synchronously save the db on disk """
if self.redis.save():
logging.info("Redis[db:%s] saved successfully" % self._db, exc_info=False)
else:
logging.error("Redis[db:%s] was NOT saved successfully" % self._db, exc_info=True)
def bgsave(self):
""" asynchronously save the db on disk """
if self.redis.bgsave():
logging.info("Redis[db:%s] saved successfully" % self._db, exc_info=False)
else:
logging.error("Redis[db:%s] was NOT saved successfully" % self._db, exc_info=True)
def last_save(self):
""" return the unix time stamp of the last successfully saving of the dataset on disk """
return self.redis.lastsave()
class RedisDatabase(BaseDatabase):
def __init__(self,
pickle_protocol=pickle.DEFAULT_PROTOCOL,
namespace=None,
**kwargs):
"""Initialize Redis DB
Args:
pickle_protocol (int, optional): https://docs.python.org/3.7/library/pickle.html#data-stream-format
namespace (str, optional): If set, key names in the Redis DB will
be preceeded by "<namespace>:".
**kwargs (optional): All these will be sent to redis.Redis(). See
https://github.com/andymccurdy/redis-py for more info.
"""
from redis import Redis
super().__init__()
self._pickle_protocol = pickle_protocol
self._namespace = namespace
# Quick check so the number of databases is sufficient
if "db" in kwargs and kwargs["db"] > 0:
redis_kwargs = kwargs.copy()
redis_kwargs["db"] = 0
redis = Redis(**redis_kwargs)
dbs = redis.config_get("databases")
if int(dbs["databases"]) <= kwargs["db"]:
redis.close()
raise ValueError(
f"Tried to open Redis DB #{kwargs['db']}, but there are only {dbs['databases']} databases"
)
redis.close()
self._redis = Redis(**kwargs)
def __setattr__(self, name, value):
if not name.startswith("_") and self._is_open:
if isinstance(value, list):
for key, item in self._schema.items():
if key == name:
value = RedisList(
self._redis,
self.get_redis_key(name),
initlist=value,
overwrite=True,
pickle_protocol=self._pickle_protocol)
break
if isinstance(value,
UserList) and not hasattr(value, "_redis_wrapped"):
value = RedisList.wrap(value,
self._redis,
self.get_redis_key(name),
overwrite=True,
pickle_protocol=self._pickle_protocol)
super().__setattr__(name, value)
def add_key(self, name, type_, default=None, **default_kwargs):
if issubclass(type_, list):
if default is None:
initlist = type_()
else:
initlist = type_(default.copy())
default = None
type_ = RedisList
default_kwargs.update({
"initlist": initlist,
"redis": self._redis,
"key": self.get_redis_key(name),
"pickle_protocol": self._pickle_protocol
})
super().add_key(name, type_, default=default, **default_kwargs)
def get_redis_key(self, name):
return f"{self._namespace}:{name}" if self._namespace else name
def setattr(self, name, value):
if not isinstance(value, (list, UserList)):
self._redis[self.get_redis_key(name)] = pickle.dumps(
value, protocol=self._pickle_protocol)
def close(self):
self.sync()
self._redis.close()
super().close()
def open(self):
for key, item in self._schema.items():
if item.type is RedisList:
setattr(self, key, item.get_default())
elif issubclass(item.type, UserList):
unique = item.default_kwargs.get("unique", False)
setattr(
self, key,
RedisList.wrap(item.get_default(),
self._redis,
self.get_redis_key(key),
unique=unique,
pickle_protocol=self._pickle_protocol))
else:
value = self._redis.get(self.get_redis_key(key))
setattr(
self, key,
item.get_default()
if value is None else pickle.loads(value))
super().open()
def sync(self, key=None):
from redis import ResponseError
for k, item in self._schema.items():
if key is not None and k != key:
continue
if not issubclass(item.type, UserList):
self._redis.set(
self.get_redis_key(k),
pickle.dumps(getattr(self, k),
protocol=self._pickle_protocol))
# Fail silently if another save is already in progress
try:
self._redis.bgsave()
except ResponseError:
pass
class RedisCacheData:
@classmethod
def check_redis_db(cls):
LUMIBOT_USE_REDIS = os.environ.get("LUMIBOT_USE_REDIS")
if LUMIBOT_USE_REDIS and LUMIBOT_USE_REDIS.upper() == "TRUE":
LUMIBOT_REDIS_HOST = os.environ.get("LUMIBOT_REDIS_HOST")
if not LUMIBOT_REDIS_HOST:
LUMIBOT_REDIS_HOST = "localhost"
LUMIBOT_REDIS_PORT = os.environ.get("LUMIBOT_REDIS_PORT")
if not LUMIBOT_REDIS_PORT:
LUMIBOT_REDIS_PORT = 6379
LUMIBOT_REDIS_DB = os.environ.get("LUMIBOT_REDIS_DB")
if not LUMIBOT_REDIS_DB:
LUMIBOT_REDIS_DB = 0
try:
return cls(LUMIBOT_REDIS_HOST, LUMIBOT_REDIS_PORT,
LUMIBOT_REDIS_DB)
except:
logging.error("Could not find the redis db")
return None
return None
@staticmethod
def decode(value):
if isinstance(value, bytes):
value = value.decode(encoding="utf8")
return value
@staticmethod
def parse_redis_value(value):
if isinstance(value, dict):
result = {}
for key, val_raw in value.items():
val = RedisCacheData.parse_redis_value(val_raw)
result[key] = val
return result
try:
return float(value)
except:
return value
@staticmethod
def build_key(source, entity, identifier, subidentifier=""):
key = f"LUMIBOT_{source}_{entity}_{identifier}"
if subidentifier:
key = f"{key}_{subidentifier}"
return key.upper()
@staticmethod
def build_pattern(source="*",
entity="*",
identifier="*",
subidentifier=""):
pattern = f"LUMIBOT_{source}_{entity}_{identifier}"
if source != "*" and entity != "*" and identifier != "*":
return f"{pattern}_*".upper()
if subidentifier and subidentifier != "*":
return f"{pattern}_{subidentifier}".upper()
return patter.upper()
def __init__(self, host="localhost", port=6379, db_number=0):
self.host = host
self.port = port
self.db_number = db_number
self.url = f"redis://{host}:{port}/{db_number}"
self._redis = Redis(host=host,
port=port,
db=db_number,
decode_responses=True)
def store_item(self, item, source, entity, identifier, subidentifier=""):
key = self.build_key(source, entity, identifier, subidentifier)
if isinstance(item, dict):
self._redis.hset(key, mapping=item)
else:
self._redis.set(key, item)
def store_bars(self, bars):
for index, row_raw in bars.df.iterrows():
timestamp = int(index.timestamp())
row = row_raw.to_dict()
row["timestamp"] = timestamp
self.store_item(row, bars.source, "bar", bars.symbol, timestamp)
def retrieve_by_key(self, key):
type_ = self._redis.type(key)
val = None
if type_ == "hash":
raw_val = self._redis.hgetall(key)
val = self.parse_redis_value(raw_val)
elif type == "string":
raw_val = self._redis.get(key)
val = self.parse_redis_value(raw_val)
return val
def retrieve_item(self, source, entity, identifier, subidentifier=""):
key = self.build_key(source, entity, identifier, subidentifier)
return self.retrieve_by_key(key)
def retrieve_store(self, source, symbols, parser=None):
if parser is not None and not callable(parser):
raise Exception("parser parameter is not callable")
result = {}
for symbol in symbols:
result[symbol] = []
pattern = self.build_pattern(source, "BAR", symbol)
for key in self._redis.scan_iter(pattern, count=1000):
val = self.retrieve_by_key(key)
if parser:
val = parser(val)
result[symbol].append(val)
return result
def bgsave(self):
self._redis.bgsave()
class RedisController(object):
"""Redis Controller implementation class."""
APP_LIST_NAME = 'appmanager.apps'
def __init__(self, conn_params, app_list_name=None, auto_save=False):
"""Constructor.
conn_params (RedisConnectionParams):
app_list_name (str):
auto_save (bool):
"""
self.conn_params = conn_params
self.auto_save = auto_save
if app_list_name is not None:
self.APP_LIST_NAME = app_list_name
self.redis = Redis(host=conn_params.host,
port=conn_params.port,
db=conn_params.db,
password=conn_params.password,
decode_responses=True,
charset="utf-8")
def ping(self):
"""Check if redis is online.
Returns:
(bool): True if redis server is online and False otherwise.
"""
try:
self.redis.ping()
return True
except Exception:
return False
def save_db(self):
"""Save database on disk. For persistence purposes."""
try:
self.redis.bgsave()
except exceptions.ResponseError:
# redis.exceptions.ResponseError:
# Background save already in progress
pass
def get_apps(self):
"""Returns the list of stored applications.
Returns:
(list):
"""
apps = self.redis.lrange(self.APP_LIST_NAME, 0, -1)
apps = [json.loads(app) for app in apps]
return apps
def get_app(self, app_name):
"""Returns a stored application given its name.
Args:
app_name (str): The name of the application.
Returns:
(dict): A dictionary that includes application information
as stored in database.
"""
apps = self.get_apps()
for _app in apps:
if _app['name'] == app_name:
# Exists
return _app
raise ValueError('Application does not exist in db.')
def get_app_type(self, app_name):
"""Returns the type of the application.
Args:
app_name (str): The name of the application
Returns:
(str): The type of the application
"""
_app = self.get_app(app_name)
return _app['type']
def app_exists(self, app_name):
"""Checks if application exists.
Args:
app_name (str): The name of the application
Returns:
(str): The type of the application
"""
apps = self.get_apps()
for _app in apps:
if _app['name'] == app_name:
# Exists
return True
return False
def add_app(self, app):
"""Add a new application to the database.
Args:
app (dict): The dictionary representing the application to be
stored in database.
"""
## TODO: Validate somehow the schema of app
created_at = int(time.time())
app['created_at'] = created_at
app['updated_at'] = -1
self.redis.lpush(self.APP_LIST_NAME, json.dumps(app))
if self.auto_save:
self.save_db()
def update_app(self, app):
"""Update an application information in db.
Args:
app (dict): The dictionary representing the application to be
stored in database.
"""
_app = self.get_app(app['name'])
app_index = self._get_app_index(app['name'])
_app['type'] = app['type']
_app['docker'] = app['docker']
_app['updated_at'] = int(time.time())
_app['ui'] = app['ui']
self.redis.lset(self.APP_LIST_NAME, app_index, json.dumps(_app))
if self.auto_save:
self.save_db()
def delete_app(self, app_name):
"""Delete an application.
Args:
app_name (str): The name of the application
"""
# app_index = self._get_app_index(app_name)
self.redis.lrem(self.APP_LIST_NAME, 1,
json.dumps(self.get_app(app_name)))
if self.auto_save:
self.save_db()
def set_app_state(self, app_name, state):
"""Sets the state of an application.
Args:
app_name (str): The name of the application.
state (int): Number representing the state of the application
Set to 0 for stopped and 1 for running.
"""
## States: 0 = NotRunning, 1 = Running
if state not in (0, 1): # Supported states
raise ValueError('State does not exist')
app = self.get_app(app_name)
app['state'] = state
if state == 0:
app['docker']['container'] = {'name': '', 'id': ''}
app_index = self._get_app_index(app_name)
self.redis.lset(self.APP_LIST_NAME, app_index, json.dumps(app))
if self.auto_save:
self.save_db()
def set_app_property(self, app_name, prop_name, prop_value):
app = self.get_app(app_name)
app[prop_name] = prop_value
app_index = self._get_app_index(app_name)
self.redis.lset(self.APP_LIST_NAME, app_index, json.dumps(app))
if self.auto_save:
self.save_db()
def app_is_running(self, app_name):
app = self.get_app(app_name)
if app['state'] == 1:
return True
return False
def get_app_image(self, app_name):
"""Returns the docker image information of an application.
Args:
app_name (str): The name of the application.
Returns:
(dict):
"""
_app = self.get_app(app_name)
return _app['docker']['image']['name']
def get_app_container(self, app_name):
"""Returns the docker container information of an application.
Args:
app_name (str): The name of the application.
Returns:
(dict):
"""
_app = self.get_app(app_name)
return _app['docker']['container']['id']
def set_app_container(self, app_name, container_name, container_id):
"""Set container information of an application.
Args:
app_name (str): The name of the application.
container_name (str): The name of the container.
container_id (str): The id of the container.
"""
_app = self.get_app(app_name)
_app['docker']['container']['name'] = container_name
_app['docker']['container']['id'] = container_id
app_index = self._get_app_index(app_name)
self.redis.lset(self.APP_LIST_NAME, app_index, json.dumps(_app))
if self.auto_save:
self.save_db()
def get_running_apps(self):
"""Returns the list of currently running applications.
Returns:
(list):
"""
apps = self.get_apps()
_r_apps = []
for app in apps:
if app['state'] == 1:
_r_apps.append(app)
return _r_apps
def _get_app_index(self, app_name):
apps = self.get_apps()
idx = 0
for _app in apps:
if _app['name'] == app_name:
# Exists
return idx
idx += 1
return -1