def __init__(self, mongodb_host="localhost", redis_host="localhost", redis_port=6379):
# self.redis = redis.Redis(redis_host, port=redis_port)
self._redis:Optional[Redis] = None
self._redis_conn = ZRedisDatabaseConnection()
self.dominant_database = ""
self.helpers = Helpers()
self.pool = ThreadPool(max_workers=cpu_count() * 6)
def __init__(self):
# print("DBHandler")
self._metatype = "event"
self._entity = ""
self._required = {}
self._query = {}
self.data = {}
self._is_event = True
self._processor: Optional[Processor] = None
self.event_proc: Optional[EventProcessor] = None
self.main_helper = Helpers()
def __init__(self,
mongodb_host="localhost",
redis_host="localhost",
redis_port=6379):
self.redis = Redis(redis_host, port=redis_port)
self.store = Store(mongodb_host).create_lib(
'events').get_store()['events']
self.pool = ThreadPool(max_workers=cpu_count() * 6)
self.mongo_conn = MongoDatabaseConnection()
self.redis_conn = ZRedisDatabaseConnection()
self.mongo_conn.connection = self.store
self.redis_conn.connection = self.redis
self.dominant_database = ""
self.helpers = Helpers()
def __init__(self):
# print("DBHandler")
self._entity = ""
self._meta_type = "storage"
self._required = {}
self._query = {}
self._data = {}
self._is_event = True
self._processor: Optional[Processor] = None
self.event_proc: Optional[EventProcessor] = None
self.main_helper: Helpers = Helpers()
self.changed_since_command: bool = False
self.is_skip_check: bool = False
self.call_params = {}
def __init__(self) -> None:
self._redis: Optional[Redis] = None
self._redis_conn = RedisFileConnection()
self.helpers = Helpers()
class Jamboree(EventProcessor):
"""Adds and retrieves events at extremely fast speeds. Use to handle portfolio and trade information quickly."""
def __init__(self,
mongodb_host="localhost",
redis_host="localhost",
redis_port=6379):
self.redis = Redis(redis_host, port=redis_port)
self.store = Store(mongodb_host).create_lib(
'events').get_store()['events']
self.pool = ThreadPool(max_workers=cpu_count() * 6)
self.mongo_conn = MongoDatabaseConnection()
self.redis_conn = ZRedisDatabaseConnection()
self.mongo_conn.connection = self.store
self.redis_conn.connection = self.redis
self.dominant_database = ""
self.helpers = Helpers()
# self.redis_conn.pool = self.pool
# self.mongo_conn.pool = self.pool
def _validate_query(self, query: dict):
""" Validates a query. Must have `type` and a second identifier at least"""
if 'type' not in query:
return False
if not isinstance(query['type'], str):
return False
if len(query) < 2:
return False
return True
def _generate_hash(self, query: dict):
_hash = ujson.dumps(query, sort_keys=True)
_hash = base64.b64encode(str.encode(_hash))
_hash = _hash.decode('utf-8')
return _hash
def _check_redis_for_prior(self, _hash: str) -> bool:
""" Checks to see if any """
prior_length = self.redis.llen(_hash)
if prior_length == 0:
return False
return True
def _omit_timestamp(self, data: dict):
""" Removes timestamp if it exists. Use it to create a copied version of a dictionary to be saved in the duplicate list """
_data = copy(data)
_data.pop("timestamp", None)
return _data
def back_to_dict(self, list_of_serialized: list):
deserialized = []
if len(list_of_serialized) == 1:
return orjson.loads(list_of_serialized[0])
for i in list_of_serialized:
deserialized.append(orjson.loads(i))
return deserialized
def _save(self, query: dict, data: dict, abs_rel="absolute"):
"""
# TODO: Readd mongo
Given a type (data entity), data and a epoch for time (utc time only), save the data in both redis and mongo.
Does it in a background process. Use with add event.
We save the information both in mongodb and redis. We assume there's many of each collection. We find a specific collection using the query.
"""
self.redis_conn.save(query, data)
# self.pool.schedule(self.mongo_conn.save, args=(query, data))
"""
RESET FUNCTIONS
"""
def _reset_count(self, query: dict):
""" Reset the count for the current mongodb query. We do this by adding records in mongo back into redis. """
# all_elements = self.mongo_conn.query_all(query)
# self.pool.schedule(self.redis_conn.reset, args=(query, all_elements))
def reset(self, query: dict):
""" Resets all of the variables """
# self.pool.schedule(self._reset_count, args=(query))
"""
DELETES FUNCTIONS
"""
def _remove(self, query: dict, details: dict):
""" Use to both remove items from redis and mongo. Add it when you need it."""
"""
Removes the given query information from the database.
It's a heavy computation on redis, as it'll require searching an entire list.
"""
# self.pool.schedule(self.mongo_conn.delete_all, args=(query, details))
self.redis_conn.delete(query, details)
self.pool.schedule(self.redis_conn.delete_all, args=(query))
def _remove_first_redis(self, _hash, query: dict):
pass
def remove_first(self, query: dict):
pass
# _hash = self._generate_hash(query)
# count = self._get_count(_hash, query)
# if count == 0:
# return
# self._remove_first_redis(_hash, query)
"""
SAVE FUNCTIONS
"""
def save(self, query: dict, data: dict, abs_rel="absolute"):
self._save(query, data)
def save_many(self, query: dict, data: List[dict]):
if self._validate_query(query) == False:
# Log a warning here instead
return
if len(data) == 0: return
data_list = [self.helpers.update_dict(query, item) for item in data]
self._bulk_save(query, data_list)
def _bulk_save(self, query, data: list):
""" Bulk adds a list to redis."""
events = self.helpers.convert_to_storable_relative(data)
self.redis_conn.save_many(query, events)
# self.pool.schedule(self.mongo_conn.save_many, args=(query, data))
def _pop_redis_multiple(self, _hash, limit: int):
return []
def pop_multiple(self, query, limit: int = 1):
""" Get multiple items """
_hash = self._generate_hash(query)
count = self._get_count(_hash, query)
if count == 0:
return []
return self._pop_redis_multiple(_hash, limit)
"""
Public Query Functions
"""
def query_direct(self, query):
""" Queries from mongodb directly. Used to search extremely large queries. """
latest_items = list(self.store.query_latest(query))
return latest_items
def query_direct_latest(self, query):
""" Queries from mongodb directly. Used to search extremely large queries. """
return self.mongo_conn.query_latest(query)
def get_latest(self, query, abs_rel="absolute"):
""" Gets the latest query"""
# Add a conditional time lock
_hash = self._generate_hash(query)
count, database = self._get_count(_hash, query)
if count > 0:
return self.redis_conn.query_latest(query, abs_rel)
return {}
# # Mongo, slowdown
# return self.mongo_conn.query_latest(query)
def get_latest_many(self, query: dict, limit=1000, abs_rel="absolute"):
if self._validate_query(query) == False: return []
_hash = self._generate_hash(query)
count = self._get_count(_hash, query)
if count == 0: return []
latest_redis_items = self.redis_conn.query_latest_many(query,
abs_rel=abs_rel,
limit=limit)
return latest_redis_items
""" New Queries """
def get_between(self,
query: dict,
min_epoch: float,
max_epoch: float,
abs_rel: str = "absolute"):
items = self.redis_conn.query_between(query, min_epoch, max_epoch,
abs_rel)
return items
def get_latest_by(self,
query: dict,
max_epoch,
abs_rel="absolute",
limit: int = 10):
item = self.redis_conn.query_latest_by_time(query, max_epoch, abs_rel)
return item
"""
SEARCH ONE FUNCTIONS
"""
def _search_one(self, item: dict, query: dict):
all_bools = []
for q in query:
if q in item:
if query[q] == query[q]:
all_bools.append(True)
else:
all_bools.append(False)
else:
all_bools.append(False)
return any(all_bools)
def _get_count(self, _hash: str, query: dict):
# Checks to see if a count already exist in redis, if not, check for a count in mongo.
count = self.redis_conn.count(_hash)
if count is not None:
return count, "redis"
records = list(self.store.query(query))
record_len = len(records)
return record_len, "mongo"
def count(self, query):
""" """
if self._validate_query(query) == False: return []
_hash = self._generate_hash(query)
count, database = self._get_count(_hash, query)
self.dominant_database = database
return count
def single_get(self, query: dict):
if self._validate_query(query) == False: return {}
item = self.redis_conn.get(query)
return item
def single_set(self, query: dict, data: dict):
if self._validate_query(query) == False: return
self.redis_conn.add(query, data)
def single_delete(self, query: dict):
if self._validate_query(query) == False: return
self.redis_conn.kill(query)
class DBHandler(BaseHandler):
"""
A simple event store using a variation of databases.
---
Currently uses zadd to work
"""
def __init__(self):
# print("DBHandler")
self._metatype = "event"
self._entity = ""
self._required = {}
self._query = {}
self.data = {}
self._is_event = True
self._processor: Optional[Processor] = None
self.event_proc: Optional[EventProcessor] = None
self.main_helper = Helpers()
def __setitem__(self, key, value):
if bool(self.required):
if key in self.required:
self._query[key] = value
return self._query
self.data[key] = value
return self.data
def __getitem__(self, key):
if key in self._query.keys():
return self._query.get(key, None)
else:
if key in self.data.keys():
return self.data.get(key, None)
return None
def setup_query(self, alt={}):
return self.setup_key_with_lru(
self._query, self.entity, self._metatype, self.data, alt
)
def setup_key_with_lru(
self, query: dict, entity: str, metatype: str, data: dict, alt: dict
):
query = copy.copy(query)
query["type"] = entity
query["mtype"] = metatype
query.update(alt)
query.update(data)
return query
@property
def is_event(self) -> bool:
""" Determines if we're going to add event ids to what we're doing. We can essentially set certain conditions"""
return self._is_event
@is_event.setter
def is_event(self, is_true: bool = False):
self._is_event = is_true
@property
def processor(self) -> Processor:
if self._processor is None:
raise AttributeError("The Processor is missing")
return self._processor
@processor.setter
def processor(self, _processor: "Processor"):
self._processor = _processor
@property
def event(self) -> Optional["EventProcessor"]:
return self.event_proc
@event.setter
def event(self, _event: EventProcessor):
# Use to process event
self.event_proc = _event
def clear_event(self) -> None:
self.event_proc = None
self._processor = None
@property
def entity(self):
return self._entity
@entity.setter
def entity(self, _entity: str):
self._entity = str(_entity)
@property
def required(self):
return self._required
@required.setter
def required(self, _required: Dict[str, Any]):
# check to make sure it's not empty
self._required = _required
@property
def query(self):
return self._query
@query.setter
def query(self, _query: Dict[str, Any]):
if len(_query.keys()) > 0:
self._query = _query
def check(self):
# self.processor
# if self.event_proc is None:
# raise AttributeError("Event processor isn't available.")
if not bool(self._entity):
raise AttributeError("Entity hasn't been set")
if not bool(self._required):
raise AttributeError(
"None of the required information has been set"
)
if not bool(self._query):
raise AttributeError(
"None of the queryable information has been set"
)
reqs = list(self._required.keys())
for req in self._required.keys():
_type = self._required[req]
if req not in self._query:
raise AttributeError(
f"{req} is not in the requirements: {reqs}"
)
if not isinstance(self._query[req], _type):
raise AttributeError(f"{req} is not a {_type}")
return True
def _get_many(self, limit: int, ar: str, alt={}):
""" Aims to get many variables """
query = self.setup_query(alt)
latest_many = self.processor.event.get_latest_many(
query, abs_rel=ar, limit=limit
)
return latest_many
def _get_latest(self, ar, alt={}):
query = self.setup_query(alt)
latest = self.processor.event.get_latest(query, abs_rel=ar)
return latest
def _last_by(self, time_index: float, ar="absolute", alt={}) -> dict:
query = self.setup_query(alt)
return self.processor.event.get_latest_by(
query, time_index, abs_rel=ar
)
def _in_between(
self,
min_epoch: float,
max_epoch: float,
ar: str = "absolute",
alt={}
):
query = self.setup_query(alt)
return self.processor.event.get_between(
query, min_epoch, max_epoch, abs_rel=ar
)
def save(self, data: dict, alt={}):
self.check()
query = self.setup_query(alt)
if self.is_event:
data = self.main_helper.add_event_id(data)
self.processor.event.save(query, data)
def save_many(self, data: list, ar="absolute", alt={}):
self.check()
if not self.main_helper.is_abs_rel(ar):
return
query = self.setup_query(alt)
if self.is_event:
data = self.main_helper.add_event_ids(data)
self.processor.event.save_many(query, data)
def last(self, ar="absolute", alt={}):
self.check()
if not self.main_helper.is_abs_rel(ar):
return {}
alt.update(self.data)
return self._get_latest(ar=ar, alt=alt)
def many(self, limit=1000, ar="absolute", alt={}):
self.check()
if not self.main_helper.is_abs_rel(ar):
return []
alt.update(self.data)
return self._get_many(limit, ar, alt=alt)
def pop(self, alt={}):
self.check()
query = self.setup_query(alt)
self.processor.event.remove_first(query)
def pop_many(self, _limit, alt={}):
self.check()
query = self.setup_query(alt)
return self.processor.event.pop_multiple(query, _limit)
def last_by(self, time_index: float, ar="absolute", alt={}):
self.check()
if not self.main_helper.is_abs_rel(ar):
return {}
item = self._last_by(time_index, ar=ar, alt=alt)
return item
def in_between(
self,
min_epoch: float,
max_epoch: float,
ar: str = "absolute",
alt={}
):
self.check()
if not self.main_helper.is_abs_rel(ar):
return []
items = self._in_between(min_epoch, max_epoch, ar=ar, alt=alt)
return items
def count(self, alt={}) -> int:
""" Aims to get many variables """
self.check()
query = self.setup_query(alt)
return self.processor.event.count(query)
def get_single(self, alt={}, is_serialized=True):
self.check()
query = self.setup_query(alt)
item = self.processor.event.single_get(
query, is_serialized=is_serialized
)
return item
def set_single(self, data: dict, alt={}, is_serialized=True):
self.check()
query = self.setup_query(alt)
self.processor.event.single_set(
query, data, is_serialized=is_serialized
)
def delete_single(self, alt={}, is_dumps=False):
self.check()
query = self.setup_query(alt)
self.processor.event.single_delete(query)
def delete_all(self, alt={}):
self.check()
query = self.setup_query(alt)
self.processor.event.delete_all(query)
def query_all(self, alt: Dict[str, Any] = {}):
self.check()
query = self.setup_query(alt)
items = self.processor.event.get_all(query)
return items
def get_minimum_time(self, alt: Dict[str, Any] = {}):
self.check()
_query = self.setup_query(alt)
_time = self.processor.event.min_time(_query)
return _time
def get_maximum_time(self, alt: Dict[str, Any] = {}):
self.check()
_query = self.setup_query(alt)
_time = self.processor.event.max_time(_query)
return _time
def copy(self):
""" Get everything about this DBHandler without the event inside """
# _event = self.event
# self.event = _event
current_dict = copy.copy(self.__dict__)
non_lock_types = {}
_process = self.processor
self._processor = None
for key, value in current_dict.items():
classified = type(value)
if inspect.isclass(classified):
if (
isinstance(value, BaseHandler)
or issubclass(classified, BaseHandler)
or issubclass(classified, BaseSearchHandler)
):
non_lock_types[key] = value
setattr(self, key, None)
self.clear_event()
copied: self = copy.deepcopy(self)
copied.processor = _process
for k, v in non_lock_types.items():
setattr(copied, k, v)
self.__dict__ = current_dict
self.processor = _process
return copied
def lock(self, alt={}):
self.check()
query = self.setup_query(alt)
return self.processor.event.lock(query)
def __init__(self) -> None:
self._connection: Optional[Union[Redis, Pipeline]] = None
self.helpers = Helpers()
self._pool = ThreadPool(max_workers=(cpu_count() * 2))