def __init__(self, *args, **kwargs):
super(CTXLinklistRegistry, self).__init__(*args, **kwargs)
self.manager = Linklist()
self.events = MongoStorage(table='events')
self.context = Context()
def __init__(self, table=None, *args, **kwargs):
"""
:param Storage storage: event storage to use.
"""
super(CTXOldRegistry, self).__init__(*args, **kwargs)
self.storage = MongoStorage(table=table)
def update_mongo2influxdb(self):
"""Convert mongo data to influxdb data."""
mongostorage = MongoStorage(table='periodic_perfdata')
count = mongostorage._backend.count()
if count: # if data have to be deleted
for document in mongostorage._backend.find():
data_id = document['i']
timestamp = int(document['t'])
values = document['v']
points = [(timestamp + int(ts), values[ts]) for ts in values]
perfdata.put(metric_id=data_id, points=points)
mongostorage.remove_elements(ids=document['_id'])
def setUp(self):
"""
initalize a storage.
"""
self.storage = MongoStorage(data_scope='test')
# self.storage.safe = False
# remove all data from collection
self.storage.drop()
# initialize numerical values
self.iteration = 5
self.count = 10000
# generate count documents
self.documents = list(
{
"_id": str(i),
"data": {"index": i}
} for i in range(self.count)
)
self.documents_to_update = list(
{
"$set": {
'data.index': None
}
} for i in range(self.count)
)
# initialize commands
self.commands = (
("insert", lambda spec, document: self.storage._insert(
document=document, cache=True)),
("update", lambda spec, document: self.storage._update(
spec=spec, document=document, multi=False, cache=True)),
("find", lambda spec, document: self.storage._find(
document=document, cache=True)),
("remove", lambda spec, document: self.storage._remove(
document=document, multi=True, cache=True))
)
self.max_connections = 1
class Bench(TestCase):
def setUp(self):
"""
initalize a storage.
"""
self.storage = MongoStorage(data_scope='test')
# self.storage.safe = False
# remove all data from collection
self.storage.drop()
# initialize numerical values
self.iteration = 5
self.count = 10000
# generate count documents
self.documents = list(
{
"_id": str(i),
"data": {"index": i}
} for i in range(self.count)
)
self.documents_to_update = list(
{
"$set": {
'data.index': None
}
} for i in range(self.count)
)
# initialize commands
self.commands = (
("insert", lambda spec, document: self.storage._insert(
document=document, cache=True)),
("update", lambda spec, document: self.storage._update(
spec=spec, document=document, multi=False, cache=True)),
("find", lambda spec, document: self.storage._find(
document=document, cache=True)),
("remove", lambda spec, document: self.storage._remove(
document=document, multi=True, cache=True))
)
self.max_connections = 1
def tearDown(self):
"""
End the test
"""
# remove all data from collection
self.storage.drop()
def test(self):
"""
Run tests.
"""
threads = [
Thread(target=self._test_CRUD) for i in range(self.max_connections)
]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
def _test_CRUD(self):
"""
Bench CRUD commands
"""
print(
'Starting bench on %s with %s documents'
%
(self.commands, self.count)
)
stats_per_command = {
command[0]: {'min': None, 'max': None, 'mean': 0}
for command in self.commands
}
min_t, max_t, mean_t = None, None, 0
for i in range(self.iteration):
total = 0
for index, command in enumerate(self.commands):
fn = command[1]
command = command[0]
start = time()
if command == 'update':
documents = self.documents_to_update
else:
documents = self.documents
for j in range(self.count):
fn(
spec={'_id': str(self.count)},
document=documents[j]
)
stop = time()
duration = stop - start
stats = stats_per_command[command]
if stats['min'] is None or stats['min'] > duration:
stats['min'] = duration
if stats['max'] is None or stats['max'] < duration:
stats['max'] = duration
stats['mean'] += duration
bench_msg = 'command: %s, iteration: %s, time: %s'
print(
bench_msg % (command, i, duration)
)
total += duration
if min_t is None or min_t > total:
min_t = total
if max_t is None or max_t < total:
max_t = total
mean_t += total
bench_msg = 'CRUD: %s' % total
print(bench_msg)
# update mean per command
for command in self.commands:
stats = stats_per_command[command[0]]
stats['mean'] = stats['mean'] / self.iteration
bench_msg = 'command: %s, min: %s, max: %s, mean: %s'
print(
bench_msg
%
(command[0], stats['min'], stats['max'], stats['mean'])
)
# update mean for all CRUD operations
mean_t = mean_t / self.iteration
bench_msg = "all command, min: %s, max: %s, mean: %s"
print(
bench_msg % (min_t, max_t, mean_t)
)
class CTXOldRegistry(CTXPropRegistry):
"""In charge of binding an old collection (not generated from the Storage
project) to context entities.
Such old collections contain all entity fields.
"""
def __init__(self, table=None, *args, **kwargs):
"""
:param Storage storage: event storage to use.
"""
super(CTXOldRegistry, self).__init__(*args, **kwargs)
self.storage = MongoStorage(table=table)
def _do(self, command, ids, query, queryname, *args, **kwargs):
"""Execute storage command related to input ids and query.
:param command: storage command.
:param list ids: entity id(s).
:param dict query: storage command query.
:param str queryname: storage command query parameter.
:return: storage command execution documents.
:rtype: list
"""
result = []
# initialize query
query = deepcopy(query) if query else {}
# get entity id field name
ctx_id_field = self._ctx_id_field()
if ids is None:
# get all existing entity ids
ids = []
events = self.storage.find_elements()
for event in events:
entity = self.context.get_entity(event)
entity_id = self.context.get_entity_id(entity)
ids.append(entity_id)
for entity_id in ids:
# get entity
entity = self.context.get_entity_by_id(entity_id)
cleaned_entity = self.context.clean(entity)
cleaned_entity['source_type'] = cleaned_entity.pop('type')
for ctx in self.context.context[1:]:
if ctx in cleaned_entity:
continue
else:
cleaned_entity[ctx] = cleaned_entity.pop(Context.NAME)
break
# update query with entity information
_query = deepcopy(query)
_query.setdefault('$and', []).append(cleaned_entity)
# update kwargs with query and queryname
kwargs[queryname] = _query
# execute the storage command
documents = command(*args, **kwargs)
if isinstance(documents, Cursor):
documents = list(documents)
# update entity_id in documents
for document in documents:
document[ctx_id_field] = entity_id
else:
documents = [
{ctx_id_field: entity_id, 'result': documents}
]
# add all documents into the result
result += documents
return result
def _get(self, ids, query, *args, **kwargs):
return self._do(
command=self.storage.find_elements,
ids=ids, query=query, queryname='query',
*args, **kwargs
)
def _count(self, ids, query, *args, **kwargs):
return self._do(
command=self.storage.count_elements,
ids=ids, query=query, queryname='query',
*args, **kwargs
)
def _delete(self, ids, query, *args, **kwargs):
return self._do(
command=self.storage.remove_elements,
ids=ids, query=query, queryname='_filter',
*args, **kwargs
)
def ids(self, query=None, *args, **kwargs):
result = []
events = self.storage.find_elements(query=query)
for event in events:
entity = self.context.get_entity(event)
entity_id = self.context.get_entity_id(entity)
result.append(entity_id)
return result
class CTXLinklistRegistry(CTXPropRegistry):
"""In charge of ctx linklist information.
"""
__datatype__ = 'linklist' #: default datatype name
def __init__(self, *args, **kwargs):
super(CTXLinklistRegistry, self).__init__(*args, **kwargs)
self.manager = Linklist()
self.events = MongoStorage(table='events')
self.context = Context()
def _get_documents(self, ids, query):
"""Get documents related to input ids and query.
:param list ids: entity ids. If None, get all documents.
:param dict query: additional selection query.
:return: list of documents.
:rtype: list
"""
result = []
# get entity id field name
ctx_id_field = self._ctx_id_field()
# get a set of entity ids for execution speed reasons
if ids is not None:
ids = set(ids)
# get documents
docs = self.manager.find(_filter=query)
for doc in docs:
try:
mfilter = loads(doc['mfilter'])
except Exception:
pass
else: # get entities from events
events = self.events.find_elements(query=mfilter)
for event in events:
entity = self.context.get_entity(event)
entity_id = self.context.get_entity_id(entity)
if ids is None or entity_id in ids:
doc[ctx_id_field] = entity_id # add eid to the doc
result.append(doc)
return result
def _get(self, ids, query, *args, **kwargs):
return self._get_documents(ids=ids, query=query)
def _delete(self, ids, query, *args, **kwargs):
result = self._get_documents(ids=ids, query=query)
ids = [doc['_id'] for doc in result]
self.manager.remove(ids=ids)
return result
def ids(self, query=None):
result = []
documents = self.manager.find(_filter=query)
for document in documents:
try:
mfilter = loads(document['mfilter'])
except Exception:
pass
else:
# get entities from events
events = self.events.find_elements(query=mfilter)
for event in events:
entity = self.context.get_entity(event)
entity_id = self.context.get_entity_id(entity)
result.append(entity_id)
return result