def setUp(self):
"""
initalize a storage.
"""
self.storage = MongoStorage(data_scope='test')
#self.storage.safe = False
# remove all data from collection
self.storage.drop()
# initialize numerical values
self.iteration = 5
self.count = 10000
# generate count documents
self.documents = list(
{
"_id": str(i),
"data": {"index": i}
} for i in range(self.count)
)
self.documents_to_update = list(
{
"$set": {
'data.index': None
}
} for i in range(self.count)
)
# initialize commands
self.commands = (
("insert", lambda spec, document: self.storage._insert(
document=document, cache=True)),
("update", lambda spec, document: self.storage._update(
spec=spec, document=document, multi=False, cache=True)),
("find", lambda spec, document: self.storage._find(
document=document, cache=True)),
("remove", lambda spec, document: self.storage._remove(
document=document, multi=True, cache=True))
)
self.max_connections = 1
class Bench(TestCase):
def setUp(self):
"""
initalize a storage.
"""
self.storage = MongoStorage(data_scope='test')
#self.storage.safe = False
# remove all data from collection
self.storage.drop()
# initialize numerical values
self.iteration = 5
self.count = 10000
# generate count documents
self.documents = list(
{
"_id": str(i),
"data": {"index": i}
} for i in range(self.count)
)
self.documents_to_update = list(
{
"$set": {
'data.index': None
}
} for i in range(self.count)
)
# initialize commands
self.commands = (
("insert", lambda spec, document: self.storage._insert(
document=document, cache=True)),
("update", lambda spec, document: self.storage._update(
spec=spec, document=document, multi=False, cache=True)),
("find", lambda spec, document: self.storage._find(
document=document, cache=True)),
("remove", lambda spec, document: self.storage._remove(
document=document, multi=True, cache=True))
)
self.max_connections = 1
def tearDown(self):
"""
End the test
"""
# remove all data from collection
self.storage.drop()
def test(self):
"""
Run tests.
"""
threads = [
Thread(target=self._test_CRUD) for i in range(self.max_connections)
]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
def _test_CRUD(self):
"""
Bench CRUD commands
"""
print(
'Starting bench on %s with %s documents'
%
(self.commands, self.count)
)
stats_per_command = {
command[0]: {'min': None, 'max': None, 'mean': 0}
for command in self.commands
}
min_t, max_t, mean_t = None, None, 0
for i in range(self.iteration):
total = 0
for index, command in enumerate(self.commands):
fn = command[1]
command = command[0]
start = time()
documents = self.documents_to_update if command == 'update' else self.documents
for j in range(self.count):
fn(
spec={'_id': str(self.count)},
document=documents[j]
)
stop = time()
duration = stop - start
stats = stats_per_command[command]
if stats['min'] is None or stats['min'] > duration:
stats['min'] = duration
if stats['max'] is None or stats['max'] < duration:
stats['max'] = duration
stats['mean'] += duration
bench_msg = 'command: %s, iteration: %s, time: %s'
print(
bench_msg % (command, i, duration)
)
total += duration
if min_t is None or min_t > total:
min_t = total
if max_t is None or max_t < total:
max_t = total
mean_t += total
bench_msg = 'CRUD: %s' % total
print(bench_msg)
# update mean per command
for command in self.commands:
stats = stats_per_command[command[0]]
stats['mean'] = stats['mean'] / self.iteration
bench_msg = 'command: %s, min: %s, max: %s, mean: %s'
print(
bench_msg
%
(command[0], stats['min'], stats['max'], stats['mean'])
)
# update mean for all CRUD operations
mean_t = mean_t / self.iteration
bench_msg = "all command, min: %s, max: %s, mean: %s"
print(
bench_msg % (min_t, max_t, mean_t)
)
def get(
self, data_ids, timewindow=None, limit=0, skip=0, sort=None,
*args, **kwargs
):
result = {}
# set a where clause for the search
where = {}
one_element = False
if isiterable(data_ids, is_str=False):
where[MongoTimedStorage.Key.DATA_ID] = {'$in': data_ids}
else:
where[MongoTimedStorage.Key.DATA_ID] = data_ids
one_element = True
# if timewindow is not None, get latest timestamp before
# timewindow.stop()
if timewindow is not None:
timestamp = timewindow.stop()
where[MongoTimedStorage.Key.TIMESTAMP] = {
'$lte': timewindow.stop()}
# do the query
cursor = self._find(document=where)
# if timewindow is None or contains only one point, get only last
# document respectively before now or before the one point
if limit:
cursor.limit(limit)
if skip:
cursor.skip(skip)
if sort is not None:
MongoStorage._update_sort(sort)
cursor.sort(sort)
# apply a specific index
cursor.hint(MongoTimedStorage.TIMESTAMP_BY_ID)
# iterate on all documents
for document in cursor:
timestamp = document[MongoTimedStorage.Key.TIMESTAMP]
value = document[MongoTimedStorage.Key.VALUE]
data_id = document[MongoTimedStorage.Key.DATA_ID]
# a value to get is composed of a timestamp, values and document id
value_to_append = {
TimedStorage.TIMESTAMP: timestamp,
TimedStorage.VALUE: value,
TimedStorage.DATA_ID: data_id
}
if data_id not in result:
result[data_id] = [value_to_append]
else:
result[data_id].append(value_to_append)
if timewindow is not None and timestamp not in timewindow:
# stop when a document is just before the start timewindow
break
# if one element has been requested, returns it
if one_element and result:
result = result[data_ids]
return result
