def test_isiterable(self):
"""Test tje isiterable.
"""
self.assertFalse(isiterable(2))
self.assertTrue(isiterable([]))
self.assertTrue(isiterable(""))
self.assertFalse(isiterable('', is_str=False))
def test_isiterable(self):
"""Test tje isiterable.
"""
self.assertFalse(isiterable(2))
self.assertTrue(isiterable([]))
self.assertTrue(isiterable(""))
self.assertFalse(isiterable('', is_str=False))
def adapt_canopsis_data_to_ember(data):
"""
Transform canopsis data to ember data (in changing ``id`` to ``cid``).
:param data: data to transform
"""
if isinstance(data, dict):
for key, item in data.iteritems():
if isinstance(item, float) and (isnan(item) or isinf(item)):
data[key] = None
else:
if isinstance(item, (tuple, frozenset)):
item = list(item)
data[key] = item
adapt_canopsis_data_to_ember(item)
elif isiterable(data, is_str=False):
for i in range(len(data)):
item = data[i]
if isinstance(item, float) and (isnan(item) or isinf(item)):
data[i] = None
else:
if isinstance(item, (tuple, frozenset)):
item = list(item)
data[i] = item
adapt_canopsis_data_to_ember(item)
def get_shared_data(self, shared_ids):
"""Get all shared data related to input shared ids.
:param shared_ids: one or more data.
:type shared_ids: list or str
:return: depending on input shared_ids::
- one shared id: one list of shared data
- list of shared ids: list of list of shared data
"""
result = []
sids = ensure_iterable(shared_ids, iterable=set)
for shared_id in sids:
query = {CompositeStorage.SHARED: shared_id}
shared_data = self.get_elements(query=query)
result.append(shared_data)
# return first or data if data is not an iterable
if not isiterable(shared_ids, is_str=False):
result = get_first(result)
return result
def adapt_canopsis_data_to_ember(data):
"""
Transform canopsis data to ember data (in changing ``id`` to ``cid``).
:param data: data to transform
"""
if isinstance(data, dict):
for key, item in data.iteritems():
if isinstance(item, float) and (isnan(item) or isinf(item)):
data[key] = None
else:
if isinstance(item, (tuple, frozenset)):
item = list(item)
data[key] = item
adapt_canopsis_data_to_ember(item)
elif isiterable(data, is_str=False):
for i in range(len(data)):
item = data[i]
if isinstance(item, float) and (isnan(item) or isinf(item)):
data[i] = None
else:
if isinstance(item, (tuple, frozenset)):
item = list(item)
data[i] = item
adapt_canopsis_data_to_ember(item)
def get_shared_data(self, shared_ids):
"""Get all shared data related to input shared ids.
:param shared_ids: one or more data.
:type shared_ids: list or str
:return: depending on input shared_ids::
- one shared id: one list of shared data
- list of shared ids: list of list of shared data
"""
result = []
sids = ensure_iterable(shared_ids, iterable=set)
for shared_id in sids:
query = {CompositeStorage.SHARED: shared_id}
shared_data = self.get_elements(query=query)
result.append(shared_data)
# return first or data if data is not an iterable
if not isiterable(shared_ids, is_str=False):
result = get_first(result)
return result
def remove(
self, path, names=None, shared=False, cache=False, *args, **kwargs
):
query = path.copy()
parameters = {}
if names is not None:
if isiterable(names, is_str=False):
query[CompositeStorage.NAME] = {'$in': names}
else:
parameters = {'justOne': 1}
query[CompositeStorage.NAME] = names
self._remove(document=query, cache=cache, **parameters)
# remove extended data
if shared:
_ids = []
data_to_remove = self.get(path=path, names=names, shared=True)
for dtr in data_to_remove:
path, name = self.get_path_with_name(data=dtr)
extended = self.get(path=path, name=name, shared=True)
_ids.append([data[MongoStorage.ID] for data in extended])
document = {MongoStorage.ID: {'$in': _ids}}
self._remove(document=document, cache=cache)
def remove(self, path, data_ids=None, shared=False, *args, **kwargs):
query = path.copy()
parameters = {}
if data_ids is not None:
if isiterable(data_ids, is_str=False):
query[Storage.DATA_ID] = {'$in': data_ids}
else:
parameters = {'justOne': 1}
query[Storage.DATA_ID] = data_ids
self._remove(document=query, **parameters)
# remove extended data
if shared:
_ids = []
data_to_remove = self.get(
path=path, data_ids=data_ids, shared=True)
for dtr in data_to_remove:
path, data_id = self.get_path_with_id(dtr)
extended = self.get(path=path, data_id=data_id, shared=True)
_ids.append([data[MongoStorage.ID] for data in extended])
document = {MongoStorage.ID: {'$in': _ids}}
self._remove(document=document)
def _search(
self, data_ids=None, timewindow=None, _filter=None,
limit=0, skip=0, sort=None,
*args, **kwargs
):
"""Process internal search query in returning a cursor."""
result = None
# set a where clause for the search
where = {}
if _filter is not None: # add value filtering
if isinstance(_filter, dict):
for name in _filter:
vname = '{0}.{1}'.format(MongoTimedStorage.Key.VALUE, name)
where[vname] = _filter[name]
else:
where[MongoTimedStorage.Key.VALUE] = _filter
if data_ids is not None:
if isiterable(data_ids, is_str=False):
where[MongoTimedStorage.Key.DATA_ID] = {'$in': data_ids}
else:
where[MongoTimedStorage.Key.DATA_ID] = data_ids
# if timewindow is not None, get latest timestamp before
# timewindow.stop()
if timewindow is not None:
timestamp = timewindow.stop()
where[MongoTimedStorage.Key.TIMESTAMP] = {'$lte': timestamp}
# do the query
result = self._find(document=where)
# if timewindow is None or contains only one point, get only last
# document respectively before now or before the one point
if limit:
result.limit(limit)
if skip:
result.skip(skip)
if sort is not None:
sort = TimedStorage._resolve_sort(sort)
result.sort(sort)
# apply a specific index
if data_ids is None:
index = MongoTimedStorage.TIMESTAMPS
else:
index = MongoTimedStorage.TIMESTAMP_BY_ID
result.hint(index)
return result
def adapt_ember_data_to_canopsis(data):
if isinstance(data, dict):
for key, item in data.iteritems():
adapt_ember_data_to_canopsis(item)
elif isiterable(data, is_str=False):
for item in data:
adapt_ember_data_to_canopsis(item)
def _get_category(self, *args, **kwargs):
"""Get configuration category for self storage."""
prefix = self.data_type
if isiterable(prefix):
prefix = reduce(lambda x, y: '%s_%s' % (x, y), prefix)
result = '{0}_{1}'.format(prefix, self.data_scope).lower()
return result
def _get_category(self, *args, **kwargs):
"""Get configuration category for self storage."""
prefix = self.data_type
if isiterable(prefix):
prefix = reduce(lambda x, y: '%s_%s' % (x, y), prefix)
result = '{0}_{1}'.format(prefix, self.data_scope).lower()
return result
def adapt_ember_data_to_canopsis(data):
if isinstance(data, dict):
#if 'id' in data and data['id']:
#data['eid'] = data['id']
#if 'cid' in data and data['cid']:
#data['id'] = data['cid']
#del data['cid']
for item in data.values():
adapt_ember_data_to_canopsis(item)
elif isiterable(data, is_str=False):
for item in data:
adapt_ember_data_to_canopsis(item)
def remove(self, data_ids, timewindow=None, *args, **kwargs):
where = {}
if isiterable(data_ids, is_str=False):
where[MongoTimedStorage.Key.DATA_ID] = {'$in': data_ids}
else:
where[MongoTimedStorage.Key.DATA_ID] = data_ids
if timewindow is not None:
where[MongoTimedStorage.Key.TIMESTAMP] = \
{'$gte': timewindow.start(), '$lte': timewindow.stop()}
self._remove(document=where)
def remove(self, data_ids, timewindow=None, cache=False, *args, **kwargs):
where = {}
if isiterable(data_ids, is_str=False):
where[MongoPeriodicalStorage.Key.DATA_ID] = {'$in': data_ids}
else:
where[MongoPeriodicalStorage.Key.DATA_ID] = data_ids
if timewindow is not None:
where[MongoPeriodicalStorage.Key.TIMESTAMP] = {
'$gte': timewindow.start(), '$lte': timewindow.stop()
}
self._remove(document=where, cache=cache)
def remove_elements(self, ids=None, _filter=None, cache=False, *args, **kwargs):
query = {}
if ids is not None:
if isiterable(ids, is_str=False):
query[MongoStorage.ID] = {"$in": ids}
else:
query[MongoStorage.ID] = ids
if _filter is not None:
query.update(_filter)
self._remove(query, cache=cache)
def adapt_canopsis_data_to_ember(data):
"""
Transform canopsis data to ember data (in changing ``id`` to ``cid``).
:param data: data to transform
"""
if isinstance(data, dict):
#if 'id' in data and data['id']:
#data['cid'] = data['id']
#del data['id']
#if 'eid' in data and data['eid']:
#data['id'] = data['eid']
for item in data.values():
adapt_canopsis_data_to_ember(item)
elif isiterable(data, is_str=False):
for item in data:
adapt_canopsis_data_to_ember(item)
def get_table(self):
"""Table name related to self table or type and data_scope.
:return: table name.
:rtype: str
"""
# try to use local table
result = self.table
if not result:
prefix = self.data_type
if isiterable(prefix, is_str=False):
prefix = reduce(lambda x, y: '%s_%s' % (x, y), prefix)
result = "{0}_{1}".format(prefix, self.data_scope).lower()
return result
def get_table(self):
"""Table name related to self table or type and data_scope.
:return: table name.
:rtype: str
"""
# try to use local table
result = self.table
if not result:
prefix = self.data_type
if isiterable(prefix, is_str=False):
prefix = reduce(lambda x, y: '%s_%s' % (x, y), prefix)
result = "{0}_{1}".format(prefix, self.data_scope).lower()
return result
def remove_elements(
self, ids=None, _filter=None, tags=None, cache=False,
*args, **kwargs
):
query = {}
if ids is not None:
if isiterable(ids, is_str=False):
query[MongoStorage.ID] = {'$in': ids}
else:
query[MongoStorage.ID] = ids
if tags:
query[MongoStorage.TAGS] = tags
if _filter is not None:
query.update(_filter)
return self._remove(query, cache=cache)
def get(
self,
path, data_ids=None, _filter=None, shared=False,
limit=0, skip=0, sort=None, with_count=False,
*args, **kwargs
):
result = []
# create a get query which is a copy of input path plus _filter
query = path.copy()
if _filter is not None:
query.update(_filter)
one_result = False
# if data_ids is given
if data_ids is not None:
# add absolute pathes into ids
if isiterable(data_ids, is_str=False):
query[Storage.DATA_ID] = {"$in": data_ids}
else:
query[Storage.DATA_ID] = data_ids
one_result = True
# get elements
result = self.find_elements(
query=query,
limit=limit,
skip=skip,
sort=sort,
with_count=with_count)
if with_count:
count = result[1]
result = result[0]
if result is not None and shared:
# if result is one value
if isinstance(result, dict):
# if result is shared
if CompositeStorage.SHARED in result:
shared_id = result[CompositeStorage.SHARED]
# result equals shared data
result = [self.get_shared_data(shared_ids=shared_id)]
else:
# else, result is a list of itself
result = [[result]]
else:
# if result is a list of data, use data_to_extend
data_to_extend = result[:]
# and initialize result such as an empty list
result = []
# for all data in result
for data in data_to_extend:
# if data is shared, get shared data
if CompositeStorage.SHARED in data:
shared_id = data[CompositeStorage.SHARED]
data = self.get_shared_data(shared_ids=shared_id)
else:
data = [data]
# append data in result
result.append(data)
if result is not None and one_result:
if result:
result = result[0]
else:
result = None
if with_count:
result = result, count
return result
def _search(
self, data_ids=None, timewindow=None, window_start_bind=False,
_filter=None, limit=0, skip=0, sort=None,
*args, **kwargs
):
"""Process internal search query in returning a cursor."""
result = None
# set a where clause for the search
where = {} if _filter is None else self._convert_value_filter(_filter)
if data_ids is not None:
if isiterable(data_ids, is_str=False):
where[MongoPeriodicalStorage.Key.DATA_ID] = {'$in': data_ids}
else:
where[MongoPeriodicalStorage.Key.DATA_ID] = data_ids
# if timewindow is not None, get latest timestamp before
# timewindow.stop()
if timewindow is not None:
stop = timewindow.stop()
time_query = [
{MongoPeriodicalStorage.Key.TIMESTAMP: {'$lte': stop}},
]
if window_start_bind:
start = timewindow.start()
time_query.append(
{MongoPeriodicalStorage.Key.TIMESTAMP: {'$gte': start}}
)
if where:
where = {'$and': [where] + time_query}
else:
where = {'$and': time_query}
# do the query
self.logger.debug("Query : {}".format(where))
result = self._find(document=where)
# if timewindow is None or contains only one point, get only last
# document respectively before now or before the one point
if limit:
result.limit(limit)
if skip:
result.skip(skip)
if sort is not None:
sort = PeriodicalStorage._resolve_sort(sort)
result.sort(sort)
# apply a specific index
if data_ids is None:
index = MongoPeriodicalStorage.TIMESTAMPS
else:
index = MongoPeriodicalStorage.TIMESTAMP_BY_ID
result.hint(index)
return result
def _search(
self, data_ids=None, timewindow=None, window_start_bind=False,
_filter=None, limit=0, skip=0, sort=None,
*args, **kwargs
):
"""Process internal search query in returning a cursor."""
result = None
# set a where clause for the search
where = {} if _filter is None else self._convert_value_filter(_filter)
if data_ids is not None:
if isiterable(data_ids, is_str=False):
where[MongoPeriodicalStorage.Key.DATA_ID] = {'$in': data_ids}
else:
where[MongoPeriodicalStorage.Key.DATA_ID] = data_ids
# if timewindow is not None, get latest timestamp before
# timewindow.stop()
if timewindow is not None:
stop = timewindow.stop()
time_query = [
{MongoPeriodicalStorage.Key.TIMESTAMP: {'$lte': stop}},
]
if window_start_bind:
start = timewindow.start()
time_query.append(
{MongoPeriodicalStorage.Key.TIMESTAMP: {'$gte': start}}
)
if where:
where = {'$and': [where] + time_query}
else:
where = {'$and': time_query}
# do the query
result = self._find(document=where)
# if timewindow is None or contains only one point, get only last
# document respectively before now or before the one point
if limit:
result.limit(limit)
if skip:
result.skip(skip)
if sort is not None:
sort = PeriodicalStorage._resolve_sort(sort)
result.sort(sort)
# apply a specific index
if data_ids is None:
index = MongoPeriodicalStorage.TIMESTAMPS
else:
index = MongoPeriodicalStorage.TIMESTAMP_BY_ID
result.hint(index)
return result
def get(
self, data_ids, timewindow=None, limit=0, skip=0, sort=None,
*args, **kwargs
):
result = {}
# set a where clause for the search
where = {}
one_element = False
if isiterable(data_ids, is_str=False):
where[MongoTimedStorage.Key.DATA_ID] = {'$in': data_ids}
else:
where[MongoTimedStorage.Key.DATA_ID] = data_ids
one_element = True
# if timewindow is not None, get latest timestamp before
# timewindow.stop()
if timewindow is not None:
timestamp = timewindow.stop()
where[MongoTimedStorage.Key.TIMESTAMP] = {
'$lte': timewindow.stop()}
# do the query
cursor = self._find(document=where)
# if timewindow is None or contains only one point, get only last
# document respectively before now or before the one point
if limit:
cursor.limit(limit)
if skip:
cursor.skip(skip)
if sort is not None:
MongoStorage._update_sort(sort)
cursor.sort(sort)
# apply a specific index
cursor.hint(MongoTimedStorage.TIMESTAMP_BY_ID)
# iterate on all documents
for document in cursor:
timestamp = document[MongoTimedStorage.Key.TIMESTAMP]
value = document[MongoTimedStorage.Key.VALUE]
data_id = document[MongoTimedStorage.Key.DATA_ID]
# a value to get is composed of a timestamp, values and document id
value_to_append = {
TimedStorage.TIMESTAMP: timestamp,
TimedStorage.VALUE: value,
TimedStorage.DATA_ID: data_id
}
if data_id not in result:
result[data_id] = [value_to_append]
else:
result[data_id].append(value_to_append)
if timewindow is not None and timestamp not in timewindow:
# stop when a document is just before the start timewindow
break
# if one element has been requested, returns it
if one_element and result:
result = result[data_ids]
return result
