def test_isiterable(self): """Test tje isiterable. """ self.assertFalse(isiterable(2)) self.assertTrue(isiterable([])) self.assertTrue(isiterable("")) self.assertFalse(isiterable('', is_str=False))
def adapt_canopsis_data_to_ember(data): """ Transform canopsis data to ember data (in changing ``id`` to ``cid``). :param data: data to transform """ if isinstance(data, dict): for key, item in data.iteritems(): if isinstance(item, float) and (isnan(item) or isinf(item)): data[key] = None else: if isinstance(item, (tuple, frozenset)): item = list(item) data[key] = item adapt_canopsis_data_to_ember(item) elif isiterable(data, is_str=False): for i in range(len(data)): item = data[i] if isinstance(item, float) and (isnan(item) or isinf(item)): data[i] = None else: if isinstance(item, (tuple, frozenset)): item = list(item) data[i] = item adapt_canopsis_data_to_ember(item)
def get_shared_data(self, shared_ids): """Get all shared data related to input shared ids. :param shared_ids: one or more data. :type shared_ids: list or str :return: depending on input shared_ids:: - one shared id: one list of shared data - list of shared ids: list of list of shared data """ result = [] sids = ensure_iterable(shared_ids, iterable=set) for shared_id in sids: query = {CompositeStorage.SHARED: shared_id} shared_data = self.get_elements(query=query) result.append(shared_data) # return first or data if data is not an iterable if not isiterable(shared_ids, is_str=False): result = get_first(result) return result
def remove( self, path, names=None, shared=False, cache=False, *args, **kwargs ): query = path.copy() parameters = {} if names is not None: if isiterable(names, is_str=False): query[CompositeStorage.NAME] = {'$in': names} else: parameters = {'justOne': 1} query[CompositeStorage.NAME] = names self._remove(document=query, cache=cache, **parameters) # remove extended data if shared: _ids = [] data_to_remove = self.get(path=path, names=names, shared=True) for dtr in data_to_remove: path, name = self.get_path_with_name(data=dtr) extended = self.get(path=path, name=name, shared=True) _ids.append([data[MongoStorage.ID] for data in extended]) document = {MongoStorage.ID: {'$in': _ids}} self._remove(document=document, cache=cache)
def remove(self, path, data_ids=None, shared=False, *args, **kwargs): query = path.copy() parameters = {} if data_ids is not None: if isiterable(data_ids, is_str=False): query[Storage.DATA_ID] = {'$in': data_ids} else: parameters = {'justOne': 1} query[Storage.DATA_ID] = data_ids self._remove(document=query, **parameters) # remove extended data if shared: _ids = [] data_to_remove = self.get( path=path, data_ids=data_ids, shared=True) for dtr in data_to_remove: path, data_id = self.get_path_with_id(dtr) extended = self.get(path=path, data_id=data_id, shared=True) _ids.append([data[MongoStorage.ID] for data in extended]) document = {MongoStorage.ID: {'$in': _ids}} self._remove(document=document)
def _search( self, data_ids=None, timewindow=None, _filter=None, limit=0, skip=0, sort=None, *args, **kwargs ): """Process internal search query in returning a cursor.""" result = None # set a where clause for the search where = {} if _filter is not None: # add value filtering if isinstance(_filter, dict): for name in _filter: vname = '{0}.{1}'.format(MongoTimedStorage.Key.VALUE, name) where[vname] = _filter[name] else: where[MongoTimedStorage.Key.VALUE] = _filter if data_ids is not None: if isiterable(data_ids, is_str=False): where[MongoTimedStorage.Key.DATA_ID] = {'$in': data_ids} else: where[MongoTimedStorage.Key.DATA_ID] = data_ids # if timewindow is not None, get latest timestamp before # timewindow.stop() if timewindow is not None: timestamp = timewindow.stop() where[MongoTimedStorage.Key.TIMESTAMP] = {'$lte': timestamp} # do the query result = self._find(document=where) # if timewindow is None or contains only one point, get only last # document respectively before now or before the one point if limit: result.limit(limit) if skip: result.skip(skip) if sort is not None: sort = TimedStorage._resolve_sort(sort) result.sort(sort) # apply a specific index if data_ids is None: index = MongoTimedStorage.TIMESTAMPS else: index = MongoTimedStorage.TIMESTAMP_BY_ID result.hint(index) return result
def adapt_ember_data_to_canopsis(data): if isinstance(data, dict): for key, item in data.iteritems(): adapt_ember_data_to_canopsis(item) elif isiterable(data, is_str=False): for item in data: adapt_ember_data_to_canopsis(item)
def _get_category(self, *args, **kwargs): """Get configuration category for self storage.""" prefix = self.data_type if isiterable(prefix): prefix = reduce(lambda x, y: '%s_%s' % (x, y), prefix) result = '{0}_{1}'.format(prefix, self.data_scope).lower() return result
def adapt_ember_data_to_canopsis(data): if isinstance(data, dict): #if 'id' in data and data['id']: #data['eid'] = data['id'] #if 'cid' in data and data['cid']: #data['id'] = data['cid'] #del data['cid'] for item in data.values(): adapt_ember_data_to_canopsis(item) elif isiterable(data, is_str=False): for item in data: adapt_ember_data_to_canopsis(item)
def remove(self, data_ids, timewindow=None, *args, **kwargs): where = {} if isiterable(data_ids, is_str=False): where[MongoTimedStorage.Key.DATA_ID] = {'$in': data_ids} else: where[MongoTimedStorage.Key.DATA_ID] = data_ids if timewindow is not None: where[MongoTimedStorage.Key.TIMESTAMP] = \ {'$gte': timewindow.start(), '$lte': timewindow.stop()} self._remove(document=where)
def remove(self, data_ids, timewindow=None, cache=False, *args, **kwargs): where = {} if isiterable(data_ids, is_str=False): where[MongoPeriodicalStorage.Key.DATA_ID] = {'$in': data_ids} else: where[MongoPeriodicalStorage.Key.DATA_ID] = data_ids if timewindow is not None: where[MongoPeriodicalStorage.Key.TIMESTAMP] = { '$gte': timewindow.start(), '$lte': timewindow.stop() } self._remove(document=where, cache=cache)
def remove_elements(self, ids=None, _filter=None, cache=False, *args, **kwargs): query = {} if ids is not None: if isiterable(ids, is_str=False): query[MongoStorage.ID] = {"$in": ids} else: query[MongoStorage.ID] = ids if _filter is not None: query.update(_filter) self._remove(query, cache=cache)
def adapt_canopsis_data_to_ember(data): """ Transform canopsis data to ember data (in changing ``id`` to ``cid``). :param data: data to transform """ if isinstance(data, dict): #if 'id' in data and data['id']: #data['cid'] = data['id'] #del data['id'] #if 'eid' in data and data['eid']: #data['id'] = data['eid'] for item in data.values(): adapt_canopsis_data_to_ember(item) elif isiterable(data, is_str=False): for item in data: adapt_canopsis_data_to_ember(item)
def get_table(self): """Table name related to self table or type and data_scope. :return: table name. :rtype: str """ # try to use local table result = self.table if not result: prefix = self.data_type if isiterable(prefix, is_str=False): prefix = reduce(lambda x, y: '%s_%s' % (x, y), prefix) result = "{0}_{1}".format(prefix, self.data_scope).lower() return result
def remove_elements( self, ids=None, _filter=None, tags=None, cache=False, *args, **kwargs ): query = {} if ids is not None: if isiterable(ids, is_str=False): query[MongoStorage.ID] = {'$in': ids} else: query[MongoStorage.ID] = ids if tags: query[MongoStorage.TAGS] = tags if _filter is not None: query.update(_filter) return self._remove(query, cache=cache)
def get( self, path, data_ids=None, _filter=None, shared=False, limit=0, skip=0, sort=None, with_count=False, *args, **kwargs ): result = [] # create a get query which is a copy of input path plus _filter query = path.copy() if _filter is not None: query.update(_filter) one_result = False # if data_ids is given if data_ids is not None: # add absolute pathes into ids if isiterable(data_ids, is_str=False): query[Storage.DATA_ID] = {"$in": data_ids} else: query[Storage.DATA_ID] = data_ids one_result = True # get elements result = self.find_elements( query=query, limit=limit, skip=skip, sort=sort, with_count=with_count) if with_count: count = result[1] result = result[0] if result is not None and shared: # if result is one value if isinstance(result, dict): # if result is shared if CompositeStorage.SHARED in result: shared_id = result[CompositeStorage.SHARED] # result equals shared data result = [self.get_shared_data(shared_ids=shared_id)] else: # else, result is a list of itself result = [[result]] else: # if result is a list of data, use data_to_extend data_to_extend = result[:] # and initialize result such as an empty list result = [] # for all data in result for data in data_to_extend: # if data is shared, get shared data if CompositeStorage.SHARED in data: shared_id = data[CompositeStorage.SHARED] data = self.get_shared_data(shared_ids=shared_id) else: data = [data] # append data in result result.append(data) if result is not None and one_result: if result: result = result[0] else: result = None if with_count: result = result, count return result
def _search( self, data_ids=None, timewindow=None, window_start_bind=False, _filter=None, limit=0, skip=0, sort=None, *args, **kwargs ): """Process internal search query in returning a cursor.""" result = None # set a where clause for the search where = {} if _filter is None else self._convert_value_filter(_filter) if data_ids is not None: if isiterable(data_ids, is_str=False): where[MongoPeriodicalStorage.Key.DATA_ID] = {'$in': data_ids} else: where[MongoPeriodicalStorage.Key.DATA_ID] = data_ids # if timewindow is not None, get latest timestamp before # timewindow.stop() if timewindow is not None: stop = timewindow.stop() time_query = [ {MongoPeriodicalStorage.Key.TIMESTAMP: {'$lte': stop}}, ] if window_start_bind: start = timewindow.start() time_query.append( {MongoPeriodicalStorage.Key.TIMESTAMP: {'$gte': start}} ) if where: where = {'$and': [where] + time_query} else: where = {'$and': time_query} # do the query self.logger.debug("Query : {}".format(where)) result = self._find(document=where) # if timewindow is None or contains only one point, get only last # document respectively before now or before the one point if limit: result.limit(limit) if skip: result.skip(skip) if sort is not None: sort = PeriodicalStorage._resolve_sort(sort) result.sort(sort) # apply a specific index if data_ids is None: index = MongoPeriodicalStorage.TIMESTAMPS else: index = MongoPeriodicalStorage.TIMESTAMP_BY_ID result.hint(index) return result
def _search( self, data_ids=None, timewindow=None, window_start_bind=False, _filter=None, limit=0, skip=0, sort=None, *args, **kwargs ): """Process internal search query in returning a cursor.""" result = None # set a where clause for the search where = {} if _filter is None else self._convert_value_filter(_filter) if data_ids is not None: if isiterable(data_ids, is_str=False): where[MongoPeriodicalStorage.Key.DATA_ID] = {'$in': data_ids} else: where[MongoPeriodicalStorage.Key.DATA_ID] = data_ids # if timewindow is not None, get latest timestamp before # timewindow.stop() if timewindow is not None: stop = timewindow.stop() time_query = [ {MongoPeriodicalStorage.Key.TIMESTAMP: {'$lte': stop}}, ] if window_start_bind: start = timewindow.start() time_query.append( {MongoPeriodicalStorage.Key.TIMESTAMP: {'$gte': start}} ) if where: where = {'$and': [where] + time_query} else: where = {'$and': time_query} # do the query result = self._find(document=where) # if timewindow is None or contains only one point, get only last # document respectively before now or before the one point if limit: result.limit(limit) if skip: result.skip(skip) if sort is not None: sort = PeriodicalStorage._resolve_sort(sort) result.sort(sort) # apply a specific index if data_ids is None: index = MongoPeriodicalStorage.TIMESTAMPS else: index = MongoPeriodicalStorage.TIMESTAMP_BY_ID result.hint(index) return result
def get( self, data_ids, timewindow=None, limit=0, skip=0, sort=None, *args, **kwargs ): result = {} # set a where clause for the search where = {} one_element = False if isiterable(data_ids, is_str=False): where[MongoTimedStorage.Key.DATA_ID] = {'$in': data_ids} else: where[MongoTimedStorage.Key.DATA_ID] = data_ids one_element = True # if timewindow is not None, get latest timestamp before # timewindow.stop() if timewindow is not None: timestamp = timewindow.stop() where[MongoTimedStorage.Key.TIMESTAMP] = { '$lte': timewindow.stop()} # do the query cursor = self._find(document=where) # if timewindow is None or contains only one point, get only last # document respectively before now or before the one point if limit: cursor.limit(limit) if skip: cursor.skip(skip) if sort is not None: MongoStorage._update_sort(sort) cursor.sort(sort) # apply a specific index cursor.hint(MongoTimedStorage.TIMESTAMP_BY_ID) # iterate on all documents for document in cursor: timestamp = document[MongoTimedStorage.Key.TIMESTAMP] value = document[MongoTimedStorage.Key.VALUE] data_id = document[MongoTimedStorage.Key.DATA_ID] # a value to get is composed of a timestamp, values and document id value_to_append = { TimedStorage.TIMESTAMP: timestamp, TimedStorage.VALUE: value, TimedStorage.DATA_ID: data_id } if data_id not in result: result[data_id] = [value_to_append] else: result[data_id].append(value_to_append) if timewindow is not None and timestamp not in timewindow: # stop when a document is just before the start timewindow break # if one element has been requested, returns it if one_element and result: result = result[data_ids] return result