def test_or_query(self): D3.COLLECTION.drop() whiskeycache.clear_cache() theese_dees = [ D3({'myJaws': 'big'}), D3({'myJaws': 'small'}), D3({'myJaws': 'just right'}) ] for d in theese_dees: d.save() queries = [{ 'myJaws': 'big' }, { 'myJaws': 'big', 'someOtherVal': None, '$or': [ { 'myJaws': 'small' }, { 'myJaws': 'just right' }, ] }, { '$or': [ { 'myJaws': 'small' }, { 'myJaws': 'just right' }, ] }, { 'some_dict': [], '$or': [ { 'myJaws': 'big' }, { 'myJaws': 'just right' }, ] }, { '$or': [{ 'some_list': [] }, { 'some_dict': {} }] }] i = 1 for query in queries: print '=' * 72 print str(i) + 'query ' + str(query) self.assertEqual(len(whiskeycache.find(D3, query, [('_id', -1)])), D3.COLLECTION.find(query).count()) i += 1
def test_or_query(self): D3.COLLECTION.drop() whiskeycache.clear_cache() theese_dees = [D3({'myJaws':'big'}),D3({'myJaws':'small'}),D3({'myJaws':'just right'})] for d in theese_dees: d.save() queries = [ {'myJaws':'big' }, { 'myJaws':'big', 'someOtherVal':None, '$or':[ {'myJaws':'small'}, {'myJaws':'just right'}, ] }, { '$or':[ {'myJaws':'small'}, {'myJaws':'just right'}, ] }, { 'some_dict':[], '$or':[ {'myJaws':'big'}, {'myJaws':'just right'}, ] }, { '$or':[ {'some_list':[]}, {'some_dict':{}} ] } ] i = 1 for query in queries: print '='*72 print str(i) + 'query ' + str(query) self.assertEqual( len(whiskeycache.find(D3, query, [('_id', -1)] )), D3.COLLECTION.find(query).count()) i += 1
def find(cls, query={}, limit=0, skip_cache=False, sort=None, skip=0): """ Returns an iterator of whiskeynodes SORTED HIGHEST TO LOWEST _id (most recent first) all params are passed to pymongo except skip_cache - this allows you to make complex queries to mongodb """ if sort is None: sort = [("_id", -1)] else: assert isinstance(sort, list) and len(sort) >= 1, "sort should be a list of tuples" assert isinstance(sort[0], tuple), "sort should be a list of tuples" existing = ( deque(whiskeycache.find(cls, query, sort)) if not skip_cache else [] ) # grab the items we already have in RAM if limit > 0: cursor = cls.COLLECTION.find(query, limit=limit + skip).sort( sort ) # otherwise, hit the db, todo, pass a $notin:_ids else: cursor = cls.COLLECTION.find(query).sort(sort) # todo - take out the if else after fixing mongo mock class WhiskeyCursor: def __init__(self, existing, cursor, limit=0, skip=0): self.existing = existing self.cursor = cursor self.__count = None self.__limit = limit self.__retrieved = 0 self.__d = None if skip > 0: skipped = 0 for s in self: skipped += 1 if skipped >= skip: self.__retrieved = 0 break def __iter__(self): return self def __next__(self): """ python 3 """ return self.next() def next(self): """ this will return the items in cache and the db""" if self.__limit == 0 or self.__retrieved < self.__limit: self.__retrieved = self.__retrieved + 1 if len(self.existing) > 0: if self.__d is None: try: self.__d = self.cursor.next() except StopIteration: return self.existing.popleft() d = self.__d attr_existing = getattr(self.existing[0], sort[0][0]) attr_d = d.get(sort[0][0]) if sort[0][1] == -1: if attr_existing > attr_d: return self.existing.popleft() else: if attr_existing < attr_d: return self.existing.popleft() if self.existing[0]._id == d["_id"]: self.__d = None return self.existing.popleft() else: self.__d = None rv = whiskeycache.from_cache(cls, d, dirty=False) try: self.existing.remove( rv ) # todo test to see if "rv in self.existing" is faster than try excepting except ValueError: pass return rv else: if self.__d: d = self.__d self.__d = None return whiskeycache.from_cache(cls, d, dirty=False) else: return whiskeycache.from_cache(cls, self.cursor.next(), dirty=False) raise StopIteration() def count(self): """ NOTE - this count isn't exactly accurate since we don't know how many items will already be in the cache, but it's pretty close """ if self.__count is None: # self.__count = len(self.existing) + self.cursor.count() self.__count = self.cursor.count() # we're only looking at what's actually in the db for now... for x in self.existing: if x._is_new_local: self.__count = self.__count + 1 return self.__count def limit(self, limit): self.__limit = self.cursor.limit = limit def __len__(self): return self.count() return WhiskeyCursor(existing, cursor, limit, skip)
def find(cls, query={}, limit=0, skip_cache=False, sort=None, skip=0): ''' Returns an iterator of whiskeynodes SORTED HIGHEST TO LOWEST _id (most recent first) all params are passed to pymongo except skip_cache - this allows you to make complex queries to mongodb ''' if sort is None: sort = [('_id', -1)] else: assert isinstance(sort, list) and len(sort) >= 1, 'sort should be a list of tuples' assert isinstance(sort[0], tuple), 'sort should be a list of tuples' existing = deque( whiskeycache.find(cls, query, sort)) if not skip_cache else [] #grab the items we already have in RAM if limit > 0: cursor = cls.COLLECTION.find(query, limit=limit+skip).sort(sort) #otherwise, hit the db, todo, pass a $notin:_ids else: cursor = cls.COLLECTION.find(query).sort(sort) #todo - take out the if else after fixing mongo mock class WhiskeyCursor(): def __init__(self, existing, cursor, limit=0, skip=0): self.existing = existing self.cursor = cursor self.__count = None self.__limit = limit self.__retrieved = 0 self.__d = None if skip > 0: skipped = 0 for s in self: skipped += 1 if skipped >= skip: self.__retrieved = 0 break def __iter__(self): return self def __next__(self): ''' python 3 ''' return self.next() def next(self): ''' this will return the items in cache and the db''' if self.__limit == 0 or self.__retrieved < self.__limit: self.__retrieved = self.__retrieved + 1 if len(self.existing) > 0: if self.__d is None: try: self.__d = self.cursor.next() except StopIteration: return self.existing.popleft() d = self.__d attr_existing = getattr(self.existing[0], sort[0][0]) attr_d = d.get(sort[0][0]) if sort[0][1] == -1: if attr_existing > attr_d: return self.existing.popleft() else: if attr_existing < attr_d: return self.existing.popleft() if self.existing[0]._id == d['_id']: self.__d = None return self.existing.popleft() else: self.__d = None rv = whiskeycache.from_cache(cls, d, dirty=False) try: self.existing.remove(rv) #todo test to see if "rv in self.existing" is faster than try excepting except ValueError: pass return rv else: if self.__d: d = self.__d self.__d = None return whiskeycache.from_cache(cls, d, dirty=False) else: return whiskeycache.from_cache(cls, self.cursor.next(), dirty=False) raise StopIteration() def count(self): ''' NOTE - this count isn't exactly accurate since we don't know how many items will already be in the cache, but it's pretty close ''' if self.__count is None: #self.__count = len(self.existing) + self.cursor.count() self.__count = self.cursor.count() #we're only looking at what's actually in the db for now... for x in self.existing: if x._is_new_local: self.__count = self.__count + 1 return self.__count def limit(self, limit): self.__limit = self.cursor.limit = limit def __len__(self): return self.count() return WhiskeyCursor(existing, cursor, limit, skip)