def test_or_query(self):
D3.COLLECTION.drop()
whiskeycache.clear_cache()
theese_dees = [
D3({'myJaws': 'big'}),
D3({'myJaws': 'small'}),
D3({'myJaws': 'just right'})
]
for d in theese_dees:
d.save()
queries = [{
'myJaws': 'big'
}, {
'myJaws': 'big',
'someOtherVal': None,
'$or': [
{
'myJaws': 'small'
},
{
'myJaws': 'just right'
},
]
}, {
'$or': [
{
'myJaws': 'small'
},
{
'myJaws': 'just right'
},
]
}, {
'some_dict': [],
'$or': [
{
'myJaws': 'big'
},
{
'myJaws': 'just right'
},
]
}, {
'$or': [{
'some_list': []
}, {
'some_dict': {}
}]
}]
i = 1
for query in queries:
print '=' * 72
print str(i) + 'query ' + str(query)
self.assertEqual(len(whiskeycache.find(D3, query, [('_id', -1)])),
D3.COLLECTION.find(query).count())
i += 1
def test_or_query(self):
D3.COLLECTION.drop()
whiskeycache.clear_cache()
theese_dees = [D3({'myJaws':'big'}),D3({'myJaws':'small'}),D3({'myJaws':'just right'})]
for d in theese_dees:
d.save()
queries = [
{'myJaws':'big'
},
{
'myJaws':'big',
'someOtherVal':None,
'$or':[
{'myJaws':'small'},
{'myJaws':'just right'},
]
},
{
'$or':[
{'myJaws':'small'},
{'myJaws':'just right'},
]
},
{
'some_dict':[],
'$or':[
{'myJaws':'big'},
{'myJaws':'just right'},
]
},
{
'$or':[
{'some_list':[]},
{'some_dict':{}}
]
}
]
i = 1
for query in queries:
print '='*72
print str(i) + 'query ' + str(query)
self.assertEqual( len(whiskeycache.find(D3, query, [('_id', -1)] )), D3.COLLECTION.find(query).count())
i += 1
def find(cls, query={}, limit=0, skip_cache=False, sort=None, skip=0):
"""
Returns an iterator of whiskeynodes SORTED HIGHEST TO LOWEST _id (most recent first)
all params are passed to pymongo except skip_cache - this allows you to make complex queries to mongodb
"""
if sort is None:
sort = [("_id", -1)]
else:
assert isinstance(sort, list) and len(sort) >= 1, "sort should be a list of tuples"
assert isinstance(sort[0], tuple), "sort should be a list of tuples"
existing = (
deque(whiskeycache.find(cls, query, sort)) if not skip_cache else []
) # grab the items we already have in RAM
if limit > 0:
cursor = cls.COLLECTION.find(query, limit=limit + skip).sort(
sort
) # otherwise, hit the db, todo, pass a $notin:_ids
else:
cursor = cls.COLLECTION.find(query).sort(sort) # todo - take out the if else after fixing mongo mock
class WhiskeyCursor:
def __init__(self, existing, cursor, limit=0, skip=0):
self.existing = existing
self.cursor = cursor
self.__count = None
self.__limit = limit
self.__retrieved = 0
self.__d = None
if skip > 0:
skipped = 0
for s in self:
skipped += 1
if skipped >= skip:
self.__retrieved = 0
break
def __iter__(self):
return self
def __next__(self):
""" python 3 """
return self.next()
def next(self):
""" this will return the items in cache and the db"""
if self.__limit == 0 or self.__retrieved < self.__limit:
self.__retrieved = self.__retrieved + 1
if len(self.existing) > 0:
if self.__d is None:
try:
self.__d = self.cursor.next()
except StopIteration:
return self.existing.popleft()
d = self.__d
attr_existing = getattr(self.existing[0], sort[0][0])
attr_d = d.get(sort[0][0])
if sort[0][1] == -1:
if attr_existing > attr_d:
return self.existing.popleft()
else:
if attr_existing < attr_d:
return self.existing.popleft()
if self.existing[0]._id == d["_id"]:
self.__d = None
return self.existing.popleft()
else:
self.__d = None
rv = whiskeycache.from_cache(cls, d, dirty=False)
try:
self.existing.remove(
rv
) # todo test to see if "rv in self.existing" is faster than try excepting
except ValueError:
pass
return rv
else:
if self.__d:
d = self.__d
self.__d = None
return whiskeycache.from_cache(cls, d, dirty=False)
else:
return whiskeycache.from_cache(cls, self.cursor.next(), dirty=False)
raise StopIteration()
def count(self):
""" NOTE - this count isn't exactly accurate
since we don't know how many items will already be in the cache, but it's pretty close """
if self.__count is None:
# self.__count = len(self.existing) + self.cursor.count()
self.__count = self.cursor.count() # we're only looking at what's actually in the db for now...
for x in self.existing:
if x._is_new_local:
self.__count = self.__count + 1
return self.__count
def limit(self, limit):
self.__limit = self.cursor.limit = limit
def __len__(self):
return self.count()
return WhiskeyCursor(existing, cursor, limit, skip)
def find(cls, query={}, limit=0, skip_cache=False, sort=None, skip=0):
'''
Returns an iterator of whiskeynodes SORTED HIGHEST TO LOWEST _id (most recent first)
all params are passed to pymongo except skip_cache - this allows you to make complex queries to mongodb
'''
if sort is None:
sort = [('_id', -1)]
else:
assert isinstance(sort, list) and len(sort) >= 1, 'sort should be a list of tuples'
assert isinstance(sort[0], tuple), 'sort should be a list of tuples'
existing = deque( whiskeycache.find(cls, query, sort)) if not skip_cache else [] #grab the items we already have in RAM
if limit > 0:
cursor = cls.COLLECTION.find(query, limit=limit+skip).sort(sort) #otherwise, hit the db, todo, pass a $notin:_ids
else:
cursor = cls.COLLECTION.find(query).sort(sort) #todo - take out the if else after fixing mongo mock
class WhiskeyCursor():
def __init__(self, existing, cursor, limit=0, skip=0):
self.existing = existing
self.cursor = cursor
self.__count = None
self.__limit = limit
self.__retrieved = 0
self.__d = None
if skip > 0:
skipped = 0
for s in self:
skipped += 1
if skipped >= skip:
self.__retrieved = 0
break
def __iter__(self):
return self
def __next__(self):
''' python 3 '''
return self.next()
def next(self):
''' this will return the items in cache and the db'''
if self.__limit == 0 or self.__retrieved < self.__limit:
self.__retrieved = self.__retrieved + 1
if len(self.existing) > 0:
if self.__d is None:
try:
self.__d = self.cursor.next()
except StopIteration:
return self.existing.popleft()
d = self.__d
attr_existing = getattr(self.existing[0], sort[0][0])
attr_d = d.get(sort[0][0])
if sort[0][1] == -1:
if attr_existing > attr_d:
return self.existing.popleft()
else:
if attr_existing < attr_d:
return self.existing.popleft()
if self.existing[0]._id == d['_id']:
self.__d = None
return self.existing.popleft()
else:
self.__d = None
rv = whiskeycache.from_cache(cls, d, dirty=False)
try:
self.existing.remove(rv) #todo test to see if "rv in self.existing" is faster than try excepting
except ValueError:
pass
return rv
else:
if self.__d:
d = self.__d
self.__d = None
return whiskeycache.from_cache(cls, d, dirty=False)
else:
return whiskeycache.from_cache(cls, self.cursor.next(), dirty=False)
raise StopIteration()
def count(self):
''' NOTE - this count isn't exactly accurate
since we don't know how many items will already be in the cache, but it's pretty close '''
if self.__count is None:
#self.__count = len(self.existing) + self.cursor.count()
self.__count = self.cursor.count() #we're only looking at what's actually in the db for now...
for x in self.existing:
if x._is_new_local:
self.__count = self.__count + 1
return self.__count
def limit(self, limit):
self.__limit = self.cursor.limit = limit
def __len__(self):
return self.count()
return WhiskeyCursor(existing, cursor, limit, skip)
