def test_cursor(self):
results = list(FieldsWithOptionsModel.objects.all())
cursor = None
for item in results:
query = FieldsWithOptionsModel.objects.all()[:1]
if cursor is not None:
set_cursor(query, cursor)
next = query[0]
self.assertEqual(next.pk, item.pk)
cursor = get_cursor(query)
query = FieldsWithOptionsModel.objects.all()[:1]
set_cursor(query, cursor)
self.assertEqual(list(query), [])
def test_cursor(self):
results = list(FieldsWithOptionsModel.objects.all())
cursor = None
for item in results:
query = FieldsWithOptionsModel.objects.all()[:1]
if cursor is not None:
set_cursor(query, cursor)
next = query[0]
self.assertEqual(next.pk, item.pk)
cursor = get_cursor(query)
query = FieldsWithOptionsModel.objects.all()[:1]
set_cursor(query, cursor)
self.assertEqual(list(query), [])
def __iter__(self):
k_range = self._key_range
# Namespaces are not supported by djangoappengine
if k_range.namespace:
return
model_class = util.for_name(self._query_spec.model_class_path)
q = model_class.objects.all()
if k_range.key_start:
if k_range.include_start:
q = q.filter(pk__gte=k_range.key_start.id_or_name())
else:
q = q.filter(pk__gt=k_range.key_start.id_or_name())
if k_range.key_end:
if k_range.include_end:
q = q.filter(pk__lte=k_range.key_end.id_or_name())
else:
q = q.filter(pk__lt=k_range.key_end.id_or_name())
q = q.order_by('pk')
q = set_config(q, batch_size=self._query_spec.batch_size)
if self._cursor:
q = set_cursor(q, self._cursor)
self._query = q
for entity in self._query.iterator():
yield entity
def denorm_instance(payload, cursor=None):
logging.info('[cursor.denorm_instance] payload %s, cursor %s' %
(payload, cursor))
data = json.loads(payload)
source_model = util.get_model_by_name(data['source_model'])
target_model = util.get_model_by_name(data['target_model'])
related_field_name = data['related_field']
fields = data['fields']
queryset = target_model.objects.filter(
**{related_field_name + '_id': data['instance_id']})
if cursor:
queryset = set_cursor(queryset, cursor)
results = queryset[0:ITEMS_PER_TASK]
cursor = get_cursor(results)
# TODO: batch save
for item in results:
#print('[denorm_instance] denorm target instance %s' % item)
item._denorm_values = fields # provide denorm values directly so that pre_save signal receiver does not lookup related field
item.save()
if len(results) == ITEMS_PER_TASK:
# there are likely more items
logging.info('[denorm_instance] queue task with cursor %s' % cursor)
deferred.defer(denorm_instance,
payload,
cursor,
_queue=data['queue_name'])
def __iter__(self):
k_range = self._key_range
# Namespaces are not supported by djangoappengine
if k_range.namespace:
return
model_class = util.for_name(self._query_spec.model_class_path)
q = model_class.objects.all()
if k_range.key_start:
if k_range.include_start:
q = q.filter(pk__gte=k_range.key_start.id_or_name())
else:
q = q.filter(pk__gt=k_range.key_start.id_or_name())
if k_range.key_end:
if k_range.include_end:
q = q.filter(pk__lte=k_range.key_end.id_or_name())
else:
q = q.filter(pk__lt=k_range.key_end.id_or_name())
q = q.order_by('pk')
q = set_config(q, batch_size=self._query_spec.batch_size)
if self._cursor:
q = set_cursor(q, self._cursor)
self._query = q
for entity in self._query.iterator():
yield entity
def denorm_instance(payload, cursor=None):
logging.info('[cursor.denorm_instance] payload %s, cursor %s' % (payload, cursor))
data = json.loads(payload)
source_model = util.get_model_by_name(data['source_model'])
target_model = util.get_model_by_name(data['target_model'])
related_field_name = data['related_field']
fields = data['fields']
queryset = target_model.objects.filter(**{related_field_name+'_id': data['instance_id']})
if cursor:
queryset = set_cursor(queryset, cursor)
results = queryset[0:ITEMS_PER_TASK]
cursor = get_cursor(results)
# TODO: batch save
for item in results:
#print('[denorm_instance] denorm target instance %s' % item)
item._denorm_values = fields # provide denorm values directly so that pre_save signal receiver does not lookup related field
item.save()
if len(results) == ITEMS_PER_TASK:
# there are likely more items
logging.info('[denorm_instance] queue task with cursor %s' % cursor)
deferred.defer(denorm_instance, payload, cursor, _queue=data['queue_name'])
def run(self, start=None, batch_size=100, count=0):
logging.info("Mapper run start at: %s" % count)
if start:
logging.info("start cursor = %s" % start)
q = set_cursor(self.QUERY, start)
else:
q = self.QUERY
entities = q[0:batch_size]
try:
end_cursor = get_cursor(entities)
except:
logging.error(traceback.format_exc())
return
for entity in entities:
logging.info("Mapper entity: %s" % entity)
count += 1
self.map(entity)
if entities:
deferred.defer(self.run,
start=end_cursor,
count=count,
_queue="maintenance")
def do_calc(start_date, end_date, start=None, batch_size=100, count=0):
from djangoappengine.db.utils import get_cursor, set_cursor
if start_date:
query = Order.objects.filter(create_date__gt=start_date, create_date__lt=end_date, type=OrderType.SHARED, debug=False)
else:
query = Order.objects.filter(create_date__lt=end_date, type=OrderType.SHARED, debug=True)
if start:
logging.info("start cursor = %s" % start)
orders = set_cursor(query, start)
else:
orders = query
orders = orders[0:batch_size]
try:
cursor = get_cursor(orders)
except :
logging.error(traceback.format_exc())
return
money_saved = Counter.objects.get(name="money_saved")
for order in orders:
try:
count += 1
if order.status not in [APPROVED, CHARGED]:
logging.info("skipping order [%s]" % order.id)
continue
logging.info("calc: order [%s]" % order.id)
if not order.price_alone:
logging.warning("no price_alone for [%s]" % order.id)
except :
logging.error("Error while calculating price for order: [%s]\n%s" % (order.id, traceback.format_exc()))
if order.price_alone:
order_savings = order.price_alone - order.price
logging.info("order [%s] saved: %s" % (order.id, order_savings))
if order_savings > 0:
money_saved.value += order_savings
logging.info("money saved updated: %s" % money_saved.value)
elif order_savings < 0:
logging.warning("negative savings for order [%s]" % order.id)
else:
logging.warning("price calculation failed: %s" % order.id)
money_saved.save()
logging.info("processed %s entities" % count)
if orders:
deferred.defer(do_calc, start_date, end_date, start=cursor, count=count, _queue="maintenance")
else:
logging.info("ALL DONE: %s" % count)
def contains_more_objects(self, next_batch_cursor):
"""
Returns a boolean telling if there are more objects in the queryset
or if there aren't.
"""
query = self.queryset.all().values_list('pk')
query = set_cursor(query, start=next_batch_cursor)
try:
query[0]
return True
except IndexError:
return False
def __getitem__(self, value):
"""
Does the query, saves the cursor for the next query to self and
returns the objects in form of a list.
"""
query = self.queryset.all()[value]
if self._start_cursor:
query = set_cursor(query, start=self._start_cursor)
self._start_cursor = None
obj_list = list(query)
try:
self._latest_cursor = get_cursor(query)
except TypeError:
# get_cursor() tries to call .urlsafe() on the cursor with fails
# if the cursor is None. So we save None if it's None. Makes sense.
self._latest_cursor = None
return obj_list
def do_calc(start_date, end_date, start=None, batch_size=100, count=0):
from djangoappengine.db.utils import get_cursor, set_cursor
if start_date:
query = Order.objects.filter(create_date__gt=start_date,
create_date__lt=end_date,
type=OrderType.SHARED,
debug=False)
else:
query = Order.objects.filter(create_date__lt=end_date,
type=OrderType.SHARED,
debug=True)
if start:
logging.info("start cursor = %s" % start)
orders = set_cursor(query, start)
else:
orders = query
orders = orders[0:batch_size]
try:
cursor = get_cursor(orders)
except:
logging.error(traceback.format_exc())
return
money_saved = Counter.objects.get(name="money_saved")
for order in orders:
try:
count += 1
if order.status not in [APPROVED, CHARGED]:
logging.info("skipping order [%s]" % order.id)
continue
logging.info("calc: order [%s]" % order.id)
if not order.price_alone:
logging.warning("no price_alone for [%s]" % order.id)
except:
logging.error("Error while calculating price for order: [%s]\n%s" %
(order.id, traceback.format_exc()))
if order.price_alone:
order_savings = order.price_alone - order.price
logging.info("order [%s] saved: %s" % (order.id, order_savings))
if order_savings > 0:
money_saved.value += order_savings
logging.info("money saved updated: %s" % money_saved.value)
elif order_savings < 0:
logging.warning("negative savings for order [%s]" % order.id)
else:
logging.warning("price calculation failed: %s" % order.id)
money_saved.save()
logging.info("processed %s entities" % count)
if orders:
deferred.defer(do_calc,
start_date,
end_date,
start=cursor,
count=count,
_queue="maintenance")
else:
logging.info("ALL DONE: %s" % count)
