예제 #1
0
def denorm_instance(payload, cursor=None):
    logging.info('[cursor.denorm_instance] payload %s, cursor %s' %
                 (payload, cursor))

    data = json.loads(payload)
    source_model = util.get_model_by_name(data['source_model'])
    target_model = util.get_model_by_name(data['target_model'])
    related_field_name = data['related_field']
    fields = data['fields']

    queryset = target_model.objects.filter(
        **{related_field_name + '_id': data['instance_id']})
    if cursor:
        queryset = set_cursor(queryset, cursor)
    results = queryset[0:ITEMS_PER_TASK]
    cursor = get_cursor(results)

    # TODO: batch save
    for item in results:
        #print('[denorm_instance] denorm target instance %s' % item)

        item._denorm_values = fields  # provide denorm values directly so that pre_save signal receiver does not lookup related field
        item.save()

    if len(results) == ITEMS_PER_TASK:
        # there are likely more items
        logging.info('[denorm_instance] queue task with cursor %s' % cursor)
        deferred.defer(denorm_instance,
                       payload,
                       cursor,
                       _queue=data['queue_name'])
예제 #2
0
def denorm_instance(payload, cursor=None):
    logging.info('[cursor.denorm_instance] payload %s, cursor %s' % (payload, cursor))

    data = json.loads(payload)
    source_model = util.get_model_by_name(data['source_model'])
    target_model = util.get_model_by_name(data['target_model'])
    related_field_name = data['related_field']
    fields = data['fields']

    queryset = target_model.objects.filter(**{related_field_name+'_id': data['instance_id']})
    if cursor:
        queryset = set_cursor(queryset, cursor)
    results = queryset[0:ITEMS_PER_TASK]
    cursor = get_cursor(results)

    # TODO: batch save
    for item in results:
        #print('[denorm_instance] denorm target instance %s' % item)

        item._denorm_values = fields # provide denorm values directly so that pre_save signal receiver does not lookup related field
        item.save()

    if len(results) == ITEMS_PER_TASK:
        # there are likely more items
        logging.info('[denorm_instance] queue task with cursor %s' % cursor)
        deferred.defer(denorm_instance, payload, cursor, _queue=data['queue_name'])
예제 #3
0
    def run(self, start=None, batch_size=100, count=0):
        logging.info("Mapper run start at: %s" % count)
        if start:
            logging.info("start cursor = %s" % start)
            q = set_cursor(self.QUERY, start)
        else:
            q = self.QUERY

        entities = q[0:batch_size]
        try:
            end_cursor = get_cursor(entities)
        except:
            logging.error(traceback.format_exc())
            return

        for entity in entities:
            logging.info("Mapper entity: %s" % entity)
            count += 1
            self.map(entity)

        if entities:
            deferred.defer(self.run,
                           start=end_cursor,
                           count=count,
                           _queue="maintenance")
예제 #4
0
파일: cron.py 프로젝트: WAYbetter/waybetter
def do_calc(start_date, end_date, start=None, batch_size=100, count=0):
    from djangoappengine.db.utils import get_cursor, set_cursor

    if start_date:
        query = Order.objects.filter(create_date__gt=start_date, create_date__lt=end_date, type=OrderType.SHARED, debug=False)
    else:
        query = Order.objects.filter(create_date__lt=end_date, type=OrderType.SHARED, debug=True)

    if start:
        logging.info("start cursor = %s" % start)
        orders = set_cursor(query, start)
    else:
        orders = query

    orders = orders[0:batch_size]
    try:
        cursor = get_cursor(orders)
    except :
        logging.error(traceback.format_exc())
        return

    money_saved = Counter.objects.get(name="money_saved")

    for order in orders:
        try:
            count += 1
            if order.status not in [APPROVED, CHARGED]:
                logging.info("skipping order [%s]" % order.id)
                continue

            logging.info("calc: order [%s]" % order.id)

            if not order.price_alone:
                logging.warning("no price_alone for [%s]" % order.id)

        except :
            logging.error("Error while calculating price for order: [%s]\n%s" % (order.id, traceback.format_exc()))

        if order.price_alone:
            order_savings = order.price_alone - order.price
            logging.info("order [%s] saved: %s" % (order.id, order_savings))
            if order_savings > 0:
                money_saved.value += order_savings
                logging.info("money saved updated: %s" % money_saved.value)
            elif order_savings < 0:
                logging.warning("negative savings for order [%s]" % order.id)
        else:
            logging.warning("price calculation failed: %s" % order.id)


    money_saved.save()
    logging.info("processed %s entities" % count)
    if orders:
        deferred.defer(do_calc, start_date, end_date, start=cursor, count=count, _queue="maintenance")
    else:
        logging.info("ALL DONE: %s" % count)
예제 #5
0
    def __getitem__(self, value):
        """
            Does the query, saves the cursor for the next query to self and
            returns the objects in form of a list.
        """
        query = self.queryset.all()[value]
        if self._start_cursor:
            query = set_cursor(query, start=self._start_cursor)
            self._start_cursor = None

        obj_list = list(query)
        try:
            self._latest_cursor = get_cursor(query)
        except TypeError:
            # get_cursor() tries to call .urlsafe() on the cursor with fails
            # if the cursor is None. So we save None if it's None. Makes sense.
            self._latest_cursor = None

        return obj_list
예제 #6
0
 def _get_cursor(self):
     if self._query is not None:
         return get_cursor(self._query)
예제 #7
0
 def _get_cursor(self):
     if self._query is not None:
         return get_cursor(self._query)
예제 #8
0
파일: cron.py 프로젝트: WAYbetter/waybetter
def do_calc(start_date, end_date, start=None, batch_size=100, count=0):
    from djangoappengine.db.utils import get_cursor, set_cursor

    if start_date:
        query = Order.objects.filter(create_date__gt=start_date,
                                     create_date__lt=end_date,
                                     type=OrderType.SHARED,
                                     debug=False)
    else:
        query = Order.objects.filter(create_date__lt=end_date,
                                     type=OrderType.SHARED,
                                     debug=True)

    if start:
        logging.info("start cursor = %s" % start)
        orders = set_cursor(query, start)
    else:
        orders = query

    orders = orders[0:batch_size]
    try:
        cursor = get_cursor(orders)
    except:
        logging.error(traceback.format_exc())
        return

    money_saved = Counter.objects.get(name="money_saved")

    for order in orders:
        try:
            count += 1
            if order.status not in [APPROVED, CHARGED]:
                logging.info("skipping order [%s]" % order.id)
                continue

            logging.info("calc: order [%s]" % order.id)

            if not order.price_alone:
                logging.warning("no price_alone for [%s]" % order.id)

        except:
            logging.error("Error while calculating price for order: [%s]\n%s" %
                          (order.id, traceback.format_exc()))

        if order.price_alone:
            order_savings = order.price_alone - order.price
            logging.info("order [%s] saved: %s" % (order.id, order_savings))
            if order_savings > 0:
                money_saved.value += order_savings
                logging.info("money saved updated: %s" % money_saved.value)
            elif order_savings < 0:
                logging.warning("negative savings for order [%s]" % order.id)
        else:
            logging.warning("price calculation failed: %s" % order.id)

    money_saved.save()
    logging.info("processed %s entities" % count)
    if orders:
        deferred.defer(do_calc,
                       start_date,
                       end_date,
                       start=cursor,
                       count=count,
                       _queue="maintenance")
    else:
        logging.info("ALL DONE: %s" % count)