コード例 #1
0
ファイル: cron.py プロジェクト: wangjun/appetsy
    def __update_listing(self, d, e):
        """update database listing d with data of etsy listing e"""
        changes = False

        if d.state != e.state:
            self.log("'%s' state '%s' --> '%s'" % (d.title, d.state, e.state))
            d.state = e.state
            if e.state != "active": #we don't care for edits much as they don't change ranking or anything
                storage.Events(listing = d, shop = d.shop, event = d.state).put()
            changes = True

        if hasattr(e, "title") and d.title != e.title:
            self.log("'%s' title --> '%s'" % (d.title, e.title))
            d.title = e.title
            changes = True

        if hasattr(e, "views") and d.views != int(e.views):
            views = int(e.views) - (d.views or 0)
            self.log("'%s' views +%d" % (d.title, views))
            d.views = int(e.views)
            self.__add_exposure(d, views)
            changes = True

        if  hasattr(e, "ending_tsz") and appetsy.zero_timezone(d.ending) != appetsy.etsy_epoch(e.ending_tsz):
            if not d.ending:
                storage.Events(listing = d,
                       shop = d.shop,
                       event = "posted",
                       created = appetsy.etsy_epoch(e.creation_tsz)).put()
            else:
                self.log("'%s' renewed" % d.title)
                storage.Events(listing = d, shop = d.shop, event = "renewed").put()

            d.ending = appetsy.etsy_epoch(e.ending_tsz)
            changes = True

        if hasattr(e, "price") and d.price != float(e.price):
            self.log("'%s' price %.2f --> %.2f" % (e.title, (d.price or 0.0), float(e.price)))
            d.price = float(e.price)
            changes = True

        if changes:
            memcache.set("listing:%d" % d.id, d) #store our memcache copy for next cron
            appetsy.invalidate_memcache("goods", str(d.shop.id)) #forget UI listings

        return changes
コード例 #2
0
ファイル: cron.py プロジェクト: wangjun/appetsy
    def get(self):
        if not self.etsy:
            return

        fan_id = self.request.get("id")

        db_fan = storage.Fans.get_by_key_name(fan_id)

        etsy_fan = None
        if fan_id != "_private":
            etsy_fan = self.etsy.getUser(fan_id, ["Profile", "Shops"])

            if hasattr(etsy_fan, "Profile") == False:
                etsy_fan = None

        if etsy_fan:
            db_fan.user_name = etsy_fan.login_name
            db_fan.image_url = etsy_fan.Profile.image_url_75x75 or "/public/dejo_75x75.png"
            db_fan.small_image_url = db_fan.image_url
            db_fan.status = "normal"
            if etsy_fan.Shops:
                db_fan.url = "http://www.etsy.com/shop/%s" % etsy_fan.Shops[0].shop_name
                db_fan.status = "seller"
            db_fan.joined_on = appetsy.etsy_epoch(etsy_fan.creation_tsz)

        else:
            db_fan.image_url = "/public/dejo_slepens_75x75.png"
            db_fan.small_image_url = "/public/dejo_slepens_25x25.png"
            db_fan.status = "private"

        logging.info("Added brand new fan: %s" % (db_fan.user_name or "secret"))

        db.put(db_fan)

        # remove from the pending list as we are done now
        pending = memcache.get("fans_pending_refresh") or []
        if fan_id in pending:
            pending.remove(fan_id)
            memcache.set("fans_pending_refresh", pending)
コード例 #3
0
ファイル: cron.py プロジェクト: wangjun/appetsy
    def get(self):
        if not self.etsy:
            return

        listing_id = self.request.get("listing")

        if not listing_id:
            self.log("no listing - exiting!")
            return

        listing = self.get_listing(listing_id)
        if not listing:
            self.log("Can't find listing with id %s" % listing_id)


        self.log("%s: Getting details for %s (%d known faves)..." % (listing.shop.shop_name,
                                                                     listing.title,
                                                                     listing.faves or 0))

        # get all etsy fans
        etsy_favorers_by_timestamp = {}
        def get_listing_info(offset = 0):
            if offset > 0:
                self.log("%d - %d..." % (offset, offset + 100))

            listing_info = self.etsy.getListing(listing.id, ["Images", "FavoredBy:100:%d" % offset])

            if len(listing_info.FavoredBy) > 0 and len(listing_info.FavoredBy) % 100 == 0:
                listing_info.FavoredBy.extend(get_listing_info(offset + 100).FavoredBy)

            return listing_info

        listing_info = get_listing_info()

        for item in listing_info.FavoredBy:
            etsy_favorers_by_timestamp[appetsy.etsy_epoch(item.creation_tsz)] = item
        etsy_timestamps = set(etsy_favorers_by_timestamp.keys())

        #update fan count for item
        listing.faves = len(etsy_timestamps)
        listing.image_url = listing_info.Images[0].url_75x75
        listing.put()

        db_timestamps = memcache.get("listing:%d_fans" % listing.id)
        if not db_timestamps:
            #get all db fans
            db_favorers_by_timestamp = {}
            item_fans = db.GqlQuery("SELECT * FROM ItemFans WHERE listing = :1", listing).fetch(1000)

            db_timestamps = set([appetsy.zero_timezone(item.favored_on) for item in item_fans])


        # if both sets match - go home
        if len(db_timestamps - etsy_timestamps) + len(etsy_timestamps - db_timestamps) == 0:
            self.log("Same fans")
            pending = memcache.get("items_pending_refresh") or []
            if listing.id in pending:
                pending.remove(listing.id)
                memcache.set("items_pending_refresh", pending)
            return


        #new fans in etsy
        item_fans = []
        for timestamp in (etsy_timestamps - db_timestamps):
            favorer = etsy_favorers_by_timestamp[timestamp]
            fan = self.get_fan(favorer)

            item_fans.append(storage.ItemFans(fan = fan,
                                      listing = listing,
                                      shop = listing.shop,
                                      favored_on = appetsy.etsy_epoch(favorer.creation_tsz)))
            self.log("  New fan: '%s'" % (fan.user_name or fan.key()))
        db.put(item_fans)

        #gone fans
        for timestamp in (db_timestamps - etsy_timestamps):
            # FIXME potentially expensive - this will be replaced by key name
            ex_favorer = storage.ItemFans.all().filter("shop =", listing.shop) \
                                       .filter("listing =", listing) \
                                       .filter("favored_on =", timestamp).get()
            self.log("  Removed fave: '%s'" % ex_favorer.fan.user_name)
            db.delete(ex_favorer)


        pending = memcache.get("items_pending_refresh") or []
        if listing.id in pending:
            pending.remove(listing.id)
            memcache.set("items_pending_refresh", pending)

        #update listing fan cache
        memcache.set("listing:%d_fans" % listing.id, etsy_timestamps)

        appetsy.invalidate_memcache("fans", namespace = str(listing.shop.id))
コード例 #4
0
ファイル: cron.py プロジェクト: wangjun/appetsy
    def get(self):
        if not self.etsy:
            return

        """
            stores most recent hundred and offers to go deeper if there is more
        """
        if self.request.get("rebuild"):
            all_fans = storage.ShopFans.all(keys_only = True)
            for i in range(0, all_fans.count(), 500):
                db.delete(all_fans[i:i+499]);

        page = int(self.request.get("page")) if self.request.get("page") else 0


        if not self.ping():
            return

        shop_id = self.request.get("shop")
        if not shop_id:
            for shop in self.shops:
                taskqueue.add(url='/cron/shop?shop=%d' % shop.id, method = 'get')
            return

        shop = storage.EtsyShops.get_by_key_name(shop_id)
        if not shop:
            self.log("Can't get shop by id %s" % shop_id)
            return


        self.write("<html><body><pre>")
        self.log("%s: Getting shop fans (hundred at a time)\n" % shop.shop_name)


        all_seen = False
        offset = page * 100
        fan_number = offset
        fans_added = 0
        fans = []

        try:
            shop_info = self.etsy.getUser(shop.shop_name, ["FavoredBy:100:%d" % offset])
            fans = shop_info.FavoredBy

            if not fans:
                self.log("No fans!")
                return

            # if we are on first page, let's just check for first fan
            # if we have seen him, we go home
            if page == 0:
                seen_last = db.Query(storage.ShopFans).filter("shop =", shop) \
                                              .filter("favored_on =", appetsy.etsy_epoch(fans[0].creation_tsz)) \
                                              .get()
                if seen_last:
                    self.log("Seen last!")
                    fans_added = -1
                    return


            all_fans = storage.ShopFans.all().filter("shop =", shop) \
                                     .filter("favored_on >=", appetsy.etsy_epoch(fans[-1].creation_tsz)).fetch(1000)

            favored_datetimes = [fan.favored_on.replace(tzinfo=appetsy.UtcTzinfo()) for fan in all_fans]

            fan_nicknames = [fan.fan.user_name for fan in all_fans]

            new_fans = []
            for fan in fans:
                fan_number +=1
                fan.date = appetsy.etsy_epoch(fan.creation_tsz)

                user_name = "secret"
                if hasattr(fan, 'user_id'):
                    user_name = fan.user_id

                if fan.date in favored_datetimes:
                    self.write("%d. Have seen this one %s (%s)" % (fan_number,
                                                                   user_name,
                                                                   fan.date))
                else:
                    fans_added +=1
                    self.log("%d. New fan %s (%s)" % (fan_number,
                                                      user_name,
                                                      fan.date))
                    db_fan = self.get_fan(fan)
                    new_fans.append(storage.ShopFans(shop = shop,
                                       fan = db_fan,
                                       favored_on = fan.date))
            db.put(new_fans)


        except DeadlineExceededError:
            self.log("*** Request timed out!")
        except Timeout:
            self.log("*** Database timed out!")
        else:
            if fans_added == 0:
                self.log("Nothing new in %d-%d range" % (offset, offset+100))
            elif fans_added > 0:
                appetsy.invalidate_memcache("fans", namespace=str(shop.id))
            if fans_added == 100: # all new means there are more
                self.log("There are more - going for next batch")
                taskqueue.add(url='/cron/shop?shop=%d&page=%d' % (shop.id, page+1), method = 'get')
        finally:
            self.write("</pre></body></html>")