def __update_listing(self, d, e): """update database listing d with data of etsy listing e""" changes = False if d.state != e.state: self.log("'%s' state '%s' --> '%s'" % (d.title, d.state, e.state)) d.state = e.state if e.state != "active": #we don't care for edits much as they don't change ranking or anything storage.Events(listing = d, shop = d.shop, event = d.state).put() changes = True if hasattr(e, "title") and d.title != e.title: self.log("'%s' title --> '%s'" % (d.title, e.title)) d.title = e.title changes = True if hasattr(e, "views") and d.views != int(e.views): views = int(e.views) - (d.views or 0) self.log("'%s' views +%d" % (d.title, views)) d.views = int(e.views) self.__add_exposure(d, views) changes = True if hasattr(e, "ending_tsz") and appetsy.zero_timezone(d.ending) != appetsy.etsy_epoch(e.ending_tsz): if not d.ending: storage.Events(listing = d, shop = d.shop, event = "posted", created = appetsy.etsy_epoch(e.creation_tsz)).put() else: self.log("'%s' renewed" % d.title) storage.Events(listing = d, shop = d.shop, event = "renewed").put() d.ending = appetsy.etsy_epoch(e.ending_tsz) changes = True if hasattr(e, "price") and d.price != float(e.price): self.log("'%s' price %.2f --> %.2f" % (e.title, (d.price or 0.0), float(e.price))) d.price = float(e.price) changes = True if changes: memcache.set("listing:%d" % d.id, d) #store our memcache copy for next cron appetsy.invalidate_memcache("goods", str(d.shop.id)) #forget UI listings return changes
def get(self): if not self.etsy: return fan_id = self.request.get("id") db_fan = storage.Fans.get_by_key_name(fan_id) etsy_fan = None if fan_id != "_private": etsy_fan = self.etsy.getUser(fan_id, ["Profile", "Shops"]) if hasattr(etsy_fan, "Profile") == False: etsy_fan = None if etsy_fan: db_fan.user_name = etsy_fan.login_name db_fan.image_url = etsy_fan.Profile.image_url_75x75 or "/public/dejo_75x75.png" db_fan.small_image_url = db_fan.image_url db_fan.status = "normal" if etsy_fan.Shops: db_fan.url = "http://www.etsy.com/shop/%s" % etsy_fan.Shops[0].shop_name db_fan.status = "seller" db_fan.joined_on = appetsy.etsy_epoch(etsy_fan.creation_tsz) else: db_fan.image_url = "/public/dejo_slepens_75x75.png" db_fan.small_image_url = "/public/dejo_slepens_25x25.png" db_fan.status = "private" logging.info("Added brand new fan: %s" % (db_fan.user_name or "secret")) db.put(db_fan) # remove from the pending list as we are done now pending = memcache.get("fans_pending_refresh") or [] if fan_id in pending: pending.remove(fan_id) memcache.set("fans_pending_refresh", pending)
def get(self): if not self.etsy: return listing_id = self.request.get("listing") if not listing_id: self.log("no listing - exiting!") return listing = self.get_listing(listing_id) if not listing: self.log("Can't find listing with id %s" % listing_id) self.log("%s: Getting details for %s (%d known faves)..." % (listing.shop.shop_name, listing.title, listing.faves or 0)) # get all etsy fans etsy_favorers_by_timestamp = {} def get_listing_info(offset = 0): if offset > 0: self.log("%d - %d..." % (offset, offset + 100)) listing_info = self.etsy.getListing(listing.id, ["Images", "FavoredBy:100:%d" % offset]) if len(listing_info.FavoredBy) > 0 and len(listing_info.FavoredBy) % 100 == 0: listing_info.FavoredBy.extend(get_listing_info(offset + 100).FavoredBy) return listing_info listing_info = get_listing_info() for item in listing_info.FavoredBy: etsy_favorers_by_timestamp[appetsy.etsy_epoch(item.creation_tsz)] = item etsy_timestamps = set(etsy_favorers_by_timestamp.keys()) #update fan count for item listing.faves = len(etsy_timestamps) listing.image_url = listing_info.Images[0].url_75x75 listing.put() db_timestamps = memcache.get("listing:%d_fans" % listing.id) if not db_timestamps: #get all db fans db_favorers_by_timestamp = {} item_fans = db.GqlQuery("SELECT * FROM ItemFans WHERE listing = :1", listing).fetch(1000) db_timestamps = set([appetsy.zero_timezone(item.favored_on) for item in item_fans]) # if both sets match - go home if len(db_timestamps - etsy_timestamps) + len(etsy_timestamps - db_timestamps) == 0: self.log("Same fans") pending = memcache.get("items_pending_refresh") or [] if listing.id in pending: pending.remove(listing.id) memcache.set("items_pending_refresh", pending) return #new fans in etsy item_fans = [] for timestamp in (etsy_timestamps - db_timestamps): favorer = etsy_favorers_by_timestamp[timestamp] fan = self.get_fan(favorer) item_fans.append(storage.ItemFans(fan = fan, listing = listing, shop = listing.shop, favored_on = appetsy.etsy_epoch(favorer.creation_tsz))) self.log(" New fan: '%s'" % (fan.user_name or fan.key())) db.put(item_fans) #gone fans for timestamp in (db_timestamps - etsy_timestamps): # FIXME potentially expensive - this will be replaced by key name ex_favorer = storage.ItemFans.all().filter("shop =", listing.shop) \ .filter("listing =", listing) \ .filter("favored_on =", timestamp).get() self.log(" Removed fave: '%s'" % ex_favorer.fan.user_name) db.delete(ex_favorer) pending = memcache.get("items_pending_refresh") or [] if listing.id in pending: pending.remove(listing.id) memcache.set("items_pending_refresh", pending) #update listing fan cache memcache.set("listing:%d_fans" % listing.id, etsy_timestamps) appetsy.invalidate_memcache("fans", namespace = str(listing.shop.id))
def get(self): if not self.etsy: return """ stores most recent hundred and offers to go deeper if there is more """ if self.request.get("rebuild"): all_fans = storage.ShopFans.all(keys_only = True) for i in range(0, all_fans.count(), 500): db.delete(all_fans[i:i+499]); page = int(self.request.get("page")) if self.request.get("page") else 0 if not self.ping(): return shop_id = self.request.get("shop") if not shop_id: for shop in self.shops: taskqueue.add(url='/cron/shop?shop=%d' % shop.id, method = 'get') return shop = storage.EtsyShops.get_by_key_name(shop_id) if not shop: self.log("Can't get shop by id %s" % shop_id) return self.write("<html><body><pre>") self.log("%s: Getting shop fans (hundred at a time)\n" % shop.shop_name) all_seen = False offset = page * 100 fan_number = offset fans_added = 0 fans = [] try: shop_info = self.etsy.getUser(shop.shop_name, ["FavoredBy:100:%d" % offset]) fans = shop_info.FavoredBy if not fans: self.log("No fans!") return # if we are on first page, let's just check for first fan # if we have seen him, we go home if page == 0: seen_last = db.Query(storage.ShopFans).filter("shop =", shop) \ .filter("favored_on =", appetsy.etsy_epoch(fans[0].creation_tsz)) \ .get() if seen_last: self.log("Seen last!") fans_added = -1 return all_fans = storage.ShopFans.all().filter("shop =", shop) \ .filter("favored_on >=", appetsy.etsy_epoch(fans[-1].creation_tsz)).fetch(1000) favored_datetimes = [fan.favored_on.replace(tzinfo=appetsy.UtcTzinfo()) for fan in all_fans] fan_nicknames = [fan.fan.user_name for fan in all_fans] new_fans = [] for fan in fans: fan_number +=1 fan.date = appetsy.etsy_epoch(fan.creation_tsz) user_name = "secret" if hasattr(fan, 'user_id'): user_name = fan.user_id if fan.date in favored_datetimes: self.write("%d. Have seen this one %s (%s)" % (fan_number, user_name, fan.date)) else: fans_added +=1 self.log("%d. New fan %s (%s)" % (fan_number, user_name, fan.date)) db_fan = self.get_fan(fan) new_fans.append(storage.ShopFans(shop = shop, fan = db_fan, favored_on = fan.date)) db.put(new_fans) except DeadlineExceededError: self.log("*** Request timed out!") except Timeout: self.log("*** Database timed out!") else: if fans_added == 0: self.log("Nothing new in %d-%d range" % (offset, offset+100)) elif fans_added > 0: appetsy.invalidate_memcache("fans", namespace=str(shop.id)) if fans_added == 100: # all new means there are more self.log("There are more - going for next batch") taskqueue.add(url='/cron/shop?shop=%d&page=%d' % (shop.id, page+1), method = 'get') finally: self.write("</pre></body></html>")