def delete(self, shop_id):
shop = storage.EtsyShops.get_by_key_name(shop_id)
def delete_records(item):
records = item.all(keys_only=True).filter("shop =", shop).fetch(300)
logging.info("Deleting %d %s" % (len(records), str(item)))
if records:
db.delete(records)
return len(records)
#we are asked to delete a shop
deleted = 0
deleted += delete_records(storage.Events)
deleted += delete_records(storage.Frontpaged)
deleted += delete_records(storage.Counters)
deleted += delete_records(storage.Totals)
deleted += delete_records(storage.Goods)
deleted += delete_records(storage.Expenses)
deleted += delete_records(storage.ShopFans)
deleted += delete_records(storage.ItemFans)
deleted += delete_records(storage.EtsyListings)
logging.info(deleted)
if deleted == 0:
db.delete(shop)
appetsy.invalidate_memcache(namespace = shop_id)
appetsy.invalidate_memcache("users")
self.redirect("/admin")
return deleted
def _mark_sold(self, listing, item = None):
if listing.state != "sold_out":
return
if item is None:
item = db.Query(storage.Goods).filter("listing =", listing).get()
if item:
item.status = "sold"
item.sold = dt.datetime.now()
if item.shop.currency == "LVL":
item.price = round(float(item.listing.price) * 0.55789873, 2) #exchange rate usd -> lvl feb-23-2010
else:
item.price = float(item.listing.price)
item.put()
storage.Totals.add_income(item.shop,
appetsy.today(item.shop),
item.price)
self.log("Marked '%s' as sold." % item.name)
listing.sold_on = dt.datetime.now()
listing.put()
appetsy.invalidate_memcache("goods", str(item.shop.id)) #forget UI listings
def get(self):
"""forget all the cached stuff"""
if self.request.get("everything"):
memcache.flush_all()
else:
for shop in storage.EtsyShops.all():
appetsy.invalidate_memcache(namespace = str(shop.id))
self.redirect("/")
def user_check(self):
users = memcache.get("users")
for email in users:
for shop_key in users[email]:
s = EtsyShops.get(shop_key)
if not s:
u = Users.get_by_key_name(self.user.email)
u.shops.remove(shop_key)
u.put()
appetsy.invalidate_memcache("users")
def update(self):
plan = storage.Counters.get_or_insert("%s:week_%s_plans" % (self.shop.id, appetsy.monday(self.shop).strftime("%U")),
shop = self.shop,
name = "weeks_plan",
count = 10,
timestamp = dt.datetime.combine(appetsy.monday(self.shop), dt.time()))
plan.count = int(self.request.get("weeks_plan"))
plan.put()
memcache.set("week_%s_plans" % appetsy.monday(self.shop).strftime("%U"), plan,
namespace = str(self.shop.id))
appetsy.invalidate_memcache("goods", namespace = str(self.shop.id))
def get(self):
#new user
email = self.request.get("email")
shop_name = self.request.get("shop_name")
if not email:
self.response.out.write("Give me email!")
return
google_user = users.User(email)
if not google_user:
return "bad user"
user = storage.Users.get_or_insert(email, email = email)
if shop_name:
from lib.etsy import Etsy2
etsy = Etsy2(storage.Params.get_param("etsy_key"))
user_info = etsy.getUser(shop_name, ["Profile", "Shops"])
import logging
logging.info(user_info)
shop_id = user_info.user_id
shop = storage.EtsyShops.get_or_insert(str(shop_id),
id = shop_id,
shop_name = shop_name)
shop.shop_name = shop_name
shop.put()
# go for etsy user info to set the icon and sold count
if shop.image_url != user_info.Profile.image_url_75x75:
self.write("Setting shop icon")
shop.image_url = user_info.Profile.image_url_75x75
shop.put()
if shop.key() not in user.shop_keys:
self.write("Adding %s to '%s' shop" % (user.email, shop.shop_name))
user.shop_keys.append(shop.key())
if self.request.get("greeting"):
user.greeting = self.request.get("greeting")
user.put()
appetsy.invalidate_memcache("users")
self.redirect("/admin")
def create(self):
price = float(self.request.get("price") or 0)
purchase_date = dt.datetime.strptime(self.request.get("date"), "%d-%b-%Y").date()
new_expense = storage.Expenses(
name = self.request.get("name"),
purchase_date = purchase_date,
price = price,
shop = self.shop
)
new_expense.put()
if price:
storage.Totals.add_expense(self.shop, purchase_date, price)
appetsy.invalidate_memcache("expenses", namespace = str(self.shop.id))
return new_expense.key()
def update(self, id):
good = storage.Goods.get(id)
listing = None
if self.request.get("listing_key"):
listing = storage.EtsyListings.get(self.request.get("listing_key"))
status = self.request.get("status")
price, sale_date = None, None
if good.listing != listing: #if listing has changed, loose the cache
if good.listing:
good.listing.in_goods = False
db.put(good.listing)
memcache.delete("listing:%d" % good.listing.id)
if listing:
listing.in_goods = True
db.put(listing)
memcache.delete("listing:%d" % listing.id)
memcache.delete("available_listings", namespace = str(self.shop.id))
if status == "sold":
sale_date = dt.datetime.strptime(self.request.get("sale_date"),
"%d-%b-%Y").replace(minute=1)
price = float(self.request.get("price") or 0)
if good.sold and good.price: #remove previous income
storage.Totals.add_income(self.shop, good.sold, -good.price)
good.name = self.request.get("good_name")
good.created = dt.datetime.strptime(self.request.get("creation_date"), "%d-%b-%Y").date()
good.status = status
good.sold = sale_date
good.price = price
good.listing = listing
good.put()
#add new income
if good.sold and good.price:
storage.Totals.add_income(self.shop, good.sold, good.price)
appetsy.invalidate_memcache("goods", namespace = str(self.shop.id))
return good.key()
def __update_listing(self, d, e):
"""update database listing d with data of etsy listing e"""
changes = False
if d.state != e.state:
self.log("'%s' state '%s' --> '%s'" % (d.title, d.state, e.state))
d.state = e.state
if e.state != "active": #we don't care for edits much as they don't change ranking or anything
storage.Events(listing = d, shop = d.shop, event = d.state).put()
changes = True
if hasattr(e, "title") and d.title != e.title:
self.log("'%s' title --> '%s'" % (d.title, e.title))
d.title = e.title
changes = True
if hasattr(e, "views") and d.views != int(e.views):
views = int(e.views) - (d.views or 0)
self.log("'%s' views +%d" % (d.title, views))
d.views = int(e.views)
self.__add_exposure(d, views)
changes = True
if hasattr(e, "ending_tsz") and appetsy.zero_timezone(d.ending) != appetsy.etsy_epoch(e.ending_tsz):
if not d.ending:
storage.Events(listing = d,
shop = d.shop,
event = "posted",
created = appetsy.etsy_epoch(e.creation_tsz)).put()
else:
self.log("'%s' renewed" % d.title)
storage.Events(listing = d, shop = d.shop, event = "renewed").put()
d.ending = appetsy.etsy_epoch(e.ending_tsz)
changes = True
if hasattr(e, "price") and d.price != float(e.price):
self.log("'%s' price %.2f --> %.2f" % (e.title, (d.price or 0.0), float(e.price)))
d.price = float(e.price)
changes = True
if changes:
memcache.set("listing:%d" % d.id, d) #store our memcache copy for next cron
appetsy.invalidate_memcache("goods", str(d.shop.id)) #forget UI listings
return changes
def update(self, id):
expense = storage.Expenses.get(id)
if expense.price:
storage.Totals.add_expense(self.shop, expense.purchase_date, -expense.price)
purchase_date = dt.datetime.strptime(self.request.get("purchase_date"),
"%d-%b-%Y").date()
price = float(self.request.get("price") or 0)
expense.name = self.request.get("expense_name")
expense.purchase_date = dt.datetime.strptime(self.request.get("purchase_date"), "%d-%b-%Y").date()
expense.price = price
expense.put()
storage.Totals.add_expense(self.shop, expense.purchase_date, expense.price)
appetsy.invalidate_memcache("expenses", namespace = str(self.shop.id))
return expense.key()
def create(self):
status = self.request.get("status")
sale_date, price = None, None
if status == "sold":
sale_date = dt.datetime.strptime(self.request.get("sale_date"), "%d-%b-%Y")
price = float(self.request.get("price") or 0)
new_good = storage.Goods(
name = self.request.get("good_name"),
created = dt.datetime.strptime(self.request.get("creation_date"), "%d-%b-%Y").date(),
status = self.request.get("status"),
sold = sale_date,
price = price,
shop = self.shop
)
new_good.put()
if price and sale_date:
storage.Totals.add_income(self.shop, sale_date, price)
appetsy.invalidate_memcache("goods", namespace = str(self.shop.id))
return new_good.key()
def listings_to_goods(self):
shop_id = self.request.get("shop")
shop = storage.EtsyShops.get_by_key_name(shop_id)
if not shop:
self.write("Give me shop")
return
goods = storage.Goods.all().filter("shop =", shop) \
.filter("status =", "in_stock") \
.fetch(500)
db.delete(goods)
listings = storage.EtsyListings.all().filter("shop =", shop) \
.filter("state =", "active") \
.fetch(500)
inserts = []
for i, listing in enumerate(listings):
inserts.append(
storage.Goods(
name = listing.title,
created = (listing.ending - dt.timedelta(days = 130)).date(),
status = "in_stock",
listing = listing,
shop = self.shop,
)
)
listing.in_goods = True
inserts.append(listing)
if i % 100 == 0:
db.put(inserts)
inserts = []
db.put(inserts)
appetsy.invalidate_memcache("goods", namespace = str(shop.id))
def fans_today(self):
today = appetsy.today(self.shop) #- dt.timedelta(days=2)
if today.date() != memcache.get("today", namespace = str(self.shop.id)):
appetsy.invalidate_memcache("fans", namespace = str(self.shop.id))
page = self.request.get("page") or "persons"
fan_list = memcache.get("fan_list", namespace = str(self.shop.id))
if not fan_list: #use cache whenever
data = {}
by_fan = {}
shopfaves = storage.ShopFans.all().filter("shop =", self.shop) \
.filter("favored_on >=", today) \
.fetch(500)
itemfaves = storage.ItemFans.all().filter("shop =", self.shop) \
.filter("favored_on >=", today) \
.fetch(500)
data["shopfave_count"] = len(shopfaves)
for fave in shopfaves:
fan = dict(user_name = fave.fan.user_name,
image_url = fave.fan.image_url,
favored_on = fave.favored_on,
key = str(fave.fan.key()),
count = 0)
by_fan[fave.fan.user_name] = fan
data["itemfave_count"] = len(itemfaves)
by_listing = {}
for fave in itemfaves:
listing = dict(title = fave.listing.title,
key = str(fave.listing.key()),
image_url = fave.listing.image_url,
count = 0,
favored_on = fave.favored_on)
by_listing.setdefault(fave.listing.id, listing)
by_listing[fave.listing.id]["count"] += 1
by_listing[fave.listing.id]["favored_on"] = min(by_listing[fave.listing.id]["favored_on"],
fave.favored_on)
fan = dict(user_name = fave.fan.user_name,
image_url = fave.fan.image_url,
key = str(fave.fan.key()),
favored_on = fave.favored_on,
count = 0)
by_fan.setdefault(fave.fan.user_name, fan)
by_fan[fave.fan.user_name]["favored_on"] = min (by_fan[fave.fan.user_name]["favored_on"],
fave.favored_on)
by_fan[fave.fan.user_name]["count"] += 1
data["shopfaves"] = sorted(by_fan.values(), key = lambda x: x["favored_on"])
data["listingfaves"] = sorted(by_listing.values(), key = lambda x: x["favored_on"])
fan_list = appetsy.get_template("index/fans_today.html").render(**data)
memcache.set("fan_list", fan_list, namespace = str(self.shop.id))
memcache.set("today", today.date(), namespace = str(self.shop.id))
#lame current page hack to avoid breaking the cache
if page == "persons":
fan_list = fan_list.replace('<div id="person_box" style="display: none">', '<div id="person_box">')
else:
fan_list = fan_list.replace('<div id="item_box" style="display: none">', '<div id="item_box">')
return fan_list
def get(self):
if not self.etsy:
return
listing_id = self.request.get("listing")
if not listing_id:
self.log("no listing - exiting!")
return
listing = self.get_listing(listing_id)
if not listing:
self.log("Can't find listing with id %s" % listing_id)
self.log("%s: Getting details for %s (%d known faves)..." % (listing.shop.shop_name,
listing.title,
listing.faves or 0))
# get all etsy fans
etsy_favorers_by_timestamp = {}
def get_listing_info(offset = 0):
if offset > 0:
self.log("%d - %d..." % (offset, offset + 100))
listing_info = self.etsy.getListing(listing.id, ["Images", "FavoredBy:100:%d" % offset])
if len(listing_info.FavoredBy) > 0 and len(listing_info.FavoredBy) % 100 == 0:
listing_info.FavoredBy.extend(get_listing_info(offset + 100).FavoredBy)
return listing_info
listing_info = get_listing_info()
for item in listing_info.FavoredBy:
etsy_favorers_by_timestamp[appetsy.etsy_epoch(item.creation_tsz)] = item
etsy_timestamps = set(etsy_favorers_by_timestamp.keys())
#update fan count for item
listing.faves = len(etsy_timestamps)
listing.image_url = listing_info.Images[0].url_75x75
listing.put()
db_timestamps = memcache.get("listing:%d_fans" % listing.id)
if not db_timestamps:
#get all db fans
db_favorers_by_timestamp = {}
item_fans = db.GqlQuery("SELECT * FROM ItemFans WHERE listing = :1", listing).fetch(1000)
db_timestamps = set([appetsy.zero_timezone(item.favored_on) for item in item_fans])
# if both sets match - go home
if len(db_timestamps - etsy_timestamps) + len(etsy_timestamps - db_timestamps) == 0:
self.log("Same fans")
pending = memcache.get("items_pending_refresh") or []
if listing.id in pending:
pending.remove(listing.id)
memcache.set("items_pending_refresh", pending)
return
#new fans in etsy
item_fans = []
for timestamp in (etsy_timestamps - db_timestamps):
favorer = etsy_favorers_by_timestamp[timestamp]
fan = self.get_fan(favorer)
item_fans.append(storage.ItemFans(fan = fan,
listing = listing,
shop = listing.shop,
favored_on = appetsy.etsy_epoch(favorer.creation_tsz)))
self.log(" New fan: '%s'" % (fan.user_name or fan.key()))
db.put(item_fans)
#gone fans
for timestamp in (db_timestamps - etsy_timestamps):
# FIXME potentially expensive - this will be replaced by key name
ex_favorer = storage.ItemFans.all().filter("shop =", listing.shop) \
.filter("listing =", listing) \
.filter("favored_on =", timestamp).get()
self.log(" Removed fave: '%s'" % ex_favorer.fan.user_name)
db.delete(ex_favorer)
pending = memcache.get("items_pending_refresh") or []
if listing.id in pending:
pending.remove(listing.id)
memcache.set("items_pending_refresh", pending)
#update listing fan cache
memcache.set("listing:%d_fans" % listing.id, etsy_timestamps)
appetsy.invalidate_memcache("fans", namespace = str(listing.shop.id))
def get(self):
if not self.etsy:
return
"""
stores most recent hundred and offers to go deeper if there is more
"""
if self.request.get("rebuild"):
all_fans = storage.ShopFans.all(keys_only = True)
for i in range(0, all_fans.count(), 500):
db.delete(all_fans[i:i+499]);
page = int(self.request.get("page")) if self.request.get("page") else 0
if not self.ping():
return
shop_id = self.request.get("shop")
if not shop_id:
for shop in self.shops:
taskqueue.add(url='/cron/shop?shop=%d' % shop.id, method = 'get')
return
shop = storage.EtsyShops.get_by_key_name(shop_id)
if not shop:
self.log("Can't get shop by id %s" % shop_id)
return
self.write("<html><body><pre>")
self.log("%s: Getting shop fans (hundred at a time)\n" % shop.shop_name)
all_seen = False
offset = page * 100
fan_number = offset
fans_added = 0
fans = []
try:
shop_info = self.etsy.getUser(shop.shop_name, ["FavoredBy:100:%d" % offset])
fans = shop_info.FavoredBy
if not fans:
self.log("No fans!")
return
# if we are on first page, let's just check for first fan
# if we have seen him, we go home
if page == 0:
seen_last = db.Query(storage.ShopFans).filter("shop =", shop) \
.filter("favored_on =", appetsy.etsy_epoch(fans[0].creation_tsz)) \
.get()
if seen_last:
self.log("Seen last!")
fans_added = -1
return
all_fans = storage.ShopFans.all().filter("shop =", shop) \
.filter("favored_on >=", appetsy.etsy_epoch(fans[-1].creation_tsz)).fetch(1000)
favored_datetimes = [fan.favored_on.replace(tzinfo=appetsy.UtcTzinfo()) for fan in all_fans]
fan_nicknames = [fan.fan.user_name for fan in all_fans]
new_fans = []
for fan in fans:
fan_number +=1
fan.date = appetsy.etsy_epoch(fan.creation_tsz)
user_name = "secret"
if hasattr(fan, 'user_id'):
user_name = fan.user_id
if fan.date in favored_datetimes:
self.write("%d. Have seen this one %s (%s)" % (fan_number,
user_name,
fan.date))
else:
fans_added +=1
self.log("%d. New fan %s (%s)" % (fan_number,
user_name,
fan.date))
db_fan = self.get_fan(fan)
new_fans.append(storage.ShopFans(shop = shop,
fan = db_fan,
favored_on = fan.date))
db.put(new_fans)
except DeadlineExceededError:
self.log("*** Request timed out!")
except Timeout:
self.log("*** Database timed out!")
else:
if fans_added == 0:
self.log("Nothing new in %d-%d range" % (offset, offset+100))
elif fans_added > 0:
appetsy.invalidate_memcache("fans", namespace=str(shop.id))
if fans_added == 100: # all new means there are more
self.log("There are more - going for next batch")
taskqueue.add(url='/cron/shop?shop=%d&page=%d' % (shop.id, page+1), method = 'get')
finally:
self.write("</pre></body></html>")