def append_review_jsoncsv(jstr, rev):
jstr = jstr or ""
if jstr:
jstr += ","
jstr += moracct.obj2JSON(rev)
if rev.ctmid:
# append coop srcrevs for client cache reference
srcrev = Review.get_by_id(int(rev.srcrev))
if not srcrev:
logging.error("Missing srcrev for Review " + str(rev.key().id()))
else:
jstr += "," + moracct.obj2JSON(srcrev)
return jstr
def get_review_feed_pool(revtype):
feedcsv, blocks = get_cached_feed_pool(revtype)
if feedcsv:
return feedcsv, blocks
feedcsv, blocks = get_precomp_feed_pool(revtype)
if feedcsv:
return feedcsv, blocks
logging.info("rebuilding feedcsv for " + revtype)
cacheavail = 1040000 # db limit max 1,048,487
feedcsv = ""
blocks = ["" for i in range(numblocks)]
bidx = 0
count = 0
where = "WHERE ctmid = 0 AND mainfeed = 1"
if revtype and revtype != "all":
where += " AND revtype = '" + revtype + "'"
where += " ORDER BY modhist DESC"
vq = VizQuery(rev.Review, where)
reviews = vq.fetch(revpoolsize, read_policy=db.EVENTUAL_CONSISTENCY,
deadline=60)
for review in reviews:
entry = feedcsventry(review)
if csv_contains(entry, feedcsv):
continue # already added, probably from earlier srcrev reference
objson = moracct.obj2JSON(review)
cacheavail -= len(entry) + len(objson)
feedcsv = append_to_csv(entry, feedcsv)
blocks[bidx] = append_to_csv(objson, blocks[bidx])
count += 1
if review.srcrev and review.srcrev > 0: # ensure src included in block
source = rev.Review.get_by_id(review.srcrev)
if source:
entry = feedcsventry(source)
if not csv_contains(entry, feedcsv):
objson = moracct.obj2JSON(source)
feedcsv = append_to_csv(entry, feedcsv)
cacheavail -= len(entry) + len(objson)
blocks[bidx] = append_to_csv(objson, blocks[bidx])
count += 1
if count >= revblocksize:
bidx += 1
count = 0
if cacheavail <= 0:
break
for i in range(numblocks):
blocks[i] = "[" + blocks[i] + "]"
write_precomp_feed_pool(revtype, feedcsv, blocks, True)
write_cached_feed_pool(revtype, feedcsv, blocks)
return feedcsv, blocks
def smart_retrieve_revinst(revid, penid, coopid=0):
jstr = memcache.get("allRevBlock0")
if jstr and str(revid) in jstr: # skip loads if id not in string
revlist = json.loads(jstr)
for rev in revlist:
if rev["_id"] == str(revid):
return rev
if int(penid) > 0: # typically pass zero for penid if already searched
jstr = memcache.get("pen" + str(penid))
if jstr and str(revid) in jstr:
revlist = json.loads(jstr)[1:] # first element is PenName
for rev in revlist:
if rev["_id"] == str(revid):
return rev
if int(coopid) > 0: # coopid passed only if relevant
jstr = memcache.get("coop" + str(coopid))
if jstr and str(revid) in jstr:
revlist = json.loads(jstr)[1:] # first element is Coop
for rev in revlist:
if rev["_id"] == str(revid):
return rev
rev = visible_get_instance(Review, revid)
if rev:
rev = json.loads(moracct.obj2JSON(rev)) # return cache representation
return rev
def get(self):
try:
pct, pgid, mypen = find_pen_or_coop_type_and_id(self)
if pgid: # pgid may be zero if no pen name yet
if self.request.get('supp'):
return supplemental_recent_reviews(self, pgid)
jstr = mblock.get_membics_json_for_profile(pct, pgid)
else: # return empty array with no first item pen
jstr = "[]"
if jstr and mypen: # replace first element with private pen data
i = 2
brackets = 1
while i < len(jstr) and brackets > 0:
if jstr[i] == '{':
brackets += 1
elif jstr[i] == '}':
brackets -= 1
i += 1
jstr = '[' + moracct.obj2JSON(mypen) + jstr[i:]
if not jstr:
srverr(handler, 404, pct + " " + str(pgid) + " not found")
else:
moracct.writeJSONResponse(jstr, self.response)
return
except Exception as e:
if str(e) == "Token expired":
return srverr(self, 401, "Your access token has expired, you will need to sign in again.")
logging.warn("FetchAllReviews failed: " + str(e))
return srverr(self, 500, "FetchAllReviews failed: " + str(e))
def prepare_content(handler, gcname, content):
# Same index retrieval used for unique group name checking
groups = Group.gql("WHERE name_c=:1 LIMIT 1", gcname)
if groups.count() != 1:
handler.error(404)
handler.response.out.write("Group identifier " + gcname + " not found")
return
group = groups[0];
qres = recent_group_reviews(group)
picurl = "img/emptyprofpic.png"
if group.picture:
picurl = "grppic?groupid=" + str(group.key().id())
# facebook doesn't like "../" relative urls
if "localhost" in handler.request.url:
picurl = "../" + picurl
else:
picurl = "http://www.fgfweb.com/" + picurl
content = re.sub('\$GROUPNAME', group.name, content)
content = re.sub('\$GROUPDESCR', group.description, content)
content = re.sub('\$IMGSRC', picurl, content)
content = re.sub('\$GROUPID', str(group.key().id()), content)
content = re.sub('\$GROUPJSON', obj2JSON(group), content)
content = re.sub('\$REVDATA', qres2JSON(
qres.objects, "", -1, ""), content)
refer = handler.request.referer
if refer:
refer = "<img src=\"../bytheimg?grpinqref=" +\
safeURIEncode(refer) + "\"/>\n"
else:
refer = "<img id=\"btwimg\" src=\"../bytheimg?grpinq=" +\
str(group.key().id()) + "\"/>\n"
content = re.sub('\$REFER', refer, content)
content = re.sub('\"', "\\\"", content) #browser interp pre-parse
return content
def get(self, revid):
review = cached_get(intz(revid), Review)
if not review:
self.error(404)
self.response.out.write("Review " + revid + " not found")
return
pen = cached_get(review.penid, PenName)
if not pen:
self.error(404)
self.response.out.write("PenName " + review.penid + " not found")
return
# filter sensitive PenName fields
pen.mid = 0
pen.gsid = "0"
pen.fbid = 0
pen.twid = 0
pen.ghid = 0
pen.abusive = ""
# write content
logging.info("request: " + str(self.request))
revtitle = review.title;
if not revtitle:
revtitle = review.name;
rdesc = descrip(review)
timg = "../img/" + typeImage(review.revtype)
simg = timg[0:-6] + "Pic2.png"
content = html
content = write_group_content(content, review)
content = re.sub('\$REVDESC', rdesc, content)
content = re.sub('\$IMGSRC', simg, content)
content = re.sub('\$PENNAME', pen.name, content)
content = re.sub('\$REVTITLE', revtitle, content)
content = re.sub('\$PENID', str(pen.key().id()), content)
content = re.sub('\$PENJSON', obj2JSON(pen), content)
content = re.sub(', "abusive": ""', '', content) #ugly field name 2 C
content = re.sub('\$REVJSON', obj2JSON(review), content)
refer = self.request.referer
if refer:
refer = "<img src=\"../bytheimg?statinqref=" +\
safeURIEncode(refer) + "\"/>\n"
else:
refer = "<img id=\"btwimg\" src=\"../bytheimg?statinq=" +\
str(review.key().id()) + "\"/>\n"
content = re.sub('\$REFER', refer, content)
content = re.sub('\"', "\\\"", content) #browser interp pre-parse
self.response.headers['Content-Type'] = "text/html; charset=UTF-8";
self.response.out.write(content);
def prepend_instance_to_json(review, jtxt):
# not safe to use prepend_to_csv due to upperbound limit
jtxt = jtxt or ""
if jtxt:
if len(jtxt) > 2:
jtxt = jtxt[1:-1] # strip array brackets
else:
jtxt = ""
if jtxt:
jtxt = "," + jtxt
jtxt = moracct.obj2JSON(review) + jtxt
return "[" + jtxt + "]"
def replace_instance_in_json(review, jtxt, remove):
objs = json.loads(jtxt)
rt = ""
for obj in objs:
idstr = str(review.key().id())
if obj["_id"] == idstr:
if not remove:
moracct.debuginfo("Replaced json for _id " + idstr)
rt = append_to_csv(moracct.obj2JSON(review), rt)
else:
moracct.debuginfo("Removed json for _id " + idstr)
else:
rt = append_to_csv(json.dumps(obj), rt)
return "[" + rt + "]"
def update_prof_cache(ckey, rev):
if isinstance(rev, db.Model):
rev = json.loads(moracct.obj2JSON(rev))
jstr = memcache.get(ckey)
if jstr and rev["_id"] in jstr:
moracct.debuginfo("update_prof_cache Review " + rev["_id"] +
" in " + ckey)
replist = []
cachelist = json.loads(jstr)
for idx, cached in enumerate(cachelist):
if idx > 0 and cached["_id"] == rev["_id"]:
replist.append(rev)
else:
replist.append(cached)
memcache.set(ckey, json.dumps(replist))
def add_instance(self, inst, supporting=False, prepend=False):
if isinstance(inst, db.Model):
inst = json.loads(moracct.obj2JSON(inst))
if str(inst["_id"]) not in self.cache:
if supporting:
self.supplist.append(inst)
else:
if prepend:
self.mainlist.insert(0, inst)
else:
self.mainlist.append(inst)
elif prepend: # need to move existing instance to top
self.mainlist = [inst] + [r for r in self.mainlist if\
r["_id"] != inst["_id"]]
self.cache[str(inst["_id"])] = inst
def rebuild_membics_block(pct, pgid):
logging.info("rebuild_membics_block " + pct + " " + str(pgid))
acc = None
pco = None
if pct == "coop":
pco = coop.Coop.get_by_id(int(pgid))
if pco and pco.preb and not coop.prebuilt_membics_stale(pco):
return pco.preb
elif pct == "pen":
pco = pen.PenName.get_by_id(int(pgid))
if pco:
pen.filter_sensitive_fields(pco)
if not pco:
logging.info("rmb " + pct + " " + str(pgid) + " not found")
return None
where = "WHERE ctmid = 0 AND penid = :1 ORDER BY modified DESC"
if pct == "coop":
where = "WHERE ctmid = :1 ORDER BY modified DESC"
vq = VizQuery(rev.Review, where, pco.key().id())
fsz = 100 # 11/30/16 complaint that 50 makes "recent" tab feel lossy
fmax = fsz
if pct == "coop":
fmax = 500
jstr = ""
js2 = ""
membics = vq.fetch(fmax, read_policy=db.EVENTUAL_CONSISTENCY, deadline=60)
idx = 0 # idx not initialized if enumerate punts due to no membics...
for idx, membic in enumerate(membics):
if idx < fsz:
jstr = rev.append_review_jsoncsv(jstr, membic)
else:
js2 = rev.append_review_jsoncsv(js2, membic)
jstr = append_top20_membics_to_jsoncsv(jstr, membics, pct, pco, 450 * 1024)
if jstr:
jstr = "," + jstr;
jstr = moracct.obj2JSON(pco) + jstr;
if pct == "coop":
pco.preb = "[" + jstr + "]"
pco.preb2 = "[" + js2 + "]"
coop.update_coop_stats(pco, idx)
# rebuild preb to include updated stats, maybe s1 off by one but ok.
pco.preb = "[" + jstr + "]"
mctr.synchronized_db_write(pco)
return "[" + jstr + "]"
def prepare_content(handler, cpen, content):
# Same index retrieval already used by pen.py NewPenName
pens = PenName.gql("WHERE name_c=:1 LIMIT 1", cpen)
if pens.count() != 1:
handler.error(404)
handler.response.out.write("Blog identifier " + cpen + " not found")
return
pen = pens[0];
# filter sensitive PenName fields
pen.mid = 0
pen.gsid = "0"
pen.fbid = 0
pen.twid = 0
pen.ghid = 0
pen.abusive = ""
# retrieve reviews
qres = fetch_blog_reviews(pen)
# write content
picurl = "img/emptyprofpic.png"
if pen.profpic:
picurl = "profpic?profileid=" + str(pen.key().id())
# facebook doesn't like "../" relative urls
if "localhost" in handler.request.url:
picurl = "../" + picurl
else:
picurl = "http://www.fgfweb.com/" + picurl
content = re.sub('\$PENNAME', pen.name, content)
content = re.sub('\$PAGEDESCR', make_page_desc(handler, pen), content)
content = re.sub('\$IMGSRC', picurl, content)
content = re.sub('\$PENID', str(pen.key().id()), content)
content = re.sub('\$PENJSON', obj2JSON(pen), content)
content = re.sub(', "abusive": ""', '', content) #bad SEO :-)
content = re.sub('\$REVDATA', qres2JSON(
qres.objects, "", -1, ""), content)
refer = handler.request.referer
if refer:
refer = "<img src=\"../bytheimg?bloginqref=" +\
safeURIEncode(refer) + "\"/>\n"
else:
refer = "<img id=\"btwimg\" src=\"../bytheimg?bloginq=" +\
str(pen.key().id()) + "\"/>\n"
content = re.sub('\$REFER', refer, content)
content = re.sub('\"', "\\\"", content) #browser interp pre-parse
return content
def update_top_membics(cacheprefix, dbprofinst, rev, prc):
topmax = 30 # how many top membics of each type to keep
jstr = dbprofinst.top20s or "{}"
orgdict = json.loads(jstr)
newdict = json.loads(jstr)
# remove any references to rev id across all types in case type changed
ridstr = str(rev.key().id())
for mtype in newdict:
if newdict[mtype] and ridstr in newdict[mtype]:
newdict[mtype].remove(ridstr)
# rebuild the top id list for the current type
tids = []
if rev.revtype in newdict:
tids = newdict[rev.revtype]
trevs = [] # list of resolved top instances to sort
if rev.srcrev not in [-604, -101]: # include rev unless deleted or future
cacherev = json.loads(moracct.obj2JSON(rev)) # cache dict repr
trevs = [cacherev]
for tidstr in tids: # append instances for existing ids to trevs
inst = fetch_validated_instance(tidstr, prc, rev, dbprofinst)
if inst:
trevs.append(inst)
# moracct.debuginfo("update_top_membics about to sort trevs.")
# for idx in range(len(trevs)):
# moracct.debuginfo("trevs[" + str(idx) + "] " + str(trevs[idx]))
trevs = sorted(trevs, key=itemgetter('rating', 'modified'),
reverse=True)
tids = idlist_from_instlist(trevs, topmax)
newdict[rev.revtype] = tids
if "latestrevtype" in orgdict:
newdict["latestrevtype"] = orgdict["latestrevtype"]
if "t20lastupdated" in orgdict:
newdict["t20lastupdated"] = orgdict["t20lastupdated"]
# moracct.debuginfo("orgdict: " + str(orgdict))
# moracct.debuginfo("newdict: " + str(newdict))
if newdict == orgdict:
logging.info("update_top_membics: no change")
return None
newdict["latestrevtype"] = rev.revtype
newdict["t20lastupdated"] = str(nowISO())
logging.info("update_top_membics: recalculated")
return json.dumps(newdict)
def set_db_prof_inst(self, dbprofinst):
self.publicprofinst = json.loads(moracct.obj2JSON(dbprofinst))
if dbprofinst.__class__.__name__ == "PenName":
pen.filter_sensitive_fields(self.publicprofinst)
