def append_review_jsoncsv(jstr, rev): jstr = jstr or "" if jstr: jstr += "," jstr += moracct.obj2JSON(rev) if rev.ctmid: # append coop srcrevs for client cache reference srcrev = Review.get_by_id(int(rev.srcrev)) if not srcrev: logging.error("Missing srcrev for Review " + str(rev.key().id())) else: jstr += "," + moracct.obj2JSON(srcrev) return jstr
def get_review_feed_pool(revtype): feedcsv, blocks = get_cached_feed_pool(revtype) if feedcsv: return feedcsv, blocks feedcsv, blocks = get_precomp_feed_pool(revtype) if feedcsv: return feedcsv, blocks logging.info("rebuilding feedcsv for " + revtype) cacheavail = 1040000 # db limit max 1,048,487 feedcsv = "" blocks = ["" for i in range(numblocks)] bidx = 0 count = 0 where = "WHERE ctmid = 0 AND mainfeed = 1" if revtype and revtype != "all": where += " AND revtype = '" + revtype + "'" where += " ORDER BY modhist DESC" vq = VizQuery(rev.Review, where) reviews = vq.fetch(revpoolsize, read_policy=db.EVENTUAL_CONSISTENCY, deadline=60) for review in reviews: entry = feedcsventry(review) if csv_contains(entry, feedcsv): continue # already added, probably from earlier srcrev reference objson = moracct.obj2JSON(review) cacheavail -= len(entry) + len(objson) feedcsv = append_to_csv(entry, feedcsv) blocks[bidx] = append_to_csv(objson, blocks[bidx]) count += 1 if review.srcrev and review.srcrev > 0: # ensure src included in block source = rev.Review.get_by_id(review.srcrev) if source: entry = feedcsventry(source) if not csv_contains(entry, feedcsv): objson = moracct.obj2JSON(source) feedcsv = append_to_csv(entry, feedcsv) cacheavail -= len(entry) + len(objson) blocks[bidx] = append_to_csv(objson, blocks[bidx]) count += 1 if count >= revblocksize: bidx += 1 count = 0 if cacheavail <= 0: break for i in range(numblocks): blocks[i] = "[" + blocks[i] + "]" write_precomp_feed_pool(revtype, feedcsv, blocks, True) write_cached_feed_pool(revtype, feedcsv, blocks) return feedcsv, blocks
def smart_retrieve_revinst(revid, penid, coopid=0): jstr = memcache.get("allRevBlock0") if jstr and str(revid) in jstr: # skip loads if id not in string revlist = json.loads(jstr) for rev in revlist: if rev["_id"] == str(revid): return rev if int(penid) > 0: # typically pass zero for penid if already searched jstr = memcache.get("pen" + str(penid)) if jstr and str(revid) in jstr: revlist = json.loads(jstr)[1:] # first element is PenName for rev in revlist: if rev["_id"] == str(revid): return rev if int(coopid) > 0: # coopid passed only if relevant jstr = memcache.get("coop" + str(coopid)) if jstr and str(revid) in jstr: revlist = json.loads(jstr)[1:] # first element is Coop for rev in revlist: if rev["_id"] == str(revid): return rev rev = visible_get_instance(Review, revid) if rev: rev = json.loads(moracct.obj2JSON(rev)) # return cache representation return rev
def get(self): try: pct, pgid, mypen = find_pen_or_coop_type_and_id(self) if pgid: # pgid may be zero if no pen name yet if self.request.get('supp'): return supplemental_recent_reviews(self, pgid) jstr = mblock.get_membics_json_for_profile(pct, pgid) else: # return empty array with no first item pen jstr = "[]" if jstr and mypen: # replace first element with private pen data i = 2 brackets = 1 while i < len(jstr) and brackets > 0: if jstr[i] == '{': brackets += 1 elif jstr[i] == '}': brackets -= 1 i += 1 jstr = '[' + moracct.obj2JSON(mypen) + jstr[i:] if not jstr: srverr(handler, 404, pct + " " + str(pgid) + " not found") else: moracct.writeJSONResponse(jstr, self.response) return except Exception as e: if str(e) == "Token expired": return srverr(self, 401, "Your access token has expired, you will need to sign in again.") logging.warn("FetchAllReviews failed: " + str(e)) return srverr(self, 500, "FetchAllReviews failed: " + str(e))
def prepare_content(handler, gcname, content): # Same index retrieval used for unique group name checking groups = Group.gql("WHERE name_c=:1 LIMIT 1", gcname) if groups.count() != 1: handler.error(404) handler.response.out.write("Group identifier " + gcname + " not found") return group = groups[0]; qres = recent_group_reviews(group) picurl = "img/emptyprofpic.png" if group.picture: picurl = "grppic?groupid=" + str(group.key().id()) # facebook doesn't like "../" relative urls if "localhost" in handler.request.url: picurl = "../" + picurl else: picurl = "http://www.fgfweb.com/" + picurl content = re.sub('\$GROUPNAME', group.name, content) content = re.sub('\$GROUPDESCR', group.description, content) content = re.sub('\$IMGSRC', picurl, content) content = re.sub('\$GROUPID', str(group.key().id()), content) content = re.sub('\$GROUPJSON', obj2JSON(group), content) content = re.sub('\$REVDATA', qres2JSON( qres.objects, "", -1, ""), content) refer = handler.request.referer if refer: refer = "<img src=\"../bytheimg?grpinqref=" +\ safeURIEncode(refer) + "\"/>\n" else: refer = "<img id=\"btwimg\" src=\"../bytheimg?grpinq=" +\ str(group.key().id()) + "\"/>\n" content = re.sub('\$REFER', refer, content) content = re.sub('\"', "\\\"", content) #browser interp pre-parse return content
def get(self, revid): review = cached_get(intz(revid), Review) if not review: self.error(404) self.response.out.write("Review " + revid + " not found") return pen = cached_get(review.penid, PenName) if not pen: self.error(404) self.response.out.write("PenName " + review.penid + " not found") return # filter sensitive PenName fields pen.mid = 0 pen.gsid = "0" pen.fbid = 0 pen.twid = 0 pen.ghid = 0 pen.abusive = "" # write content logging.info("request: " + str(self.request)) revtitle = review.title; if not revtitle: revtitle = review.name; rdesc = descrip(review) timg = "../img/" + typeImage(review.revtype) simg = timg[0:-6] + "Pic2.png" content = html content = write_group_content(content, review) content = re.sub('\$REVDESC', rdesc, content) content = re.sub('\$IMGSRC', simg, content) content = re.sub('\$PENNAME', pen.name, content) content = re.sub('\$REVTITLE', revtitle, content) content = re.sub('\$PENID', str(pen.key().id()), content) content = re.sub('\$PENJSON', obj2JSON(pen), content) content = re.sub(', "abusive": ""', '', content) #ugly field name 2 C content = re.sub('\$REVJSON', obj2JSON(review), content) refer = self.request.referer if refer: refer = "<img src=\"../bytheimg?statinqref=" +\ safeURIEncode(refer) + "\"/>\n" else: refer = "<img id=\"btwimg\" src=\"../bytheimg?statinq=" +\ str(review.key().id()) + "\"/>\n" content = re.sub('\$REFER', refer, content) content = re.sub('\"', "\\\"", content) #browser interp pre-parse self.response.headers['Content-Type'] = "text/html; charset=UTF-8"; self.response.out.write(content);
def prepend_instance_to_json(review, jtxt): # not safe to use prepend_to_csv due to upperbound limit jtxt = jtxt or "" if jtxt: if len(jtxt) > 2: jtxt = jtxt[1:-1] # strip array brackets else: jtxt = "" if jtxt: jtxt = "," + jtxt jtxt = moracct.obj2JSON(review) + jtxt return "[" + jtxt + "]"
def replace_instance_in_json(review, jtxt, remove): objs = json.loads(jtxt) rt = "" for obj in objs: idstr = str(review.key().id()) if obj["_id"] == idstr: if not remove: moracct.debuginfo("Replaced json for _id " + idstr) rt = append_to_csv(moracct.obj2JSON(review), rt) else: moracct.debuginfo("Removed json for _id " + idstr) else: rt = append_to_csv(json.dumps(obj), rt) return "[" + rt + "]"
def update_prof_cache(ckey, rev): if isinstance(rev, db.Model): rev = json.loads(moracct.obj2JSON(rev)) jstr = memcache.get(ckey) if jstr and rev["_id"] in jstr: moracct.debuginfo("update_prof_cache Review " + rev["_id"] + " in " + ckey) replist = [] cachelist = json.loads(jstr) for idx, cached in enumerate(cachelist): if idx > 0 and cached["_id"] == rev["_id"]: replist.append(rev) else: replist.append(cached) memcache.set(ckey, json.dumps(replist))
def add_instance(self, inst, supporting=False, prepend=False): if isinstance(inst, db.Model): inst = json.loads(moracct.obj2JSON(inst)) if str(inst["_id"]) not in self.cache: if supporting: self.supplist.append(inst) else: if prepend: self.mainlist.insert(0, inst) else: self.mainlist.append(inst) elif prepend: # need to move existing instance to top self.mainlist = [inst] + [r for r in self.mainlist if\ r["_id"] != inst["_id"]] self.cache[str(inst["_id"])] = inst
def rebuild_membics_block(pct, pgid): logging.info("rebuild_membics_block " + pct + " " + str(pgid)) acc = None pco = None if pct == "coop": pco = coop.Coop.get_by_id(int(pgid)) if pco and pco.preb and not coop.prebuilt_membics_stale(pco): return pco.preb elif pct == "pen": pco = pen.PenName.get_by_id(int(pgid)) if pco: pen.filter_sensitive_fields(pco) if not pco: logging.info("rmb " + pct + " " + str(pgid) + " not found") return None where = "WHERE ctmid = 0 AND penid = :1 ORDER BY modified DESC" if pct == "coop": where = "WHERE ctmid = :1 ORDER BY modified DESC" vq = VizQuery(rev.Review, where, pco.key().id()) fsz = 100 # 11/30/16 complaint that 50 makes "recent" tab feel lossy fmax = fsz if pct == "coop": fmax = 500 jstr = "" js2 = "" membics = vq.fetch(fmax, read_policy=db.EVENTUAL_CONSISTENCY, deadline=60) idx = 0 # idx not initialized if enumerate punts due to no membics... for idx, membic in enumerate(membics): if idx < fsz: jstr = rev.append_review_jsoncsv(jstr, membic) else: js2 = rev.append_review_jsoncsv(js2, membic) jstr = append_top20_membics_to_jsoncsv(jstr, membics, pct, pco, 450 * 1024) if jstr: jstr = "," + jstr; jstr = moracct.obj2JSON(pco) + jstr; if pct == "coop": pco.preb = "[" + jstr + "]" pco.preb2 = "[" + js2 + "]" coop.update_coop_stats(pco, idx) # rebuild preb to include updated stats, maybe s1 off by one but ok. pco.preb = "[" + jstr + "]" mctr.synchronized_db_write(pco) return "[" + jstr + "]"
def prepare_content(handler, cpen, content): # Same index retrieval already used by pen.py NewPenName pens = PenName.gql("WHERE name_c=:1 LIMIT 1", cpen) if pens.count() != 1: handler.error(404) handler.response.out.write("Blog identifier " + cpen + " not found") return pen = pens[0]; # filter sensitive PenName fields pen.mid = 0 pen.gsid = "0" pen.fbid = 0 pen.twid = 0 pen.ghid = 0 pen.abusive = "" # retrieve reviews qres = fetch_blog_reviews(pen) # write content picurl = "img/emptyprofpic.png" if pen.profpic: picurl = "profpic?profileid=" + str(pen.key().id()) # facebook doesn't like "../" relative urls if "localhost" in handler.request.url: picurl = "../" + picurl else: picurl = "http://www.fgfweb.com/" + picurl content = re.sub('\$PENNAME', pen.name, content) content = re.sub('\$PAGEDESCR', make_page_desc(handler, pen), content) content = re.sub('\$IMGSRC', picurl, content) content = re.sub('\$PENID', str(pen.key().id()), content) content = re.sub('\$PENJSON', obj2JSON(pen), content) content = re.sub(', "abusive": ""', '', content) #bad SEO :-) content = re.sub('\$REVDATA', qres2JSON( qres.objects, "", -1, ""), content) refer = handler.request.referer if refer: refer = "<img src=\"../bytheimg?bloginqref=" +\ safeURIEncode(refer) + "\"/>\n" else: refer = "<img id=\"btwimg\" src=\"../bytheimg?bloginq=" +\ str(pen.key().id()) + "\"/>\n" content = re.sub('\$REFER', refer, content) content = re.sub('\"', "\\\"", content) #browser interp pre-parse return content
def update_top_membics(cacheprefix, dbprofinst, rev, prc): topmax = 30 # how many top membics of each type to keep jstr = dbprofinst.top20s or "{}" orgdict = json.loads(jstr) newdict = json.loads(jstr) # remove any references to rev id across all types in case type changed ridstr = str(rev.key().id()) for mtype in newdict: if newdict[mtype] and ridstr in newdict[mtype]: newdict[mtype].remove(ridstr) # rebuild the top id list for the current type tids = [] if rev.revtype in newdict: tids = newdict[rev.revtype] trevs = [] # list of resolved top instances to sort if rev.srcrev not in [-604, -101]: # include rev unless deleted or future cacherev = json.loads(moracct.obj2JSON(rev)) # cache dict repr trevs = [cacherev] for tidstr in tids: # append instances for existing ids to trevs inst = fetch_validated_instance(tidstr, prc, rev, dbprofinst) if inst: trevs.append(inst) # moracct.debuginfo("update_top_membics about to sort trevs.") # for idx in range(len(trevs)): # moracct.debuginfo("trevs[" + str(idx) + "] " + str(trevs[idx])) trevs = sorted(trevs, key=itemgetter('rating', 'modified'), reverse=True) tids = idlist_from_instlist(trevs, topmax) newdict[rev.revtype] = tids if "latestrevtype" in orgdict: newdict["latestrevtype"] = orgdict["latestrevtype"] if "t20lastupdated" in orgdict: newdict["t20lastupdated"] = orgdict["t20lastupdated"] # moracct.debuginfo("orgdict: " + str(orgdict)) # moracct.debuginfo("newdict: " + str(newdict)) if newdict == orgdict: logging.info("update_top_membics: no change") return None newdict["latestrevtype"] = rev.revtype newdict["t20lastupdated"] = str(nowISO()) logging.info("update_top_membics: recalculated") return json.dumps(newdict)
def set_db_prof_inst(self, dbprofinst): self.publicprofinst = json.loads(moracct.obj2JSON(dbprofinst)) if dbprofinst.__class__.__name__ == "PenName": pen.filter_sensitive_fields(self.publicprofinst)