def recent_activity(thresh):
tlf = {}
sav = {}
vq = appuser.VizQuery(daycount.DayCount, "ORDER BY tstamp DESC")
dcs = vq.fetch(42000, read_policy=db.EVENTUAL_CONSISTENCY, deadline=20)
for dc in dcs:
if dc.tstamp < thresh:
break
detail = json.loads(dc.detail)
tlid = detail["tlid"]
if dc.rtype == "tlfetch":
if tlid in tlf:
tlf[tlid] += 1
else:
tlf[tlid] = 1
elif dc.rtype == "tlsave":
if detail["uidp"] not in sav: # save not already noted
sav[detail["uidp"]] = len(detail["pts"].split(","))
summary = ""
for key in tlf:
name = key
tl = timeline.fetch_timeline_by_id(int(key))
if tl:
name += " (" + tl.name + ")"
summary += "tlfetch " + name + ": " + str(tlf[key]) + "\n"
for key in sav:
summary += " tlsave " + key + ": " + str(sav[key]) + "\n"
return summary
def get(self):
acc = appuser.authenticated(self.request)
if not acc:
return srverr(self, 401, "Authentication failed")
if acc.orgid != 1 or acc.lev != 2:
return appuser.srverr(self, 403, "Admin access only.")
vq = appuser.VizQuery(service.AppService, "WHERE name=:1", "pubpts")
svcs = vq.fetch(1, read_policy=db.EVENTUAL_CONSISTENCY, deadline=20)
if not len(svcs): # create the entry as a placeholder
svc = service.AppService(name="pubpts", ckey="", csec="", data="")
svc.put()
res = [] # result accumulator
if len(svcs) > 0 and len(svcs[0].data) > 100:
for ptid in svcs[0].data.split(","):
pt = Point.get_by_id(int(ptid))
if is_deleted_point(pt):
continue
res.append(pt)
else: # no point ids to process, fetch everything
pts = Point.all()
for pt in pts:
if is_deleted_point(pt):
continue
res.append(pt)
appuser.return_json(self, res)
def update_org_recent_points(pt):
# Rebuild org.recpre if empty or null, otherwise splice in pt.
# Maintain max recpre size of 512k to stay within 1mb db obj limit.
organization = org.Organization.get_by_id(int(pt.orgid))
preb = organization.recpre
if preb and len(preb) > 3000: # save db quota, splice in pt
ptid = str(pt.key().id())
preb = json.loads(preb)
# remove existing instance from prebuilt if it exists
upda = [x for x in preb if "instid" in x and x["instid"] != ptid]
# prepend the modified point unless deleted
if not is_deleted_point(pt):
upda.insert(0, point_summary_dict(pt))
preb = json.dumps(upda)
if len(preb) >= 512 * 1024: # exceeded total size, knock last off
upda = upda[0:len(upda) - 1]
preb = json.dumps(upda)
else: # rebuild from database query
vq = appuser.VizQuery(Point, "WHERE orgid=:1 ORDER BY modified DESC",
pt.orgid)
pts = vq.fetch(1000, read_policy=db.EVENTUAL_CONSISTENCY, deadline=20)
preb = ""
for pt in pts:
if len(preb) >= 512 * 1024:
break
if is_deleted_point(pt):
continue
if preb:
preb += ","
preb += json.dumps(point_summary_dict(pt))
preb = "[" + preb + "]"
organization.recpre = preb
appuser.cached_put(None, organization)
def get(self):
acc = appuser.authenticated(self.request)
if not acc:
return srverr(self, 401, "Authentication failed")
params = appuser.read_params(self, ["tlid"]);
tlid = int(params["tlid"])
vq = appuser.VizQuery(TLComp, "WHERE tlid=:1 LIMIT 50", tlid)
res = vq.fetch(50, read_policy=db.EVENTUAL_CONSISTENCY, deadline=40)
appuser.return_json(self, res)
def get_point_by_id_or_source(ptid, source):
pt = None
if ptid:
pt = Point.get_by_id(int(ptid))
if not pt and source:
vq = appuser.VizQuery(Point, "WHERE source=:1 LIMIT 1", source)
pts = vq.fetch(1, read_policy=db.EVENTUAL_CONSISTENCY, deadline=20)
if len(pts) > 0:
pt = pts[0]
return pt
def recent_timeline_edits(thresh):
vq = appuser.VizQuery(timeline.Timeline, "ORDER BY modified DESC")
tls = vq.fetch(1000, read_policy=db.EVENTUAL_CONSISTENCY, deadline=20)
tlsums = ""
for tl in tls:
if tl.modified < thresh:
break
if tlsums:
tlsums += "\n"
tlsum = ("[tlid " + str(tl.key().id()) + "] " + tl.name)
logging.info(tlsum)
tlsums += tlsum
return tlsums
def recent_point_edits(thresh):
vq = appuser.VizQuery(point.Point, "ORDER BY modified DESC")
pts = vq.fetch(1000, read_policy=db.EVENTUAL_CONSISTENCY, deadline=20)
ptsums = ""
for pt in pts:
if pt.modified < thresh:
break
if ptsums:
ptsums += "\n"
ptsum = ("[ptid " + str(pt.key().id()) + "] " + pt.date + " " +
pt.text[0:512])
logging.info(ptsum)
ptsums += ptsum
return ptsums
def get(self):
if not appuser.verify_secure_comms(self):
return
acc = appuser.authenticated(self.request)
if not acc:
return srverr(self, 401, "Authentication failed")
if acc.orgid != 1 or acc.lev != 2:
return appuser.srverr(self, 403, "System admin access only.")
pn = "Placeholder"
vq = appuser.VizQuery(Organization, "WHERE name=:1 LIMIT 1", pn)
orgs = vq.fetch(1, read_policy=db.EVENTUAL_CONSISTENCY, deadline=20)
if len(orgs) > 0:
org = orgs[0]
else:
org = Organization(name=pn)
org.put()
appuser.return_json(self, [org])
def verify_listed_timelines(handler):
url = "https://pastkey.org/docs/tlrec.json"
misms = ""
if appuser.is_local_devenv(handler):
# example handler request.url http://0.0.0.0:9080/periodic
ru = handler.request.url
url = ru[0:ru.rfind("/")] + "/docs/tldev.json"
fpath = "docs" + url[url.rfind("/"):]
jts = "[]"
try:
result = urlfetch.fetch(url, deadline=10)
if not result or result.status_code != 200:
misms += "vlt fetch status " + str(result.status_code) + "\n"
else:
jts = result.content
except Exception as e:
misms += "vlt fetch fail: " + str(e)
jts = json.loads(jts)
rj = ""
vq = appuser.VizQuery(timeline.Timeline,
"WHERE featured < 'Unlisted' ORDER BY featured")
dts = vq.fetch(200, read_policy=db.EVENTUAL_CONSISTENCY, deadline=20)
for idx, dt in enumerate(dts):
if idx >= len(jts):
misms += " [" + str(idx) + "] added"
else:
jt = jts[idx]
# if instid diff, then the ordering has changed.
# if modified, then important details might have changed.
# changes to featured have some urgency in getting reflected.
misms += vlt_strval_comp(idx, dt, jt,
["instid", "featured", "modified"])
if rj:
rj += ",\n"
rj += appuser.dbo2json(dt, skips=["preb"])
rj = "[" + rj + "]\n"
msg = fpath + " up to date."
if misms:
msg = fpath + " OUTDATED. Update with emailed content."
msg += "\n" + misms
try:
appuser.mailgun_send(None, "*****@*****.**", msg, rj)
except Exception as e:
logging.warn("verify_listed_timelines send fail: " + str(e))
return msg
def get(self):
if not appuser.verify_secure_comms(self):
return
acc = appuser.authenticated(self.request)
if not acc:
return srverr(self, 401, "Authentication failed")
params = appuser.read_params(self, ["orgid"])
if not params["orgid"] or int(params["orgid"]) != acc.orgid:
return appuser.srverr(self, 403, "Not your Organization")
vq = appuser.VizQuery(appuser.AppUser, "WHERE orgid=:1",
int(params["orgid"]))
res = vq.fetch(500, read_policy=db.EVENTUAL_CONSISTENCY, deadline=20)
oms = []
for user in res: # only public info and org info, no email etc..
oms.append(public_member_record(user))
logging.info("Org " + params["orgid"] + " has " + str(len(oms)) +
" members")
appuser.return_json(self, oms)
def recent_completions(thresh):
vq = appuser.VizQuery(TLComp, "ORDER BY created DESC")
tlcs = vq.fetch(30, read_policy=db.EVENTUAL_CONSISTENCY, deadline=20)
cs = ""
for tc in tlcs:
if tc.created < thresh:
break
data = json.loads(tc.data)
pavg, pttl, pcount = stats_from_data(data["pts"])
savg, sttl, scount = 0, 0, 0
if "svs" in data:
savg, sttl, scount = stats_from_data(data["svs"])
ttl = pttl + sttl
uname = tc.username or "NONAME"
cs += (uname + " (uid: " + str(tc.userid) + ") completed " +
tc.tlname + " (tlid: " + str(tc.tlid) + ") on " + tc.created +
" ptavg: " + usec(pavg, "s") +
", time: " + usec(ttl, "m") + "\n")
return cs
def fetch_timeline_by_slug(slug):
logging.info("fetch_timeline_by_slug: " + slug)
instid = memcache.get(slug)
if instid:
# if slug was a tlid, then we found the cached version and it should
# be reconstituted. Otherwise we found the tlid
try:
tl = pickle.loads(instid)
return tl
except:
return fetch_timeline_by_id(instid)
# If slug is all numbers, look it up as a tlid
if re.match(r'^[0-9]+$', slug):
return fetch_timeline_by_id(slug)
vq = appuser.VizQuery(Timeline, "WHERE slug=:1 LIMIT 1", slug)
instances = vq.fetch(1, read_policy=db.EVENTUAL_CONSISTENCY, deadline=20)
if len(instances) > 0:
timeline = instances[0]
memcache.set(slug, str(timeline.key().id()))
appuser.cached_put(None, timeline)
return timeline
return None
def recent_access_accounts(thresh):
dtnow = datetime.datetime.utcnow()
remax = appuser.dt2ISO(dtnow - datetime.timedelta(31))
# The leading spaces are needed for lexicographical comparison
sched = {
" 1 day": appuser.dt2ISO(dtnow - datetime.timedelta(hours=4)),
" 7 day": appuser.dt2ISO(dtnow - datetime.timedelta(7)),
"14 day": appuser.dt2ISO(dtnow - datetime.timedelta(14)),
"30 day": appuser.dt2ISO(dtnow - datetime.timedelta(30))
}
names = ""
remsent = ""
vq = appuser.VizQuery(appuser.AppUser, "ORDER BY accessed DESC")
accs = vq.fetch(1000, read_policy=db.EVENTUAL_CONSISTENCY, deadline=20)
for acc in accs:
if acc.accessed < remax:
break # everything else is too old to bother with
remsent += send_reminder_if_needed(acc, sched)
if acc.accessed >= thresh:
if names:
names += ", "
names += acc.email
return names + "\n" + remsent
def update_or_create_timeline(handler, acc, params):
timeline = None
now = appuser.nowISO()
instid = params["instid"] or 0
cname = canonize(params["name"])
vq = appuser.VizQuery(Timeline, "WHERE cname=:1 LIMIT 1", cname)
tls = vq.fetch(1, read_policy=db.EVENTUAL_CONSISTENCY, deadline=20)
if len(tls) > 0:
if int(instid) != tls[0].key().id():
return appuser.srverr(handler, 406, "Name already used")
timeline = tls[0]
if instid and not timeline:
timeline = Timeline.get_by_id(int(instid))
if not timeline:
return appuser.srverr(handler, 404, "No Timeline " + instid)
if not may_edit_timeline(handler, acc, timeline):
return
if not timeline: # not found, create new
timeline = Timeline(name=params["name"], created=now)
timeline.name = params["name"]
timeline.cname = canonize(timeline.name)
timeline.slug = params["slug"] or ""
timeline.title = params["title"] or ""
timeline.subtitle = params["subtitle"] or ""
timeline.featured = params["featured"] or ""
timeline.lang = params["lang"] or "en-US"
timeline.comment = params["comment"] or ""
timeline.about = params["about"] or ""
timeline.ctype = params["ctype"]
timeline.cids = params["cids"] or ""
timeline.svs = params["svs"] or ""
timeline.preb = rebuild_prebuilt_timeline_points(timeline)
timeline.orgid = timeline.orgid or acc.orgid
timeline.modified = now
appuser.cached_put(None, timeline)
return timeline