def update_meta(self):
dataproxy = DataStore(self.short_name)
try:
newcount = 0
datasummary = dataproxy.request({"maincommand": "sqlitecommand", "command": "datasummary", "limit": -1})
if "error" not in datasummary:
if "total_rows" in datasummary:
self.record_count = datasummary["total_rows"]
else:
for tabledata in datasummary.get("tables", {}).values():
newcount += tabledata["count"]
# Only update the record count when we have definitely not failed.
self.record_count = newcount
else:
print "logthis", datasummary
except Exception, e:
print "logthis", e
def update_meta(self):
dataproxy = DataStore(self.short_name)
try:
newcount = 0
datasummary = dataproxy.request({
"maincommand": "sqlitecommand",
"command": "datasummary",
"limit": -1
})
if "error" not in datasummary:
if 'total_rows' in datasummary:
self.record_count = datasummary['total_rows']
else:
for tabledata in datasummary.get("tables", {}).values():
newcount += tabledata["count"]
# Only update the record count when we have definitely not failed.
self.record_count = newcount
else:
print "logthis", datasummary
except Exception, e:
print "logthis", e
def scraperinfo(scraper, history_start_date, quietfields, rev):
info = {}
info["short_name"] = scraper.short_name
info["language"] = scraper.language
info["created"] = scraper.created_at.isoformat()
info["title"] = scraper.title
info["description"] = scraper.description_safepart()
info["tags"] = [tag.name for tag in Tag.objects.get_for_object(scraper)]
info["wiki_type"] = scraper.wiki_type
info["privacy_status"] = scraper.privacy_status
if scraper.wiki_type == "scraper":
info["last_run"] = scraper.scraper.last_run and scraper.scraper.last_run.isoformat() or ""
info["run_interval"] = scraper.scraper.run_interval
attachables = []
for cp in CodePermission.objects.filter(code=scraper).all():
if cp.permitted_object.privacy_status != "deleted":
attachables.append(cp.permitted_object.short_name)
info["attachables"] = attachables
# these ones have to be filtering out the incoming private scraper names
# (the outgoing attach to list doesn't because they're refered in the code as well)
info["attachable_here"] = []
for cp in CodePermission.objects.filter(permitted_object=scraper).all():
if cp.code.privacy_status not in ["deleted", "private"]:
info["attachable_here"].append(cp.code.short_name)
if scraper.wiki_type == "scraper":
info["records"] = scraper.scraper.record_count # old style datastore
if "datasummary" not in quietfields:
dataproxy = DataStore(scraper.short_name)
sqlitedata = dataproxy.request(
{"maincommand": "sqlitecommand", "command": "datasummary", "val1": 0, "val2": None}
)
if sqlitedata and type(sqlitedata) not in [str, unicode]:
info["datasummary"] = sqlitedata
if "userroles" not in quietfields:
info["userroles"] = {}
for ucrole in scraper.usercoderole_set.all():
if ucrole.role not in info["userroles"]:
info["userroles"][ucrole.role] = []
info["userroles"][ucrole.role].append(ucrole.user.username)
status = scraper.get_vcs_status(rev)
if "code" not in quietfields:
info["code"] = status["code"]
for committag in ["currcommit", "prevcommit", "nextcommit"]:
if committag not in quietfields:
if committag in status:
info[committag] = convert_history(status[committag])
if "currcommit" not in status and "prevcommit" in status and not status["ismodified"]:
if "filemodifieddate" in status:
info["modifiedcommitdifference"] = str(status["filemodifieddate"] - status["prevcommit"]["date"])
info["filemodifieddate"] = status["filemodifieddate"].isoformat()
if "history" not in quietfields:
history = []
commitentries = scraper.get_commit_log("code")
for commitentry in commitentries:
if history_start_date and commitentry["date"] < history_start_date:
continue
history.append(convert_history(commitentry))
history.reverse()
info["history"] = history
if scraper.wiki_type == "scraper" and "runevents" not in quietfields:
if history_start_date:
runevents = scraper.scraper.scraperrunevent_set.filter(run_ended__gte=history_start_date).order_by(
"-run_started"
)
else:
runevents = scraper.scraper.scraperrunevent_set.all().order_by("-run_started")[:2]
info["runevents"] = []
for runevent in runevents:
info["runevents"].append(convert_run_event(runevent))
return info
def sqlite_handler(request):
short_name = request.GET.get("name")
apikey = request.GET.get("apikey", None)
scraper, err = getscraperorresponse(short_name)
if err:
result = json.dumps({"error": err, "short_name": short_name})
if request.GET.get("callback"):
result = "%s(%s)" % (request.GET.get("callback"), result)
return HttpResponse(result)
u, s, kd = None, None, ""
if request.user.is_authenticated():
u = request.user
if scraper.privacy_status != "private":
s = scraper # XX why this only when not private? FAI
kd = short_name
else:
# When private we MUST have an apikey and it should match
if not scraper.api_actionauthorized(apikey):
result = json.dumps({"error": "Invalid API Key", "short_name": short_name})
if request.GET.get("callback"):
result = "%s(%s)" % (request.GET.get("callback"), result)
return HttpResponse(result)
APIMetric.record("sqlite", key_data=kd, user=u, code_object=s)
dataproxy = DataStore(request.GET.get("name"))
lattachlist = request.GET.get("attach", "").split(";")
attachlist = []
for aattach in lattachlist:
if aattach:
aa = aattach.split(",")
attachi = {"name": aa[0], "asname": (len(aa) == 2 and aa[1] or None)}
attachlist.append(attachi)
dataproxy.request(
{
"maincommand": "sqlitecommand",
"command": "attach",
"name": attachi["name"],
"asname": attachi["asname"],
}
)
sqlquery = request.GET.get("query", "")
format = request.GET.get("format", "json")
if format == "json":
format = "jsondict"
req = {"maincommand": "sqliteexecute", "sqlquery": sqlquery, "data": None, "attachlist": attachlist}
if format == "csv":
req["streamchunking"] = 1000
# This is inlined from the dataproxy.request() function to allow for
# receiveoneline to perform multiple readlines in this case.
# (this is the stream-chunking thing. the right interface is not yet
# apparent)
dataproxy.m_socket.sendall(json.dumps(req) + "\n")
if format not in ["jsondict", "jsonlist", "csv", "htmltable", "rss2"]:
dataproxy.close()
return HttpResponse("Error: the format '%s' is not supported" % format)
if format in ["csv", "htmltable"]:
return out_csvhtml(dataproxy, scraper.short_name, format)
if format == "rss2":
return out_rss2(dataproxy, scraper)
return out_json(dataproxy, request.GET.get("callback"), scraper.short_name, format)
def scraperinfo(scraper, history_start_date, quietfields, rev):
info = {}
info['short_name'] = scraper.short_name
info['language'] = scraper.language
info['created'] = scraper.created_at.isoformat()
info['title'] = scraper.title
info['description'] = scraper.description_safepart()
info['tags'] = [tag.name for tag in Tag.objects.get_for_object(scraper)]
info['wiki_type'] = scraper.wiki_type
info['privacy_status'] = scraper.privacy_status
if scraper.wiki_type == 'scraper':
info[
'last_run'] = scraper.scraper.last_run and scraper.scraper.last_run.isoformat(
) or ''
info['run_interval'] = scraper.scraper.run_interval
attachables = []
for cp in CodePermission.objects.filter(code=scraper).all():
if cp.permitted_object.privacy_status != "deleted":
attachables.append(cp.permitted_object.short_name)
info["attachables"] = attachables
# these ones have to be filtering out the incoming private scraper names
# (the outgoing attach to list doesn't because they're refered in the code as well)
info["attachable_here"] = []
for cp in CodePermission.objects.filter(permitted_object=scraper).all():
if cp.code.privacy_status not in ["deleted", "private"]:
info["attachable_here"].append(cp.code.short_name)
if scraper.wiki_type == 'scraper':
info['records'] = scraper.scraper.record_count # old style datastore
if 'datasummary' not in quietfields:
dataproxy = DataStore(scraper.short_name)
sqlitedata = dataproxy.request({
"maincommand": "sqlitecommand",
"command": "datasummary",
"val1": 0,
"val2": None
})
if sqlitedata and type(sqlitedata) not in [str, unicode]:
info['datasummary'] = sqlitedata
if 'userroles' not in quietfields:
info['userroles'] = {}
for ucrole in scraper.usercoderole_set.all():
if ucrole.role not in info['userroles']:
info['userroles'][ucrole.role] = []
info['userroles'][ucrole.role].append(ucrole.user.username)
status = scraper.get_vcs_status(rev)
if 'code' not in quietfields:
info['code'] = status["code"]
for committag in ["currcommit", "prevcommit", "nextcommit"]:
if committag not in quietfields:
if committag in status:
info[committag] = convert_history(status[committag])
if "currcommit" not in status and "prevcommit" in status and not status[
"ismodified"]:
if 'filemodifieddate' in status:
info["modifiedcommitdifference"] = str(
status["filemodifieddate"] - status["prevcommit"]["date"])
info['filemodifieddate'] = status['filemodifieddate'].isoformat()
if 'history' not in quietfields:
history = []
commitentries = scraper.get_commit_log("code")
for commitentry in commitentries:
if history_start_date and commitentry['date'] < history_start_date:
continue
history.append(convert_history(commitentry))
history.reverse()
info['history'] = history
if scraper.wiki_type == 'scraper' and 'runevents' not in quietfields:
if history_start_date:
runevents = scraper.scraper.scraperrunevent_set.filter(
run_ended__gte=history_start_date).order_by('-run_started')
else:
runevents = scraper.scraper.scraperrunevent_set.all().order_by(
'-run_started')[:2]
info['runevents'] = []
for runevent in runevents:
info['runevents'].append(convert_run_event(runevent))
return info
def sqlite_handler(request):
short_name = request.GET.get('name')
apikey = request.GET.get('apikey', None)
scraper, err = getscraperorresponse(short_name)
if err:
result = json.dumps({'error': err, "short_name": short_name})
if request.GET.get("callback"):
result = "%s(%s)" % (request.GET.get("callback"), result)
return HttpResponse(result)
u, s, kd = None, None, ""
if request.user.is_authenticated():
u = request.user
if scraper.privacy_status != "private":
s = scraper # XX why this only when not private? FAI
kd = short_name
else:
# When private we MUST have an apikey and it should match
if not scraper.api_actionauthorized(apikey):
result = json.dumps({
'error': "Invalid API Key",
"short_name": short_name
})
if request.GET.get("callback"):
result = "%s(%s)" % (request.GET.get("callback"), result)
return HttpResponse(result)
APIMetric.record("sqlite", key_data=kd, user=u, code_object=s)
dataproxy = DataStore(request.GET.get('name'))
lattachlist = request.GET.get('attach', '').split(";")
attachlist = []
for aattach in lattachlist:
if aattach:
aa = aattach.split(",")
attachi = {
"name": aa[0],
"asname": (len(aa) == 2 and aa[1] or None)
}
attachlist.append(attachi)
dataproxy.request({
"maincommand": "sqlitecommand",
"command": "attach",
"name": attachi["name"],
"asname": attachi["asname"]
})
sqlquery = request.GET.get('query', "")
format = request.GET.get("format", "json")
if format == "json":
format = "jsondict"
req = {
"maincommand": "sqliteexecute",
"sqlquery": sqlquery,
"data": None,
"attachlist": attachlist
}
if format == "csv":
req["streamchunking"] = 1000
# This is inlined from the dataproxy.request() function to allow for
# receiveoneline to perform multiple readlines in this case.
# (this is the stream-chunking thing. the right interface is not yet
# apparent)
dataproxy.m_socket.sendall(json.dumps(req) + '\n')
if format not in [
"jsondict", "jsonlist", "csv", "htmltable", "rss2",
"base64singleton", "htmltable_unescaped"
]:
dataproxy.close()
return HttpResponse("Error: the format '%s' is not supported" % format)
if format in ["csv", 'htmltable', 'htmltable_unescaped']:
return out_csvhtml(dataproxy, scraper.short_name, format)
if format == "rss2":
return out_rss2(dataproxy, scraper)
if format == "base64singleton":
return out_base64singleton(dataproxy,
request.GET.get('mimetype', "text/plain"))
return out_json(dataproxy, request.GET.get("callback"), scraper.short_name,
format)
