def delete_duplicates(entity):
yield op.counters.Increment("total-counter")
dupl_query = CrashReport.all(keys_only=True)
dupl_query.filter('crashTime =', entity.crashTime)
matched_crashes = dupl_query.fetch(50000)
mustDelete = False
counted = False
matched = False
if len(matched_crashes) > 1:
for crkey in matched_crashes:
cr = CrashReport.get(crkey)
if cr.availableInternalMemory == entity.availableInternalMemory:
matched = True
if not counted:
counted = True
yield op.counters.Increment("individual-counter")
if cr.sendTime < entity.sendTime:
mustDelete = True
break
if mustDelete:
yield op.counters.Increment("delete-counter")
yield op.db.Delete(entity)
else:
if matched:
yield op.counters.Increment("survivor-counter")
else:
yield op.counters.Increment("single-counter")
def delete_duplicates(entity):
yield op.counters.Increment("total-counter")
dupl_query = CrashReport.all(keys_only=True)
dupl_query.filter('crashTime =', entity.crashTime)
matched_crashes = dupl_query.fetch(50000)
mustDelete = False
counted = False
matched = False
if len(matched_crashes) > 1:
for crkey in matched_crashes:
cr = CrashReport.get(crkey)
if cr.availableInternalMemory == entity.availableInternalMemory:
matched = True
if not counted:
counted = True
yield op.counters.Increment("individual-counter")
if cr.sendTime < entity.sendTime:
mustDelete = True
break
if mustDelete:
yield op.counters.Increment("delete-counter")
yield op.db.Delete(entity)
else:
if matched:
yield op.counters.Increment("survivor-counter")
else:
yield op.counters.Increment("single-counter")
def recalc_counts_on_vesions(entity):
crash_query = CrashReport.all(keys_only=True)
crash_query.filter('versionName =', entity.name)
crash_query.order('-crashTime')
newcount = crash_query.count(20000)
if entity.crashCount != newcount:
entity.crashCount = newcount
cr = CrashReport.get(crash_query.fetch(1)[0])
entity.lastIncident = cr.crashTime
#logging.warning("Wrong count for bug %d, adjusting to %d" % entity.key().id(), count)
yield op.counters.Increment("touched-versions")
yield op.db.Put(entity)
if newcount == 0:
yield op.counters.Increment("empty-versions")
yield op.counters.Increment("crash-counter", newcount)
def recalc_counts_on_vesions(entity):
crash_query = CrashReport.all(keys_only=True)
crash_query.filter('versionName =', entity.name)
crash_query.order('-crashTime')
newcount = crash_query.count(20000)
if entity.crashCount != newcount:
entity.crashCount = newcount
cr = CrashReport.get(crash_query.fetch(1)[0])
entity.lastIncident = cr.crashTime
#logging.warning("Wrong count for bug %d, adjusting to %d" % entity.key().id(), count)
yield op.counters.Increment("touched-versions")
yield op.db.Put(entity)
if newcount == 0:
yield op.counters.Increment("empty-versions")
yield op.counters.Increment("crash-counter", newcount)
def get(self):
batch = RebuildBugs.batch_size
crashes_query = CrashReport.all()
crashes = []
page = int(self.request.get('page', 0))
if page == 0:
# Remove Bugs
bugs_query = Bug.all()
bugs = bugs_query.fetch(2000)
for b in bugs:
b.delete()
total_results = crashes_query.count(1000000)
last_page = max((total_results - 1) // batch, 0)
if page > last_page:
page = last_page
logging.info("Admin ops - total_results: ", str(total_results) + ", page: " + str(page) + "/" + str(last_page))
crashes = crashes_query.fetch(batch, page * batch)
valueSet = {}
valueSet["unlinked"] = 0
# Main ops loop
for cr in crashes:
cr.bugKey = None
cr.crashSignature = CrashReport.getCrashSignature(cr.report)
cr.put()
if cr.crashSignature == '\n':
logging.warning("Can't get signature for CrashReport: " + str(cr.key().id()))
valueSet["unlinked"] = valueSet["unlinked"] + 1
else:
cr.signHash = hashlib.sha256(cr.crashSignature).hexdigest()
cr.linkToBug()
bugId = str(cr.bugKey.key().id())
if bugId in valueSet:
valueSet[bugId] = valueSet[bugId] + 1
else:
valueSet[bugId] = 1
template_values = {
'values': valueSet,
'page': page,
'last_page': last_page,
'page_size': batch,
'op_link': 'rebuild_bugs',
'column_key': 'BugId',
'column_value': 'Count',
'page_results': len(crashes),
'total_results': total_results}
path = os.path.join(os.path.dirname(__file__), 'templates/admin_ops.html')
self.response.out.write(template.render(path, template_values))
def get(self):
hospital_query = HospitalizedReport.all()
total_hospital = hospital_query.count()
hospital_query = HospitalizedReport.all()
hospital_query.filter('processed =', False)
sick_hospital = hospital_query.count()
crashes_query = CrashReport.all()
bugId = self.request.get('bug_id')
page = int(self.request.get('page', 0))
crashes = []
if bugId:
bug = Bug.get_by_id(long(bugId))
crashes_query.filter("bugKey =", bug)
crashes_query.order("-crashTime")
total_results = crashes_query.count(1000000)
last_page = max((total_results - 1) // 20, 0)
if page > last_page:
page = last_page
crashes = crashes_query.fetch(20, int(page)*20)
template_values = {'crashes_list': crashes,
'total_results': total_results,
'page_size': 20,
'page': page,
'last_page': last_page,
'bug_id': bugId,
'total_hospital': total_hospital,
'sick_hospital': sick_hospital}
path = os.path.join(os.path.dirname(__file__), 'templates/crash_list.html')
self.response.out.write(template.render(path, template_values))
def get(self):
#bugs_query = Bug.all()
#bugs = []
#bugs = bugs_query.fetch(200)
#for bg in bugs:
# bg.delete()
crashes_query = CrashReport.all()
crashes_query.filter("bugKey =", None)
crashes = []
crashes = crashes_query.fetch(200)
results_list=[]
tags=set()
for cr in crashes:
#cr.linkToBug()
pass
#signa = CrashReport.getCrashSignature(cr.report)
#logging.debug("ID: " + str(cr.key().id()) + " sign: '" + signa + "'")
#cr.crashSignature = CrashReport.getCrashSignature(cr.report)
#cr.signHash = hashlib.sha256(cr.crashSignature).hexdigest()
#cr.bugKey = None
#cr.put()
#cr.linkToBug()
#if CrashReport.getCrashSignature(cr.report) != self.getCrashSignature2(cr.report):
results_list.append({'id': cr.key().id(), 'sig1': cr.crashId, 'sig2': str(cr.key().id())})
template_values = {'results_list': results_list, 'tags': tags}
path = os.path.join(os.path.dirname(__file__), 'templates/admin_ops.html')
self.response.out.write(template.render(path, template_values))
def get(self):
crId = self.request.get('crash_id')
crash = CrashReport.get_by_id(long(crId))
template_values = {'cr': crash}
path = os.path.join(os.path.dirname(__file__),
'templates/crash_view.html')
self.response.out.write(template.render(path, template_values))
def get(self):
versions = Lst.get('all_version_names_list')
selectedVersion = self.request.get('filter_version', '')
crashes = []
crashes_query = CrashReport.all()
bugId = self.request.get('bug_id')
if not bugId and not selectedVersion:
selectedVersion = versions[0]
cacheId = "CrashReport"
if selectedVersion:
crashes_query.filter("versionName =", selectedVersion)
if bugId:
bug = Bug.get_by_id(long(bugId))
crashes_query.filter("bugKey =", bug)
cacheId += bugId
if not selectedVersion:
versions.insert(0, '')
cacheId += selectedVersion
cacheId += "_counter"
crashes_query.order("-crashTime")
logging.info("version: " + selectedVersion)
page = int(self.request.get('page', 0))
crashes = crashes_query.fetch(20, int(page)*20)
template_values = {'crashes_list': crashes,
'versions_list': versions,
'filter_version': selectedVersion,
'page_size': 20,
'page': page,
'bug_id': bugId}
path = os.path.join(os.path.dirname(__file__), 'templates/crash_list.html')
self.response.out.write(template.render(path, template_values))
def rebuild_signatures(entity): signature = CrashReport.getCrashSignature(entity.report) entity.crashSignature = signature entity.signHash = hashlib.sha1(signature).hexdigest() entity.adminOpsflag = 6 entity.bugkey = None entity.linkToBug(False) yield op.db.Put(entity)
def rebuild_signatures(entity):
signature = CrashReport.getCrashSignature(entity.report)
entity.crashSignature = signature
entity.signHash = hashlib.sha1(signature).hexdigest()
entity.adminOpsflag = 6
entity.bugkey = None
entity.linkToBug(False)
yield op.db.Put(entity)
def get(self):
crId = long(self.request.get('id'))
cr = CrashReport.get_by_id(crId)
m = re.search(r'^(.*--\> END REPORT 1 \<--).*$', cr.report, re.S)
new_report = m.group(1)
template_values = {'crash_body': cr.report, 'new_crash_body': new_report}
logging.info(cr.report)
logging.info(new_report)
path = os.path.join(os.path.dirname(__file__), 'templates/admin_ops_show.html')
self.response.out.write(template.render(path, template_values))
def get(self):
versions_query = AppVersion.all()
versions_query.order("-activeFrom")
versions_objs = versions_query.fetch(2000)
versions = [v.name for v in versions_objs]
versions.insert(0, "all")
selectedVersion = self.request.get('filter_version', "all")
bugs = []
page = int(self.request.get('page', 0))
if selectedVersion != "all":
crashes = []
bugs_map = {}
crashes_query = CrashReport.all()
crashes_query.filter("versionName =", selectedVersion)
crashes = crashes_query.fetch(1000000)
for cr in crashes:
if cr.bugKey.key().id() in bugs_map:
bugs_map[cr.bugKey.key().id()].count += 1
if bugs_map[cr.bugKey.key().id()].lastIncident < cr.crashTime:
bugs_map[cr.bugKey.key().id()].lastIncident = cr.crashTime
else:
bug = cr.bugKey
bug.count = 1
bug.lastIncident = cr.crashTime
bugs_map[cr.bugKey.key().id()] = bug
unsorted_bugs = bugs_map.values()
bugs = sorted(unsorted_bugs, key=attrgetter('count'), reverse=True)
total_results = len(bugs)
last_page = max((total_results - 1) // 20, 0)
if page > last_page:
page = last_page
# trim results to a single page
bugs[(page+1)*20:] = []
bugs[0:page*20] = []
else:
bugs_query = Bug.all()
bugs_query.order("-count")
total_results = bugs_query.count(1000000)
last_page = max((total_results - 1) // 20, 0)
if page > last_page:
page = last_page
bugs = bugs_query.fetch(20, int(page)*20)
template_values = {'bugs_list': bugs,
'versions_list': versions,
'filter_version': selectedVersion,
'total_results': total_results,
'page_size': 20,
'page': page,
'last_page': last_page}
path = os.path.join(os.path.dirname(__file__), 'templates/bug_list.html')
self.response.out.write(template.render(path, template_values))
def get(self):
#versions_query = AppVersion.all()
#versions_query.order("-activeFrom")
#versions_objs = versions_query.fetch(2000)
#versions = [v.name for v in versions_objs]
#versions.insert(0, "all")
versions = Lst.get('all_version_names_list')
selectedVersion = self.request.get('filter_version', versions[0])
bugs = []
page = int(self.request.get('page', 0))
#if selectedVersion != "all":
crashes = []
bugs_map = {}
crashes_query = CrashReport.all()
crashes_query.filter("versionName =", selectedVersion)
crashes = crashes_query.fetch(1000000)
for cr in crashes:
if cr.bugKey.key().id() in bugs_map:
bugs_map[cr.bugKey.key().id()].count += 1
if bugs_map[cr.bugKey.key().id()].lastIncident < cr.crashTime:
bugs_map[cr.bugKey.key().id()].lastIncident = cr.crashTime
else:
bug = cr.bugKey
bug.count = 1
bug.lastIncident = cr.crashTime
bugs_map[cr.bugKey.key().id()] = bug
unsorted_bugs = bugs_map.values()
bugs = sorted(unsorted_bugs, key=attrgetter('count'), reverse=True)
# trim results to a single page
bugs[(page + 1) * 20:] = []
bugs[0:page * 20] = []
#else:
# bugs_query = Bug.all()
# bugs_query.order("-count")
# bugs = bugs_query.fetch(20, int(page)*20)
template_values = {
'bugs_list': bugs,
'versions_list': versions,
'filter_version': selectedVersion,
'page_size': 20,
'page': page
}
path = os.path.join(os.path.dirname(__file__),
'templates/bug_list.html')
self.response.out.write(template.render(path, template_values))
def get(self):
crashes_query = CrashReport.all()
bugId = self.request.get('bug_id')
crashes = []
if bugId:
bug = Bug.get_by_id(long(bugId))
crashes_query.filter("bugKey =", bug)
crashes_query.order("-crashTime")
total_results = crashes_query.count(1000000)
crashes = crashes_query.fetch(total_results, 0)
template_values = {'crashes_list': crashes,
'total_results': total_results,
'bug_id': bugId}
path = os.path.join(os.path.dirname(__file__), 'templates/crash_list.csv')
self.response.out.write(template.render(path, template_values))
def get(self):
versions_query = AppVersion.all()
versions_query.order("-activeFrom")
versions_objs = versions_query.fetch(2000)
versions = [v.name for v in versions_objs]
versions.insert(0, "all")
hospital_query = HospitalizedReport.all()
total_hospital = hospital_query.count()
hospital_query = HospitalizedReport.all()
hospital_query.filter('processed =', False)
sick_hospital = hospital_query.count()
crashes_query = CrashReport.all()
bugId = self.request.get('bug_id')
page = int(self.request.get('page', 0))
selectedVersion = self.request.get('filter_version', "all")
logging.info("version: " + selectedVersion)
crashes = []
if bugId:
bug = Bug.get_by_id(long(bugId))
crashes_query.filter("bugKey =", bug)
if selectedVersion != "all":
crashes_query.filter("versionName =", selectedVersion)
crashes_query.order("-crashTime")
total_results = crashes_query.count(1000000)
last_page = max((total_results - 1) // 20, 0)
if page > last_page:
page = last_page
crashes = crashes_query.fetch(20, int(page)*20)
template_values = {'crashes_list': crashes,
'versions_list': versions,
'filter_version': selectedVersion,
'total_results': total_results,
'page_size': 20,
'page': page,
'last_page': last_page,
'bug_id': bugId,
'total_hospital': total_hospital,
'sick_hospital': sick_hospital}
path = os.path.join(os.path.dirname(__file__), 'templates/crash_list.html')
self.response.out.write(template.render(path, template_values))
def get(self):
versions = Lst.get('all_version_names_list')
selectedVersion = self.request.get('filter_version', '')
crashes = []
crashes_query = CrashReport.all()
bugId = self.request.get('bug_id')
if not bugId and not selectedVersion:
selectedVersion = versions[0]
cacheId = "CrashReport"
if selectedVersion:
crashes_query.filter("versionName =", selectedVersion)
if bugId:
bug = Bug.get_by_id(long(bugId))
crashes_query.filter("bugKey =", bug)
cacheId += bugId
if not selectedVersion:
versions.insert(0, '')
cacheId += selectedVersion
cacheId += "_counter"
crashes_query.order("-crashTime")
logging.info("version: " + selectedVersion)
page = int(self.request.get('page', 0))
crashes = crashes_query.fetch(20, int(page) * 20)
template_values = {
'crashes_list': crashes,
'versions_list': versions,
'filter_version': selectedVersion,
'page_size': 20,
'page': page,
'bug_id': bugId
}
path = os.path.join(os.path.dirname(__file__),
'templates/crash_list.html')
self.response.out.write(template.render(path, template_values))
def post(self):
post_args = self.request.arguments()
page = self.request.get('page', 0)
if "remove_processed" in post_args:
# Remove successfully processed hospitalized reports
hospital_query = HospitalizedReport.all()
hospital_query.filter('processed =', True)
hr_list = hospital_query.fetch(1000000)
for hr in hr_list:
if not hr.diagnosis:
logging.info("Deleting hospitalized report: " + str(hr.key().id()))
hr.delete()
elif "fix_report" in post_args:
attemped_fix_id = self.request.get('crash_id', 0)
hr = HospitalizedReport.get_by_id(long(attemped_fix_id))
if hr and not hr.processed:
cr = CrashReport(email = hr.email, crashId = hr.crashId, report = hr.crashBody)
hr.diagnosis = cr.parseReport()
if not hr.diagnosis:
hr.processed = True
cr.put()
cr.linkToBug()
hr.put()
self.redirect(r'hospital?page=' + page)
def get(self):
crashId = self.request.get('crash_id')
crash = CrashReport.get_by_id(long(crashId))
template_values = {'cr': crash}
path = os.path.join(os.path.dirname(__file__), 'templates/crash_view.html')
self.response.out.write(template.render(path, template_values))
def get(self):
batch = RebuildVersions.batch_size
crashes_query = CrashReport.all()
crashes = []
page = int(self.request.get('page', 0))
if page == 0:
# Reset appVersion crashCount & lastIncident
versions_query = AppVersion.all()
versions = versions_query.fetch(2000)
for v in versions:
v.crashCount = 0
v.lastIncident = datetime(2000,1,1)
v.put()
total_results = crashes_query.count(1000000)
logging.info("Admin ops - total_results: ", total_results)
last_page = max((total_results - 1) // batch, 0)
if page > last_page:
page = last_page
crashes = crashes_query.fetch(batch, page * batch)
versionCounts = {}
versionLastIncidents = {}
for cr in crashes:
vname = cr.versionName.strip()
if cr.versionName <> vname:
cr.versionName = vname
cr.put()
if vname in versionCounts:
versionCounts[vname] = versionCounts[vname] + 1
else:
versionCounts[vname] = 1
if cr.versionName in versionLastIncidents:
if versionLastIncidents[vname] < cr.crashTime:
versionLastIncidents[vname] = cr.crashTime
else:
versionLastIncidents[vname] = cr.crashTime
for vname in versionCounts:
versions_query = AppVersion.all()
versions_query.filter('name =', vname)
versions = versions_query.fetch(1)
if versions:
version = versions[0]
version.crashCount = version.crashCount + versionCounts[vname]
if version.lastIncident < versionLastIncidents[vname]:
version.lastIncident = versionLastIncidents[vname]
version.put()
else:
logging.info("missing version: " + vname)
template_values = {
'tags': versionCounts,
'page': page,
'last_page': last_page,
'page_size': batch,
'op_link': 'rebuild_versions',
'column_key': 'Version',
'column_value': 'Count',
'total_results': len(crashes)}
path = os.path.join(os.path.dirname(__file__), 'templates/admin_ops.html')
self.response.out.write(template.render(path, template_values))
