def getCrawlLogData():
data = {
"errorRows": [],
"newestRows": []
}
errorRecords = DB.getCrawlRecordsWithErrors()
crawlRecords = DB.getNewestCrawlRecords(limit=100)
def pr(rows, crawlRecords):
seenUrls = {}
for cr in crawlRecords:
dashboardRow = {
"url": cr['url'],
"renderTime": cr['renderTime'],
"serverErrors": len(cr['serverErrors']),
"browserErrors": len(cr['browserErrors']),
"errorsPresent": cr['errorsPresent']
}
if dashboardRow['url'] not in seenUrls:
rows.append(dashboardRow)
seenUrls[dashboardRow['url']] = True
pr(data['newestRows'], crawlRecords)
pr(data['errorRows'], errorRecords)
return data