def processAgentResponse(resp):
if resp is not None and resp['result'] is not None:
d = json.loads(resp['result'])
for outgoing in d['outLinks']:
tryAddUrlToQueue(outgoing)
crawlRecord = {
"url": d['url'],
"renderTime": d['renderTime'],
"serverErrors": d['serverErrors'],
"browserErrors": d['browserErrors'],
"date": datetime.datetime.utcnow()
}
updateCrawlRecordErrors(crawlRecord)
DB.addCrawlRecord(crawlRecord)
Stats.updateStats(crawlRecord)
