def _bots(self, queue): bot_id_statuses = QueueStatus.all(projection=['bot_id'], distinct=True).filter( 'queue_name =', queue.name()).fetch(500) bot_ids = list(entry.bot_id for entry in bot_id_statuses) result = [] for bot_id in bot_ids: status = QueueStatus.all().filter('bot_id =', bot_id).order('-date').get() result.append({ "bot_id": bot_id, "status_page": self.request.host_url + "/queue-status/" + queue.name() + "/bots/" + bot_id, "latest_message": status.message, "latest_message_time": status.date, "latest_output": self.request.host_url + "/results/" + str(status.key().id()) if status.results_file else None, "active_bug_id": status.active_bug_id, "active_patch_id": status.active_patch_id, }) return result
def _bots(self, queue): # First, collect all bots that ever served this queue. bot_id_statuses = QueueStatus.all(projection=['bot_id'], distinct=True).filter( 'queue_name =', queue.name()).fetch(500) bot_ids = list(entry.bot_id for entry in bot_id_statuses) result = [] for bot_id in bot_ids: status = QueueStatus.all().filter('bot_id =', bot_id).order('-date').get() if status.queue_name != queue.name(): # The bot got re-purposed, and is serving a different queue now. continue result.append({ "bot_id": bot_id, "status_page": self.request.host_url + "/queue-status/" + queue.name() + "/bots/" + bot_id, "latest_message": status.message, "latest_message_time": status.date, "latest_message_bug_id": status.active_bug_id, "latest_message_patch_id": status.active_patch_id, "latest_output": self.request.host_url + "/results/" + str(status.key().id()) if status.results_file else None, }) return result
def _bots(self, queue): # First, collect all bots that ever served this queue. bot_id_statuses = QueueStatus.all( projection=['bot_id'], distinct=True).filter( 'queue_name =', queue.name()).fetch(500) bot_ids = list(entry.bot_id for entry in bot_id_statuses) result = [] for bot_id in bot_ids: status = QueueStatus.all().filter('bot_id =', bot_id).order('-date').get() if status.queue_name != queue.name(): # The bot got re-purposed, and is serving a different queue now. continue result.append({ "bot_id": bot_id, "status_page": self.request.host_url + "/queue-status/" + queue.name() + "/bots/" + bot_id, "latest_message": status.message, "latest_message_time": status.date, "latest_message_bug_id": status.active_bug_id, "latest_message_patch_id": status.active_patch_id, "latest_output": self.request.host_url + "/results/" + str(status.key().id()) if status.results_file else None, }) return result
def _bots(self, queue): bot_id_statuses = QueueStatus.all(projection=['bot_id'], distinct=True).filter('queue_name =', queue.name()).fetch(500) bot_ids = list(entry.bot_id for entry in bot_id_statuses) result = [] for bot_id in bot_ids: status = QueueStatus.all().filter('bot_id =', bot_id).order('-date').get() result.append({ "bot_id": bot_id, "status_page": self.request.host_url + "/queue-status/" + queue.name() + "/bots/" + bot_id, "latest_message": status.message, "latest_message_time": status.date, "latest_output": self.request.host_url + "/results/" + str(status.key().id()) if status.results_file else None, "active_bug_id": status.active_bug_id, "active_patch_id": status.active_patch_id, }) return result
def get(self, queue_name, attachment_id): statuses = QueueStatus.all().filter('queue_name =', queue_name).filter( 'active_patch_id =', int(attachment_id)).order('-date').fetch(1) if not statuses: self.error(404) return self.response.out.write(statuses[0].message)
def _rows_for_work_items(self, queue): queued_items = queue.work_items() active_items = queue.active_work_items() if not queued_items: return [] rows = [] for item_id in queued_items.item_ids: patchStatusQuery = QueueStatus.all().filter('queue_name =', queue.name()).filter('active_patch_id =', item_id).order('-date') statuses = patchStatusQuery.fetch(1) message = None message_time = None bug_id = None results_url = None if statuses: message = statuses[0].message message_time = statuses[0].date bug_id = statuses[0].active_bug_id results_url = self.request.host_url + "/results/" + str(statuses[0].key().id()) if statuses[0].results_file else None rows.append({ "attachment_id": item_id, "bug_id": bug_id, "active": active_items and active_items.time_for_item(item_id) != None, "active_since": active_items and active_items.time_for_item(item_id), "latest_message": message, "latest_message_time": message_time, "message_count": patchStatusQuery.count(), "status_page": self.request.host_url + "/patch/" + str(item_id), "latest_results": results_url, }) return rows
def get(self, status_id): status = QueueStatus.get_by_id(int(status_id)) if not status: self.error(404) return self.response.headers["Content-Type"] = "text/plain; charset=utf-8" self.response.out.write(status.results_file)
def _fetch_summary(self): summary = { "attachment_id" : self.id } first_status = QueueStatus.all().filter('active_patch_id =', self.id).get() if not first_status: # We don't have any record of this attachment. return summary summary["bug_id"] = first_status.active_bug_id for queue in queues: summary[queue] = None status = QueueStatus.all().filter('queue_name =', queue).filter('active_patch_id =', self.id).order('-date').get() if status: summary[name_with_underscores(queue)] = { "state": self.state_from_queue_status(status), "status": status, } return summary
def _fetch_summary(self): summary = { "attachment_id" : self.id } first_status = QueueStatus.all().filter('active_patch_id =', self.id).get() if not first_status: # We don't have any record of this attachment. return summary summary["bug_id"] = first_status.active_bug_id for queue in Queue.all(): summary[queue.name_with_underscores()] = None status = QueueStatus.all().filter('queue_name =', queue.name()).filter('active_patch_id =', self.id).order('-date').get() if status: # summary() is a horrible API and should be killed. summary[queue.name_with_underscores()] = { "status": status, } return summary
def _fetch_summary(self): summary = { "attachment_id" : self.id } first_status = QueueStatus.all().filter('active_patch_id =', self.id).get() if not first_status: # We don't have any record of this attachment. return summary summary["bug_id"] = first_status.active_bug_id for queue in queues: summary[queue] = None status = QueueStatus.all().filter('queue_name =', queue).filter('active_patch_id =', self.id).order('-date').get() if status: summary[self._dash_to_underscore(queue)] = { "state" : self._state_from_status(status), "status" : status, } return summary
def post(self): queue_status = QueueStatus() if users.get_current_user(): queue_status.author = users.get_current_user() bug_id = self._int_from_request("bug_id") patch_id = self._int_from_request("patch_id") queue_name = self.request.get("queue_name") queue_status.queue_name = queue_name queue_status.active_bug_id = bug_id queue_status.active_patch_id = patch_id queue_status.message = self.request.get("status") results_file = self.request.get("results_file") queue_status.results_file = db.Blob(str(results_file)) queue_status.put() Attachment.dirty(patch_id) self.response.out.write(queue_status.key().id())
def get(self, queue_name="commit-queue"): queue_status = {} for queue in queues: statuses = QueueStatus.all().filter("queue_name =", queue).order("-date").fetch(6) if not statuses: continue queue_status[queue] = statuses template_values = {"queue_status": queue_status} self.response.out.write(template.render("templates/recentstatus.html", template_values))
def get(self): statuses = QueueStatus.all().order("-date") seen_queues = set() for status in statuses: if status.active_patch_id or status.active_bug_id: continue if status.queue_name in seen_queues: status.delete() seen_queues.add(status.queue_name) self.response.out.write("Done!")
def _fetch_summary(self): summary = { "attachment_id" : self.id } first_status = QueueStatus.all().filter('active_patch_id =', self.id).get() if not first_status: # We don't have any record of this attachment. return summary summary["bug_id"] = first_status.active_bug_id for queue in Queue.all(): summary[queue.name_with_underscores()] = None status = QueueStatus.all().filter('queue_name =', queue.name()).filter('active_patch_id =', self.id).order('-date').get() if status: # summary() is a horrible API and should be killed. summary[queue.name_with_underscores()] = { "state": self.state_from_queue_status(status), "status": status, } return summary
def get(self, queue_name="commit-queue"): queue_status = {} for queue in queues: statuses = QueueStatus.all().filter("queue_name =", queue).order("-date").fetch(6) if not statuses: continue queue_status[queue] = statuses template_values = { "queue_status" : queue_status, } self.response.out.write(template.render("templates/recentstatus.html", template_values))
def _queue_status_from_request(self): queue_status = QueueStatus() # FIXME: I think this can be removed, no one uses it. if users.get_current_user(): queue_status.author = users.get_current_user() bug_id = self._int_from_request("bug_id") patch_id = self._int_from_request("patch_id") queue_name = self.request.get("queue_name") bot_id = self.request.get("bot_id") queue_status.queue_name = queue_name queue_status.bot_id = bot_id queue_status.active_bug_id = bug_id queue_status.active_patch_id = patch_id queue_status.message = self.request.get("status") results_file = self.request.get("results_file") queue_status.results_file = db.Blob(str(results_file)) return queue_status
def _rows_for_work_items(self, queue): queued_items = queue.work_items() active_items = queue.active_work_items() if not queued_items: return [] rows = [] for item_id in queued_items.item_ids: patchStatusQuery = QueueStatus.all().filter( 'queue_name =', queue.name()).filter('active_patch_id =', item_id).order('-date') statuses = patchStatusQuery.fetch(1) message = None message_time = None bug_id = None results_url = None if statuses: message = statuses[0].message message_time = statuses[0].date bug_id = statuses[0].active_bug_id results_url = self.request.host_url + "/results/" + str( statuses[0].key().id( )) if statuses[0].results_file else None row = { "attachment_id": item_id, "bug_id": bug_id, "active": active_items and active_items.time_for_item(item_id) != None, "active_since": active_items and active_items.time_for_item(item_id), "latest_message": message, "latest_message_time": message_time, "status_page": self.request.host_url + "/patch/" + str(item_id), "latest_results": results_url, } patch_log = PatchLog.lookup_if_exists(item_id, queue.name()) if patch_log and patch_log.retry_count: row["retry_count"] = patch_log.retry_count rows.append(row) return rows
def recent(cls, limit=1): statuses = QueueStatus.all().order("-date") # Notice that we use both a set and a list here to keep the -date ordering. ids = [] visited_ids = set() for status in statuses: attachment_id = status.active_patch_id if not attachment_id: continue if attachment_id in visited_ids: continue visited_ids.add(attachment_id) ids.append(attachment_id) if len(visited_ids) >= limit: break return map(cls, ids)
def _rows_for_work_items(self, queue): queued_items = queue.work_items() active_items = queue.active_work_items() if not queued_items: return [] rows = [] for item_id in queued_items.item_ids: statuses = QueueStatus.all().filter('queue_name =', queue.name()).filter('active_patch_id =', item_id).order('-date').fetch(1) status = statuses[0].message if statuses else "" rows.append({ "attachment_id": item_id, "active": active_items and active_items.time_for_item(item_id) != None, "status": status, "status_page": self.request.host_url + "/patch/" + str(item_id), }) return rows
def get(self, attachment_id_string): attachment_id = int(attachment_id_string) statuses = QueueStatus.all().filter("active_patch_id =", attachment_id).order("-date") bug_id = None queue_status = {} for status in statuses: bug_id = status.active_bug_id # Should be the same for every status. per_queue_statuses = queue_status.get(status.queue_name, []) per_queue_statuses.append(status) queue_status[status.queue_name] = per_queue_statuses template_values = { "attachment_id" : attachment_id, "bug_id" : bug_id, "queue_status" : queue_status, } self.response.out.write(template.render("templates/patch.html", template_values))
def get(self, attachment_id_string): attachment_id = int(attachment_id_string) statuses = QueueStatus.all().filter("active_patch_id =", attachment_id).order("-date") bug_id = None queue_status = {} for status in statuses: bug_id = status.active_bug_id # Should be the same for every status. per_queue_statuses = queue_status.get(status.queue_name, []) per_queue_statuses.append(status) queue_status[status.queue_name] = per_queue_statuses template_values = { "attachment_id": attachment_id, "bug_id": bug_id, "queue_status": queue_status, } self.response.out.write( template.render("templates/patch.html", template_values))
def _rows_for_work_items(self, queue): queued_items = queue.work_items() active_items = queue.active_work_items() if not queued_items: return [] rows = [] for item_id in queued_items.item_ids: statuses = QueueStatus.all().filter( 'queue_name =', queue.name()).filter('active_patch_id =', item_id).order('-date').fetch(1) status = statuses[0].message if statuses else "" rows.append({ "attachment_id": item_id, "active": active_items and active_items.time_for_item(item_id) != None, "status": status, "status_page": self.request.host_url + "/patch/" + str(item_id), }) return rows
def __init__(self, queue): self._queue = queue self._work_items = queue.work_items() self._last_status = QueueStatus.all().filter("queue_name =", queue.name()).order("-date").get()
def _build_bubble(self, queue, attachment, queue_position): bubble = { "name": queue.short_name().lower(), "attachment_id": attachment.id, } # 10 recent statuses is enough to always include a resultative one, if there were any at all. statuses = QueueStatus.all().filter( 'queue_name =', queue.name()).filter('active_patch_id =', attachment.id).order('-date').fetch(limit=10) if not statuses: bubble[ "had_resultative_status_other_than_failure_to_apply"] = False if attachment.id in queue.active_work_items().item_ids: bubble["state"] = "started" bubble[ "details_message"] = "Started processing, no output yet.\n\n" + self._iso_time( queue.active_work_items().time_for_item(attachment.id)) else: real_queue_position = self._real_queue_position( queue, queue_position) bubble["state"] = "none" bubble[ "details_message"] = "Waiting in queue, processing has not started yet.\n\nPosition in queue: " + str( real_queue_position) bubble["queue_position"] = real_queue_position else: latest_resultative_status = self._latest_resultative_status( statuses) bubble["had_resultative_status_other_than_failure_to_apply"] = any( map( lambda status: latest_resultative_status and latest_resultative_status.message != "Error: " + queue. name() + " unable to apply patch.", statuses)) if not latest_resultative_status: bubble["state"] = "started" bubble["details_message"] = ( "Recent messages:\n\n" + "\n".join([status.message for status in statuses]) + "\n\n" + self._iso_time(statuses[0].date)) elif statuses[0].message == "Pass": bubble["state"] = "pass" bubble["details_message"] = "Pass\n\n" + self._iso_time( statuses[0].date) elif statuses[0].message == "Fail": bubble["state"] = "fail" message_to_display = statuses[1].message if len( statuses) > 1 else statuses[0].message bubble[ "details_message"] = message_to_display + "\n\n" + self._iso_time( statuses[0].date) elif "did not process patch" in statuses[0].message: bubble["state"] = "none" bubble[ "details_message"] = "The patch is no longer eligible for processing." if "Bug is already closed" in statuses[0].message: bubble[ "details_message"] += " Bug was already closed when EWS attempted to process it." elif "Patch is marked r-" in statuses[0].message: bubble[ "details_message"] += " Patch was already marked r- when EWS attempted to process it." elif "Patch is obsolete" in statuses[0].message: bubble[ "details_message"] += " Patch was obsolete when EWS attempted to process it." elif "No patch committer found" in statuses[0].message: bubble[ "details_message"] += " Patch was not authorized by a commmitter." if len(statuses) > 1: if len(statuses) == 2: bubble[ "details_message"] += "\nOne message was logged while the patch was still eligible:\n\n" else: bubble[ "details_message"] += "\nSome messages were logged while the patch was still eligible:\n\n" bubble["details_message"] += "\n".join([ status.message for status in statuses[1:] ]) + "\n\n" + self._iso_time(statuses[0].date) elif statuses[0].message == "Error: " + queue.name( ) + " unable to apply patch.": bubble["state"] = "fail" message_to_display = statuses[1].message if len( statuses) > 1 else statuses[0].message bubble[ "details_message"] = message_to_display + "\n\n" + self._iso_time( statuses[0].date) bubble["failed_to_apply"] = True elif statuses[0].message.startswith("Error: "): bubble["state"] = "error" bubble["details_message"] = "\n".join([ status.message for status in statuses ]) + "\n\n" + self._iso_time(statuses[0].date) elif queue_position: bubble["state"] = "provisional-fail" bubble[ "details_message"] = self._build_message_for_provisional_failure( queue, attachment, queue_position, statuses) else: bubble["state"] = "error" bubble["details_message"] = ( "Internal error. Latest status implies that the patch should be in queue, but it is not. Recent messages:\n\n" + "\n".join([status.message for status in statuses]) + "\n\n" + self._iso_time(statuses[0].date)) if "details_message" in bubble: bubble["details_message"] = queue.display_name( ) + "\n\n" + bubble["details_message"] return bubble
def _build_bubble(self, queue, attachment, queue_position): bubble = { "name": queue.short_name().lower(), "attachment_id": attachment.id, } # 10 recent statuses is enough to always include a resultative one, if there were any at all. statuses = QueueStatus.all().filter('queue_name =', queue.name()).filter('active_patch_id =', attachment.id).order('-date').fetch(limit=10) if not statuses: bubble["had_resultative_status_other_than_failure_to_apply"] = False if attachment.id in queue.active_work_items().item_ids: bubble["state"] = "started" bubble["details_message"] = "Started processing, no output yet.\n\n" + self._iso_time(queue.active_work_items().time_for_item(attachment.id)) else: real_queue_position = self._real_queue_position(queue, queue_position) bubble["state"] = "none" bubble["details_message"] = "Waiting in queue, processing has not started yet.\n\nPosition in queue: " + str(real_queue_position) bubble["queue_position"] = real_queue_position else: latest_resultative_status = self._latest_resultative_status(statuses) bubble["had_resultative_status_other_than_failure_to_apply"] = any(map(lambda status: latest_resultative_status and latest_resultative_status.message != "Error: " + queue.name() + " unable to apply patch.", statuses)) if not latest_resultative_status: bubble["state"] = "started" bubble["details_message"] = ("Recent messages:\n\n" + "\n".join([status.message for status in statuses]) + "\n\n" + self._iso_time(statuses[0].date)) elif statuses[0].message == "Pass": bubble["state"] = "pass" bubble["details_message"] = "Pass\n\n" + self._iso_time(statuses[0].date) elif statuses[0].message == "Fail": bubble["state"] = "fail" message_to_display = statuses[1].message if len(statuses) > 1 else statuses[0].message bubble["details_message"] = message_to_display + "\n\n" + self._iso_time(statuses[0].date) elif "did not process patch" in statuses[0].message: bubble["state"] = "none" bubble["details_message"] = "The patch is no longer eligible for processing." if "Bug is already closed" in statuses[0].message: bubble["details_message"] += " Bug was already closed when EWS attempted to process it." elif "Patch is marked r-" in statuses[0].message: bubble["details_message"] += " Patch was already marked r- when EWS attempted to process it." elif "Patch is obsolete" in statuses[0].message: bubble["details_message"] += " Patch was obsolete when EWS attempted to process it." elif "No patch committer found" in statuses[0].message: bubble["details_message"] += " Patch was not authorized by a commmitter." if len(statuses) > 1: if len(statuses) == 2: bubble["details_message"] += "\nOne message was logged while the patch was still eligible:\n\n" else: bubble["details_message"] += "\nSome messages were logged while the patch was still eligible:\n\n" bubble["details_message"] += "\n".join([status.message for status in statuses[1:]]) + "\n\n" + self._iso_time(statuses[0].date) elif statuses[0].message == "Error: " + queue.name() + " unable to apply patch.": bubble["state"] = "fail" message_to_display = statuses[1].message if len(statuses) > 1 else statuses[0].message bubble["details_message"] = message_to_display + "\n\n" + self._iso_time(statuses[0].date) bubble["failed_to_apply"] = True elif statuses[0].message.startswith("Error: "): bubble["state"] = "error" bubble["details_message"] = "\n".join([status.message for status in statuses]) + "\n\n" + self._iso_time(statuses[0].date) elif queue_position: bubble["state"] = "provisional-fail" bubble["details_message"] = self._build_message_for_provisional_failure(queue, attachment, queue_position, statuses) else: bubble["state"] = "error" bubble["details_message"] = ("Internal error. Latest status implies that the patch should be in queue, but it is not. Recent messages:\n\n" + "\n".join([status.message for status in statuses]) + "\n\n" + self._iso_time(statuses[0].date)) if "details_message" in bubble: bubble["details_message"] = queue.display_name() + "\n\n" + bubble["details_message"] return bubble
def get(self, queue_name, attachment_id): statuses = QueueStatus.all().filter('queue_name =', queue_name).filter('active_patch_id =', int(attachment_id)).order('-date').fetch(1) if not statuses: self.error(404) return self.response.out.write(statuses[0].message)
def _build_bubble(self, queue, attachment, queue_position): bubble = { "name": queue.short_name().lower(), "attachment_id": attachment.id, } # 10 recent statuses is enough to always include a resultative one, if there were any at all. statuses = QueueStatus.all().filter( 'queue_name =', queue.name()).filter('active_patch_id =', attachment.id).order('-date').fetch(limit=10) if not statuses: if attachment.id in queue.active_work_items().item_ids: bubble["state"] = "started" bubble[ "details_message"] = "Started processing, no output yet.\n\n" + self._iso_time( queue.active_work_items().time_for_item(attachment.id)) else: real_queue_position = self._real_queue_position( queue, queue_position) bubble["state"] = "none" bubble[ "details_message"] = "Waiting in queue, processing has not started yet.\n\nPosition in queue: " + str( real_queue_position) bubble["queue_position"] = real_queue_position else: latest_resultative_status = self._latest_resultative_status( statuses) if not latest_resultative_status: bubble["state"] = "started" bubble["details_message"] = ( "Started processing.\n\nRecent messages:\n\n" + "\n".join([status.message for status in statuses]) + "\n\n" + self._iso_time(statuses[0].date)) elif statuses[0].message == "Pass": bubble["state"] = "pass" bubble["details_message"] = "Pass\n\n" + self._iso_time( statuses[0].date) elif statuses[0].message == "Fail": bubble["state"] = "fail" bubble["details_message"] = statuses[ 1].message + "\n\n" + self._iso_time(statuses[0].date) elif statuses[0].message == "Error: " + queue.name( ) + " did not process patch.": bubble["state"] = "none" bubble[ "details_message"] = "The patch is no longer eligible for processing." if len(statuses) > 1: if len(statuses) == 2: bubble[ "details_message"] += " One message was logged while the patch was still eligible:\n\n" else: bubble[ "details_message"] += " Some messages were logged while the patch was still eligible:\n\n" bubble["details_message"] += "\n".join([ status.message for status in statuses[1:] ]) + "\n\n" + self._iso_time(statuses[0].date) elif statuses[0].message == "Error: " + queue.name( ) + " unable to apply patch.": bubble["state"] = "fail" bubble["details_message"] = statuses[ 1].message + "\n\n" + self._iso_time(statuses[0].date) elif statuses[0].message.startswith("Error: "): bubble["state"] = "error" bubble["details_message"] = "\n".join([ status.message for status in statuses ]) + "\n\n" + self._iso_time(statuses[0].date) elif queue_position: bubble["state"] = "provisional-fail" bubble[ "details_message"] = self._build_message_for_provisional_failure( queue, attachment, queue_position, statuses) else: bubble["state"] = "error" bubble["details_message"] = ( "Internal error. Latest status implies that the patch should be in queue, but it is not. Recent messages:\n\n" + "\n".join([status.message for status in statuses]) + "\n\n" + self._iso_time(statuses[0].date)) if "details_message" in bubble: bubble["details_message"] = queue.display_name( ) + "\n\n" + bubble["details_message"] return bubble