def _do_almost_everything(self, localpath): page_data = super(OneBoxAction, self)._do_almost_everything( localpath + '/' + ONE_BOX_URL) if page_data: builders = self.request.GET.getall('builder') if builders and len(builders): one_box = BeautifulSoup(page_data['content']) all_tds = one_box.findAll('td') for td in all_tds: if td.a and td.a['title'] not in builders: td.extract() page_data['content'] = self.ContentsToHtml(one_box) return page_data
def _do_almost_everything(self, localpath): page_data = super(OneBoxAction, self)._do_almost_everything(localpath + '/' + ONE_BOX_URL) if page_data: builders = self.request.GET.getall('builder') if builders and len(builders): one_box = BeautifulSoup(page_data['content']) all_tds = one_box.findAll('td') for td in all_tds: if td.a and td.a['title'] not in builders: td.extract() page_data['content'] = self.ContentsToHtml(one_box) return page_data
def update_status(master, status_html, status_dict): """Parses build status information and saves it to a status dictionary.""" builder_soup = BeautifulSoup(status_html) builders_by_category = builder_soup.findAll('table') for i, c in enumerate(data.ordered_categories[master]): status_dict[master].setdefault(c, {}) statuses_by_builder = builders_by_category[i].findAll( 'td', 'DevStatusBox') # If we didn't get anything, it's because we're parsing the overall # summary, so look for Slave boxes instead of Status boxes. if not statuses_by_builder: statuses_by_builder = builders_by_category[i].findAll( 'td', 'DevSlaveBox') for j, b in enumerate(data.ordered_builders[master][c]): # Save the whole link as the status to keep ETA and build number info. status = unicode(statuses_by_builder[j].a) status_dict[master][c][b] = status
def update_status(master, status_html, status_dict): """Parses build status information and saves it to a status dictionary.""" builder_soup = BeautifulSoup(status_html) builders_by_category = builder_soup.findAll('table') for i, c in enumerate(data.ordered_categories[master]): status_dict[master].setdefault(c, {}) statuses_by_builder = builders_by_category[i].findAll('td', 'DevStatusBox') # If we didn't get anything, it's because we're parsing the overall # summary, so look for Slave boxes instead of Status boxes. if not statuses_by_builder: statuses_by_builder = builders_by_category[i].findAll('td', 'DevSlaveBox') for j, b in enumerate(data.ordered_builders[master][c]): # Save the whole link as the status to keep ETA and build number info. status = unicode(statuses_by_builder[j].a) status_dict[master][c][b] = status
def bootstrap(self): """Fills an empty MergerData with 100 rows of data.""" # Populate the categories, masters, status, and failures data. for m in self.ordered_masters: for d in (self.ordered_builders, self.ordered_categories, self.status, self.failures): d.setdefault(m, {}) # Get the category data and construct the list of categories # for this master. category_data = app.get_and_cache_pagedata( '%s/console/categories' % m) if not category_data['content']: category_list = [u'default'] else: category_soup = BeautifulSoup(category_data['content']) category_list = [ tag.string.strip() for tag in category_soup.findAll('td', 'DevStatus') ] self.ordered_categories[m] = category_list # Get the builder status data. builder_data = app.get_and_cache_pagedata('%s/console/summary' % m) if not builder_data['content']: continue builder_soup = BeautifulSoup(builder_data['content']) builders_by_category = builder_soup.tr.findAll('td', 'DevSlave', recursive=False) # Construct the list of builders for this category. for i, c in enumerate(self.ordered_categories[m]): self.ordered_builders[m].setdefault(c, {}) builder_list = [ tag['title'] for tag in builders_by_category[i].findAll( 'a', 'DevSlaveBox') ] self.ordered_builders[m][c] = builder_list # Fill in the status data for all of this master's builders. update_status(m, builder_data['content'], self.status) # Copy that status data over into the failures dictionary too. for c in self.ordered_categories[m]: self.failures[m].setdefault(c, {}) for b in self.ordered_builders[m][c]: if self.status[m][c][b] not in ('success', 'running', 'notstarted'): self.failures[m][c][b] = True else: self.failures[m][c][b] = False # Populate the individual row data, saving status info in the same # master/category/builder tree format constructed above. latest_rev = int(app.get_and_cache_rowdata('latest_rev')['rev_number']) if not latest_rev: logging.error( "MergerData.bootstrap(): Didn't get latest_rev. Aborting.") return n = latest_rev num_rows_saved = num_rows_skipped = 0 while num_rows_saved < self.SIZE and num_rows_skipped < 10: curr_row = RowData() for m in self.ordered_masters: update_row(n, m, curr_row) # If we didn't get any data, that revision doesn't exist, so skip on. if not curr_row.revision: num_rows_skipped += 1 n -= 1 continue self.rows[n] = curr_row num_rows_skipped = 0 num_rows_saved += 1 n -= 1 self.latest_rev = max(self.rows.keys())
def bootstrap(self): """Fills an empty MergerData with 100 rows of data.""" # Populate the categories, masters, status, and failures data. for m in self.ordered_masters: for d in (self.ordered_builders, self.ordered_categories, self.status, self.failures): d.setdefault(m, {}) # Get the category data and construct the list of categories # for this master. category_data = app.get_and_cache_pagedata('%s/console/categories' % m) if not category_data['content']: category_list = [u'default'] else: category_soup = BeautifulSoup(category_data['content']) category_list = [tag.string.strip() for tag in category_soup.findAll('td', 'DevStatus')] self.ordered_categories[m] = category_list # Get the builder status data. builder_data = app.get_and_cache_pagedata('%s/console/summary' % m) if not builder_data['content']: continue builder_soup = BeautifulSoup(builder_data['content']) builders_by_category = builder_soup.tr.findAll('td', 'DevSlave', recursive=False) # Construct the list of builders for this category. for i, c in enumerate(self.ordered_categories[m]): self.ordered_builders[m].setdefault(c, {}) builder_list = [tag['title'] for tag in builders_by_category[i].findAll('a', 'DevSlaveBox')] self.ordered_builders[m][c] = builder_list # Fill in the status data for all of this master's builders. update_status(m, builder_data['content'], self.status) # Copy that status data over into the failures dictionary too. for c in self.ordered_categories[m]: self.failures[m].setdefault(c, {}) for b in self.ordered_builders[m][c]: if self.status[m][c][b] not in ('success', 'running', 'notstarted'): self.failures[m][c][b] = True else: self.failures[m][c][b] = False # Populate the individual row data, saving status info in the same # master/category/builder tree format constructed above. latest_rev = int(app.get_and_cache_rowdata('latest_rev')['rev_number']) if not latest_rev: logging.error("MergerData.bootstrap(): Didn't get latest_rev. Aborting.") return n = latest_rev num_rows_saved = num_rows_skipped = 0 while num_rows_saved < self.SIZE and num_rows_skipped < 10: curr_row = RowData() for m in self.ordered_masters: update_row(n, m, curr_row) # If we didn't get any data, that revision doesn't exist, so skip on. if not curr_row.revision: num_rows_skipped += 1 n -= 1 continue self.rows[n] = curr_row num_rows_skipped = 0 num_rows_saved += 1 n -= 1 self.latest_rev = max(self.rows.keys())
def console_merger(localpath, remoteurl, page_data, masters_to_merge=None, num_rows_to_merge=None): masters_to_merge = masters_to_merge or DEFAULT_MASTERS_TO_MERGE num_rows_to_merge = num_rows_to_merge or 25 console_data = ConsoleData() surroundings = get_and_cache_pagedata( '%s/console/surroundings' % masters_to_merge[0]) merged_page = BeautifulSoup(surroundings['content']) merged_tag = merged_page.find('table', 'ConsoleData') if merged_tag is None: msg = 'console_merger("%s", "%s", "%s"): merged_tag cannot be None.' % ( localpath, remoteurl, page_data) logging.error(msg) raise Exception(msg) latest_rev = int(get_and_cache_rowdata('latest_rev')['rev_number']) if not latest_rev: logging.error('console_merger(\'%s\', \'%s\', \'%s\'): cannot get latest ' 'revision number.' % ( localpath, remoteurl, page_data)) return fetch_timestamp = datetime.datetime.now() for master in masters_to_merge: # Fetch the summary one-box-per-builder for the master. # If we don't get it, something is wrong, skip the master entirely. master_summary = get_and_cache_pagedata('%s/console/summary' % master) if not master_summary['content']: continue console_data.SawMaster(master) # Get the categories for this builder. If the builder doesn't have any # categories, just use the default empty-string category. category_list = [] master_categories = get_and_cache_pagedata('%s/console/categories' % master) if not master_categories['content']: category_list.append('') else: category_row = BeautifulSoup(master_categories['content']) category_list = [c.text for c in category_row.findAll('td', 'DevStatus')] # Get the corresponding summary box(es). summary_row = BeautifulSoup(master_summary['content']) summary_list = summary_row.findAll('table') for category, summary in zip(category_list, summary_list): console_data.AddCategory(category, summary) # Fetch all of the rows that we need. rows_fetched = 0 revs_skipped = 0 current_rev = latest_rev while rows_fetched < num_rows_to_merge and current_rev >= 0: # Don't get stuck looping backwards forever into data we don't have. # How hard we try scales with how many rows the person wants. if revs_skipped > max(num_rows_to_merge, 10): break row_data = get_and_cache_rowdata('%s/console/%s' % (master, current_rev)) if not row_data: current_rev -= 1 revs_skipped += 1 continue console_data.AddRow(row_data) current_rev -= 1 revs_skipped = 0 rows_fetched += 1 # Convert the merged content into console content. console_data.Finish() template_environment = Environment() template_environment.loader = FileSystemLoader('.') def notstarted(builder_status): """Convert a BeautifulSoup Tag from builder status to a notstarted line.""" builder_status = re.sub(r'DevSlaveBox', 'DevStatusBox', str(builder_status)) builder_status = re.sub(r'class=\'([^\']*)\' target=', 'class=\'DevStatusBox notstarted\' target=', builder_status) builder_status = re.sub(r'class="([^"]*)" target=', 'class="DevStatusBox notstarted" target=', builder_status) return builder_status template_environment.filters['notstarted'] = notstarted merged_template = template_environment.from_string(console_template) merged_console = merged_template.render(data=console_data) # For debugging: # logging.info('%r' % merged_console) # import code # code.interact(local=locals()) # Place merged console at |merged_tag|'s location in |merged_page|, and put # the result in |merged_content|. merged_tag.replaceWith(merged_console) merged_content = utf8_convert(merged_page) merged_content = re.sub( r'\'\<a href="\'', '\'<a \' + attributes + \' href="\'', merged_content) merged_content = re.sub( r'\'\<table\>\'', r"'<table ' + attributes + '>'", merged_content) merged_content = re.sub( r'\'\<div\>\'', r"'<div ' + attributes + '>'", merged_content) merged_content = re.sub( r'\'\<td\>\'', r"'<td ' + attributes + '>'", merged_content) merged_content = re.sub( r'\<iframe\>\</iframe\>', '<iframe \' + attributes + \' src="\' + url + \'"></iframe>', merged_content) # Update the merged console page. merged_page = get_or_create_page(localpath, None, maxage=30) logging.info('console_merger: saving merged console') page_data = get_and_cache_pagedata(localpath) page_data['title'] = 'BuildBot: Chromium' page_data['offsite_base'] = 'http://build.chromium.org/p/chromium' page_data['body_class'] = 'interface' page_data['content'] = merged_content save_page(merged_page, localpath, fetch_timestamp, page_data) return
def console_merger(localpath, remoteurl, page_data, masters_to_merge=None, num_rows_to_merge=None): masters_to_merge = masters_to_merge or DEFAULT_MASTERS_TO_MERGE num_rows_to_merge = num_rows_to_merge or 25 console_data = ConsoleData() surroundings = get_and_cache_pagedata( '%s/console/surroundings' % masters_to_merge[0]) merged_page = BeautifulSoup(surroundings['content']) merged_tag = merged_page.find('table', 'ConsoleData') if merged_tag is None: msg = 'console_merger("%s", "%s", "%s"): merged_tag cannot be None.' % ( localpath, remoteurl, page_data) logging.error(msg) raise Exception(msg) latest_rev = int(get_and_cache_rowdata('latest_rev')['rev_number']) if not latest_rev: logging.error('console_merger(\'%s\', \'%s\', \'%s\'): cannot get latest ' 'revision number.' % ( localpath, remoteurl, page_data)) return fetch_timestamp = datetime.datetime.now() for master in masters_to_merge: # Fetch the summary one-box-per-builder for the master. # If we don't get it, something is wrong, skip the master entirely. master_summary = get_and_cache_pagedata('%s/console/summary' % master) if not master_summary['content']: continue console_data.SawMaster(master) # Get the categories for this builder. If the builder doesn't have any # categories, just use the default empty-string category. category_list = [] master_categories = get_and_cache_pagedata('%s/console/categories' % master) if not master_categories['content']: category_list.append('') else: category_row = BeautifulSoup(master_categories['content']) category_list = [c.text for c in category_row.findAll('td', 'DevStatus')] # Get the corresponding summary box(es). summary_row = BeautifulSoup(master_summary['content']) summary_list = summary_row.findAll('table') for category, summary in zip(category_list, summary_list): console_data.AddCategory(category, summary) # Fetch all of the rows that we need. rows_fetched = 0 revs_skipped = 0 current_rev = latest_rev while rows_fetched < num_rows_to_merge and current_rev >= 0: # Don't get stuck looping backwards forever into data we don't have. # How hard we try scales with how many rows the person wants. if revs_skipped > max(num_rows_to_merge, 10): break row_data = get_and_cache_rowdata('%s/console/%s' % (master, current_rev)) if not row_data: current_rev -= 1 revs_skipped += 1 continue console_data.AddRow(row_data) current_rev -= 1 revs_skipped = 0 rows_fetched += 1 # Convert the merged content into console content. console_data.Finish() template_environment = Environment() template_environment.loader = FileSystemLoader('.') def notstarted(builder_status): """Convert a BeautifulSoup Tag from builder status to a notstarted line.""" builder_status = re.sub(r'DevSlaveBox', 'DevStatusBox', str(builder_status)) builder_status = re.sub(r'class=\'([^\']*)\' target=', 'class=\'DevStatusBox notstarted\' target=', builder_status) builder_status = re.sub(r'class="([^"]*)" target=', 'class="DevStatusBox notstarted" target=', builder_status) return builder_status template_environment.filters['notstarted'] = notstarted merged_template = template_environment.from_string(console_template) merged_console = merged_template.render(data=console_data) # For debugging: # logging.info('%r' % merged_console) # import code # code.interact(local=locals()) # Place merged console at |merged_tag|'s location in |merged_page|, and put # the result in |merged_content|. merged_tag.replaceWith(merged_console) merged_content = utf8_convert(merged_page) merged_content = re.sub( r'\'\<a href="\'', '\'<a \' + attributes + \' href="\'', merged_content) merged_content = re.sub( r'\'\<table\>\'', r"'<table ' + attributes + '>'", merged_content) merged_content = re.sub( r'\'\<div\>\'', r"'<div ' + attributes + '>'", merged_content) merged_content = re.sub( r'\'\<td\>\'', r"'<td ' + attributes + '>'", merged_content) merged_content = re.sub( r'\<iframe\>\</iframe\>', '<iframe \' + attributes + \' src="\' + url + \'"></iframe>', merged_content) # Update the merged console page. merged_page = get_or_create_page(localpath, None, maxage=30) logging.info('console_merger: saving merged console') page_data = get_and_cache_pagedata(localpath) page_data['title'] = 'BuildBot: Chromium' page_data['offsite_base'] = 'http://build.chromium.org/p/chromium' page_data['body_class'] = 'interface' page_data['content'] = merged_content save_page(merged_page, localpath, fetch_timestamp, page_data) return