def get(self):
latest_rev = int(app.get_and_cache_rowdata('latest_rev')['rev_number'])
# We may have brand new rows, so store them.
if latest_rev not in data.rows:
for n in xrange(data.latest_rev + 1, latest_rev + 1):
curr_row = RowData()
for m in data.ordered_masters:
update_row(n, m, curr_row)
# If we didn't get any data, that revision doesn't exist, so skip on.
if not curr_row.revision:
continue
data.rows[n] = curr_row
# Update our stored latest_rev to reflect the new data.
data.latest_rev = max(data.rows.keys())
# Now update the status of the rest of the rows.
offset = 0
while offset < data.SIZE:
n = data.latest_rev - offset
if n not in data.rows:
offset += 1
continue
curr_row = data.rows[n]
for m in data.ordered_masters:
row_data = app.get_and_cache_rowdata('%s/console/%s' % (m, n))
if not row_data:
continue
update_status(m, row_data['status'], curr_row.status)
offset += 1
# Finally delete any extra rows that we don't want to keep around.
if len(data.rows) > data.SIZE:
old_revs = sorted(data.rows.keys(), reverse=True)[data.SIZE:]
for rev in old_revs:
del data.rows[rev]
self.response.out.write('Update completed (rows %s - %s).' %
(min(data.rows.keys()), max(data.rows.keys())))
def get(self):
latest_rev = int(app.get_and_cache_rowdata('latest_rev')['rev_number'])
# We may have brand new rows, so store them.
if latest_rev not in data.rows:
for n in xrange(data.latest_rev + 1, latest_rev + 1):
curr_row = RowData()
for m in data.ordered_masters:
update_row(n, m, curr_row)
# If we didn't get any data, that revision doesn't exist, so skip on.
if not curr_row.revision:
continue
data.rows[n] = curr_row
# Update our stored latest_rev to reflect the new data.
data.latest_rev = max(data.rows.keys())
# Now update the status of the rest of the rows.
offset = 0
while offset < data.SIZE:
n = data.latest_rev - offset
if n not in data.rows:
offset += 1
continue
curr_row = data.rows[n]
for m in data.ordered_masters:
row_data = app.get_and_cache_rowdata('%s/console/%s' % (m, n))
if not row_data:
continue
update_status(m, row_data['status'], curr_row.status)
offset += 1
# Finally delete any extra rows that we don't want to keep around.
if len(data.rows) > data.SIZE:
old_revs = sorted(data.rows.keys(), reverse=True)[data.SIZE:]
for rev in old_revs:
del data.rows[rev]
self.response.out.write('Update completed (rows %s - %s).' %
(min(data.rows.keys()), max(data.rows.keys())))
def test_parse_master_utf8(self):
in_console = {'content': self.read_file('in_console.html')}
app.parse_master(
localpath='chromium/console',
remoteurl='http://build.chromium.org/p/chromium/console',
page_data=in_console)
test_revision = '314921'
rowdata = app.get_and_cache_rowdata('chromium/console/' + test_revision)
summary = app.get_and_cache_pagedata('chromium/console/summary')['content']
act_row = {}
exp_row = {}
for item in ['rev', 'name', 'status', 'comment']:
# We only want to test specific values in rowdata, so we create a new
# hash that has just those values.
act_row[item] = rowdata[item]
# Uncomment if deeper inspection is needed of the returned console.
# This is also useful if changing the site layout and you need to
# 'retrain' the test expectations.
# self.write_file('exp_%s.html' % item,
# act_row[item].encode('utf-8'))
# self.write_file('exp_summary.html',
# summary.encode('utf-8'))
exp_row[item] = self.read_file('exp_%s.html' % item).decode('utf-8')
exp_summary = self.read_file('exp_summary.html').decode('utf-8')
self.assertEquals(exp_row, act_row, 'Unexpected row data found')
self.assertEquals(exp_summary, summary, 'Unexpected build summary found')
def test_parse_master(self):
in_console = {'content': self.read_file('in_console.html')}
app.parse_master(
localpath='chromium/console',
remoteurl='http://build.chromium.org/p/chromium/console',
page_data=in_console)
test_revision = '314671'
rowdata = app.get_and_cache_rowdata('chromium/console/' +
test_revision)
summary = app.get_and_cache_pagedata(
'chromium/console/summary')['content']
act_row = {}
exp_row = {}
for item in ['rev', 'name', 'status', 'comment']:
# We only want to test specific values in rowdata, so we create a new
# hash that has just those values.
act_row[item] = rowdata[item]
# Uncomment if deeper inspection is needed of the returned console.
# This is also useful if changing the site layout and you need to
# 'retrain' the test expectations.
# self.write_file('exp_%s.html' % item,
# act_row[item].encode('utf-8'))
# self.write_file('exp_summary.html',
# summary.encode('utf-8'))
exp_row[item] = self.read_file('exp_%s.html' %
item).decode('utf-8')
exp_summary = self.read_file('exp_summary.html').decode('utf-8')
self.assertEquals(exp_row, act_row, 'Unexpected row data found')
self.assertEquals(exp_summary, summary,
'Unexpected build summary found')
def update_row(revision, master, row):
"""Fetches a row from the datastore and puts it in a RowData object."""
# Fetch the relevant data from the datastore / cache.
row_data = app.get_and_cache_rowdata('%s/console/%s' % (master, revision))
if not row_data:
return
# Only grab the common data from the main master.
if master == 'chromium.main':
row.revision = int(row_data['rev_number'])
row.revlink = row_data['rev']
row.committer = row_data['name']
row.comment = row_data['comment']
row.details = row_data['details']
row.status.setdefault(master, {})
update_status(master, row_data['status'], row.status)
def update_row(revision, master, row):
"""Fetches a row from the datastore and puts it in a RowData object."""
# Fetch the relevant data from the datastore / cache.
row_data = app.get_and_cache_rowdata('%s/console/%s' % (master, revision))
if not row_data:
return
# Only grab the common data from the main master.
if master == 'chromium.main':
row.revision = int(row_data['rev_number'])
row.revlink = row_data['rev']
row.committer = row_data['name']
row.comment = row_data['comment']
row.details = row_data['details']
row.status.setdefault(master, {})
update_status(master, row_data['status'], row.status)
def bootstrap(self):
"""Fills an empty MergerData with 100 rows of data."""
# Populate the categories, masters, status, and failures data.
for m in self.ordered_masters:
for d in (self.ordered_builders, self.ordered_categories,
self.status, self.failures):
d.setdefault(m, {})
# Get the category data and construct the list of categories
# for this master.
category_data = app.get_and_cache_pagedata(
'%s/console/categories' % m)
if not category_data['content']:
category_list = [u'default']
else:
category_soup = BeautifulSoup(category_data['content'])
category_list = [
tag.string.strip()
for tag in category_soup.findAll('td', 'DevStatus')
]
self.ordered_categories[m] = category_list
# Get the builder status data.
builder_data = app.get_and_cache_pagedata('%s/console/summary' % m)
if not builder_data['content']:
continue
builder_soup = BeautifulSoup(builder_data['content'])
builders_by_category = builder_soup.tr.findAll('td',
'DevSlave',
recursive=False)
# Construct the list of builders for this category.
for i, c in enumerate(self.ordered_categories[m]):
self.ordered_builders[m].setdefault(c, {})
builder_list = [
tag['title'] for tag in builders_by_category[i].findAll(
'a', 'DevSlaveBox')
]
self.ordered_builders[m][c] = builder_list
# Fill in the status data for all of this master's builders.
update_status(m, builder_data['content'], self.status)
# Copy that status data over into the failures dictionary too.
for c in self.ordered_categories[m]:
self.failures[m].setdefault(c, {})
for b in self.ordered_builders[m][c]:
if self.status[m][c][b] not in ('success', 'running',
'notstarted'):
self.failures[m][c][b] = True
else:
self.failures[m][c][b] = False
# Populate the individual row data, saving status info in the same
# master/category/builder tree format constructed above.
latest_rev = int(app.get_and_cache_rowdata('latest_rev')['rev_number'])
if not latest_rev:
logging.error(
"MergerData.bootstrap(): Didn't get latest_rev. Aborting.")
return
n = latest_rev
num_rows_saved = num_rows_skipped = 0
while num_rows_saved < self.SIZE and num_rows_skipped < 10:
curr_row = RowData()
for m in self.ordered_masters:
update_row(n, m, curr_row)
# If we didn't get any data, that revision doesn't exist, so skip on.
if not curr_row.revision:
num_rows_skipped += 1
n -= 1
continue
self.rows[n] = curr_row
num_rows_skipped = 0
num_rows_saved += 1
n -= 1
self.latest_rev = max(self.rows.keys())
def bootstrap(self):
"""Fills an empty MergerData with 100 rows of data."""
# Populate the categories, masters, status, and failures data.
for m in self.ordered_masters:
for d in (self.ordered_builders,
self.ordered_categories,
self.status,
self.failures):
d.setdefault(m, {})
# Get the category data and construct the list of categories
# for this master.
category_data = app.get_and_cache_pagedata('%s/console/categories' % m)
if not category_data['content']:
category_list = [u'default']
else:
category_soup = BeautifulSoup(category_data['content'])
category_list = [tag.string.strip() for tag in
category_soup.findAll('td', 'DevStatus')]
self.ordered_categories[m] = category_list
# Get the builder status data.
builder_data = app.get_and_cache_pagedata('%s/console/summary' % m)
if not builder_data['content']:
continue
builder_soup = BeautifulSoup(builder_data['content'])
builders_by_category = builder_soup.tr.findAll('td', 'DevSlave',
recursive=False)
# Construct the list of builders for this category.
for i, c in enumerate(self.ordered_categories[m]):
self.ordered_builders[m].setdefault(c, {})
builder_list = [tag['title'] for tag in
builders_by_category[i].findAll('a', 'DevSlaveBox')]
self.ordered_builders[m][c] = builder_list
# Fill in the status data for all of this master's builders.
update_status(m, builder_data['content'], self.status)
# Copy that status data over into the failures dictionary too.
for c in self.ordered_categories[m]:
self.failures[m].setdefault(c, {})
for b in self.ordered_builders[m][c]:
if self.status[m][c][b] not in ('success', 'running', 'notstarted'):
self.failures[m][c][b] = True
else:
self.failures[m][c][b] = False
# Populate the individual row data, saving status info in the same
# master/category/builder tree format constructed above.
latest_rev = int(app.get_and_cache_rowdata('latest_rev')['rev_number'])
if not latest_rev:
logging.error("MergerData.bootstrap(): Didn't get latest_rev. Aborting.")
return
n = latest_rev
num_rows_saved = num_rows_skipped = 0
while num_rows_saved < self.SIZE and num_rows_skipped < 10:
curr_row = RowData()
for m in self.ordered_masters:
update_row(n, m, curr_row)
# If we didn't get any data, that revision doesn't exist, so skip on.
if not curr_row.revision:
num_rows_skipped += 1
n -= 1
continue
self.rows[n] = curr_row
num_rows_skipped = 0
num_rows_saved += 1
n -= 1
self.latest_rev = max(self.rows.keys())
