def testPrepColNameForSS(self): tests = { 'foo': 'foo', 'Foo': 'foo', 'FOO': 'foo', 'foo bar': 'foobar', 'Foo Bar': 'foobar', 'F O O B A R': 'foobar', 'Foo/Bar': 'foobar', 'Foo Bar/Dude': 'foobardude', 'foo/bar': 'foobar', } for col in tests: expected = tests[col] self.assertEquals(expected, gdata_lib.PrepColNameForSS(col)) self.assertEquals(expected, gdata_lib.PrepColNameForSS(expected))
def __init__(self, row, cols=None): self.custom = {} if not cols: # If columns not specified, then column order doesn't matter. cols = row.keys() for col in cols: ss_col = gdata_lib.PrepColNameForSS(col) val = row[col] ss_val = gdata_lib.PrepValForSS(val) self.custom[ss_col] = SSEntry(ss_val)
def _UploadChangedRows(self): """Upload all rows in table that need to be changed in spreadsheet.""" rows_unchanged, rows_updated, rows_inserted = (0, 0, 0) # Go over all rows in csv table. Identify existing row by the 'Package' # column. Either update existing row or create new one. for csv_row in self._csv_table: # Seed new row values from csv_row values, with column translation. new_row = dict((gdata_lib.PrepColNameForSS(key), csv_row[key]) for key in csv_row) # Retrieve row values already in spreadsheet, along with row index. csv_package = csv_row[self.ID_COL] ss_row = self._GetSSRowForPackage(csv_package) if ss_row: changed = [] # Gather changes for log message. # Check each key/value in new_row to see if it is different from what # is already in spreadsheet (ss_row). Keep only differences to get # the row delta. row_delta = {} for col in new_row: if col in ss_row: ss_val = ss_row[col] new_val = new_row[col] if (ss_val or new_val) and ss_val != new_val: changed.append('%s="%s"->"%s"' % (col, ss_val, new_val)) row_delta[col] = new_val if row_delta: self._scomm.UpdateRowCellByCell(ss_row.ss_row_num, gdata_lib.PrepRowForSS(row_delta)) rows_updated += 1 oper.Info('C %-30s: %s' % (csv_package, ', '.join(changed))) else: rows_unchanged += 1 oper.Info('S %-30s:' % csv_package) else: self._scomm.InsertRow(gdata_lib.PrepRowForSS(new_row)) rows_inserted += 1 row_descr_list = [] for col in sorted(new_row.keys()): if col != self.ID_COL: row_descr_list.append('%s="%s"' % (col, new_row[col])) oper.Info('A %-30s: %s' % (csv_package, ', '.join(row_descr_list))) return (rows_unchanged, rows_updated, rows_inserted)
def _CreateRowCache(self, table): """Recreate the expected row cache (by pkg) from |table|.""" if not table: return None row_cache = {} for rowIx, row in enumerate(table): pkg = row[self.COL_PKG] # Translate column names now. ss_row_dict = {} for col in row: ss_row_dict[gdata_lib.PrepColNameForSS(col)] = row[col] ss_row = gdata_lib.SpreadsheetRow('OrigRow%d' % (rowIx + 2), rowIx + 2, ss_row_dict) entry = row_cache.get(pkg) if not entry: row_cache[pkg] = ss_row elif type(entry) == list: row_cache[pkg] = entry + [ss_row] else: row_cache[pkg] = [entry, ss_row] return row_cache
class SyncerTest(cros_test_lib.MoxOutputTestCase): col_amd64 = utable.UpgradeTable.GetColumnName( utable.UpgradeTable.COL_STATE, 'amd64') col_amd64 = gdata_lib.PrepColNameForSS(col_amd64) col_arm = utable.UpgradeTable.GetColumnName(utable.UpgradeTable.COL_STATE, 'arm') col_arm = gdata_lib.PrepColNameForSS(col_arm) col_x86 = utable.UpgradeTable.GetColumnName(utable.UpgradeTable.COL_STATE, 'x86') col_x86 = gdata_lib.PrepColNameForSS(col_x86) def testInit(self): mocked_syncer = self.mox.CreateMock(sps.Syncer) tcomm, scomm = 'TComm', 'SComm' # Replay script self.mox.ReplayAll() # Verify sps.Syncer.__init__(mocked_syncer, tcomm, scomm) self.mox.VerifyAll() self.assertEquals(scomm, mocked_syncer.scomm) self.assertEquals(tcomm, mocked_syncer.tcomm) self.assertEquals(None, mocked_syncer.teams) self.assertEquals(None, mocked_syncer.owners) self.assertEquals(False, mocked_syncer.pretend) self.assertEquals(False, mocked_syncer.verbose) def testReduceTeamName(self): syncer = sps.Syncer('tcomm_obj', 'scomm_obj') tests = { 'build/bdavirro': 'build', 'build/rtc': 'build', 'build': 'build', 'UI/zel': 'ui', 'UI': 'ui', 'Build': 'build', None: None, } # Verify for key in tests: result = syncer._ReduceTeamName(key) self.assertEquals(tests[key], result) def testReduceOwnerName(self): syncer = sps.Syncer('tcomm_obj', 'scomm_obj') tests = { 'joe': 'joe', 'Joe': 'joe', '*****@*****.**': 'joe', '*****@*****.**': 'joe', '*****@*****.**': 'joe.bob', None: None, } # Verify for key in tests: result = syncer._ReduceOwnerName(key) self.assertEquals(tests[key], result) def testSetTeamFilterOK(self): syncer = sps.Syncer('tcomm_obj', 'scomm_obj') tests = { 'build:system:ui': set(['build', 'system', 'ui']), 'Build:system:UI': set(['build', 'system', 'ui']), 'kernel': set(['kernel']), 'KERNEL': set(['kernel']), None: None, '': None, } # Verify for test in tests: syncer.SetTeamFilter(test) self.assertEquals(tests[test], syncer.teams) def testSetTeamFilterError(self): syncer = sps.Syncer('tcomm_obj', 'scomm_obj') # "systems" is not valid (should be "system") teamarg = 'build:systems' # Verify with self.OutputCapturer(): self.assertRaises(SystemExit, sps.Syncer.SetTeamFilter, syncer, teamarg) def testSetOwnerFilter(self): syncer = sps.Syncer('tcomm_obj', 'scomm_obj') tests = { 'joe:bill:bob': set(['joe', 'bill', 'bob']), 'Joe:Bill:BOB': set(['joe', 'bill', 'bob']), '[email protected]:bill:bob': set(['joe', 'bill', 'bob']), 'joe': set(['joe']), '*****@*****.**': set(['joe']), '': None, None: None, } # Verify for test in tests: syncer.SetOwnerFilter(test) self.assertEquals(tests[test], syncer.owners) def testRowPassesFilters(self): mocked_syncer = self.mox.CreateMock(sps.Syncer) row1 = {sps.COL_TEAM: 'build', sps.COL_OWNER: 'joe'} row2 = {sps.COL_TEAM: 'build', sps.COL_OWNER: 'bob'} row3 = {sps.COL_TEAM: 'build', sps.COL_OWNER: None} row4 = {sps.COL_TEAM: None, sps.COL_OWNER: None} teams1 = set(['build']) teams2 = set(['kernel']) teams3 = set(['build', 'ui']) owners1 = set(['joe']) owners2 = set(['bob']) owners3 = set(['joe', 'dan']) tests = [ { 'row': row1, 'teams': None, 'owners': None, 'result': True }, { 'row': row1, 'teams': teams1, 'owners': None, 'result': True }, { 'row': row1, 'teams': teams2, 'owners': None, 'result': False }, { 'row': row1, 'teams': teams3, 'owners': None, 'result': True }, { 'row': row1, 'teams': teams1, 'owners': owners1, 'result': True }, { 'row': row1, 'teams': None, 'owners': owners2, 'result': False }, { 'row': row1, 'teams': None, 'owners': owners3, 'result': True }, { 'row': row2, 'teams': None, 'owners': None, 'result': True }, { 'row': row2, 'teams': teams1, 'owners': None, 'result': True }, { 'row': row2, 'teams': teams2, 'owners': None, 'result': False }, { 'row': row2, 'teams': teams3, 'owners': None, 'result': True }, { 'row': row2, 'teams': teams1, 'owners': owners1, 'result': False }, { 'row': row2, 'teams': None, 'owners': owners2, 'result': True }, { 'row': row2, 'teams': None, 'owners': owners3, 'result': False }, { 'row': row3, 'teams': None, 'owners': None, 'result': True }, { 'row': row3, 'teams': teams1, 'owners': None, 'result': True }, { 'row': row3, 'teams': teams2, 'owners': None, 'result': False }, { 'row': row3, 'teams': teams3, 'owners': None, 'result': True }, { 'row': row3, 'teams': teams1, 'owners': owners1, 'result': False }, { 'row': row3, 'teams': None, 'owners': owners2, 'result': False }, { 'row': row3, 'teams': None, 'owners': owners3, 'result': False }, { 'row': row4, 'teams': None, 'owners': None, 'result': True }, { 'row': row4, 'teams': teams1, 'owners': None, 'result': False }, { 'row': row4, 'teams': teams1, 'owners': owners1, 'result': False }, { 'row': row4, 'teams': None, 'owners': owners2, 'result': False }, ] # Replay script for test in tests: done = False if test['teams']: row_team = test['row'][sps.COL_TEAM] mocked_syncer._ReduceTeamName(row_team).AndReturn(row_team) done = row_team not in test['teams'] if not done and test['owners']: row_owner = test['row'][sps.COL_OWNER] mocked_syncer._ReduceOwnerName(row_owner).AndReturn(row_owner) self.mox.ReplayAll() # Verify for test in tests: mocked_syncer.teams = test['teams'] mocked_syncer.owners = test['owners'] result = sps.Syncer._RowPassesFilters(mocked_syncer, test['row']) msg = ('Expected following row to %s filter, but it did not:\n%r' % ('pass' if test['result'] else 'fail', test['row'])) msg += '\n Using teams filter : %r' % mocked_syncer.teams msg += '\n Using owners filter: %r' % mocked_syncer.owners self.assertEquals(test['result'], result, msg) self.mox.VerifyAll() def testSyncMissingTrackerColumn(self): mocked_syncer = self.mox.CreateMock(sps.Syncer) mocked_scomm = self.mox.CreateMock(gdata_lib.SpreadsheetComm) mocked_tcomm = self.mox.CreateMock(gdata_lib.TrackerComm) mocked_syncer.scomm = mocked_scomm mocked_syncer.tcomm = mocked_tcomm # Replay script mocked_scomm.GetColumnIndex(sps.COL_TRACKER).AndReturn(None) self.mox.ReplayAll() # Verify self.assertRaises(sps.SyncError, sps.Syncer.Sync, mocked_syncer) self.mox.VerifyAll() def testSyncNewIssues(self): mocked_syncer = self.mox.CreateMock(sps.Syncer) mocked_scomm = self.mox.CreateMock(gdata_lib.SpreadsheetComm) mocked_tcomm = self.mox.CreateMock(gdata_lib.TrackerComm) mocked_syncer.scomm = mocked_scomm mocked_syncer.tcomm = mocked_tcomm rows = [ { sps.COL_PACKAGE: 'd/f', sps.COL_TEAM: 'build', sps.COL_OWNER: None }, { sps.COL_PACKAGE: 'd/f', sps.COL_TEAM: 'build', sps.COL_OWNER: 'joe' }, ] # Replay script mocked_scomm.GetColumnIndex(sps.COL_TRACKER).AndReturn( 1) # Any index ok. mocked_scomm.GetRows().AndReturn(rows) for ix in xrange(len(rows)): mocked_syncer._RowPassesFilters(rows[ix]).AndReturn(True) mocked_syncer._GenIssueForRow(rows[ix]).AndReturn('NewIssue%d' % ix) mocked_syncer._GetRowTrackerId(rows[ix]).AndReturn(None) mocked_syncer._CreateRowIssue(ix + 2, rows[ix], 'NewIssue%d' % ix) self.mox.ReplayAll() # Verify sps.Syncer.Sync(mocked_syncer) self.mox.VerifyAll() def testSyncClearIssues(self): mocked_syncer = self.mox.CreateMock(sps.Syncer) mocked_scomm = self.mox.CreateMock(gdata_lib.SpreadsheetComm) mocked_tcomm = self.mox.CreateMock(gdata_lib.TrackerComm) mocked_syncer.scomm = mocked_scomm mocked_syncer.tcomm = mocked_tcomm rows = [ { sps.COL_PACKAGE: 'd/f', sps.COL_TEAM: 'build', sps.COL_OWNER: None }, { sps.COL_PACKAGE: 'd/f', sps.COL_TEAM: 'build', sps.COL_OWNER: 'joe' }, ] # Replay script mocked_scomm.GetColumnIndex(sps.COL_TRACKER).AndReturn( 1) # Any index ok. mocked_scomm.GetRows().AndReturn(rows) for ix in xrange(len(rows)): mocked_syncer._RowPassesFilters(rows[ix]).AndReturn(True) mocked_syncer._GenIssueForRow(rows[ix]).AndReturn(None) mocked_syncer._GetRowTrackerId(rows[ix]).AndReturn(123 + ix) mocked_syncer._ClearRowIssue(ix + 2, rows[ix]) self.mox.ReplayAll() # Verify sps.Syncer.Sync(mocked_syncer) self.mox.VerifyAll() def testSyncFilteredOut(self): mocked_syncer = self.mox.CreateMock(sps.Syncer) mocked_scomm = self.mox.CreateMock(gdata_lib.SpreadsheetComm) mocked_tcomm = self.mox.CreateMock(gdata_lib.TrackerComm) mocked_syncer.scomm = mocked_scomm mocked_syncer.tcomm = mocked_tcomm rows = [ { sps.COL_PACKAGE: 'd/f', sps.COL_TEAM: 'build', sps.COL_OWNER: None }, { sps.COL_PACKAGE: 'd/f', sps.COL_TEAM: 'build', sps.COL_OWNER: 'joe' }, ] # Replay script mocked_scomm.GetColumnIndex(sps.COL_TRACKER).AndReturn( 1) # Any index ok. mocked_scomm.GetRows().AndReturn(rows) for ix in xrange(len(rows)): mocked_syncer._RowPassesFilters(rows[ix]).AndReturn(False) self.mox.ReplayAll() # Verify sps.Syncer.Sync(mocked_syncer) self.mox.VerifyAll() def testGetRowValue(self): mocked_syncer = self.mox.CreateMock(sps.Syncer) row = { self.col_amd64: 'ABC', self.col_arm: 'XYZ', self.col_x86: 'FooBar', sps.COL_TEAM: 'build', } # Replay script self.mox.ReplayAll() # Verify result = sps.Syncer._GetRowValue(mocked_syncer, row, 'stateonamd64', 'amd64') self.assertEquals('ABC', result) result = sps.Syncer._GetRowValue(mocked_syncer, row, 'stateonarm', 'arm') self.assertEquals('XYZ', result) result = sps.Syncer._GetRowValue(mocked_syncer, row, 'stateonamd64', 'amd64') self.assertEquals('ABC', result) result = sps.Syncer._GetRowValue(mocked_syncer, row, sps.COL_TEAM) self.assertEquals('build', result) self.mox.VerifyAll() def _TestGenIssueForRowNeedsUpgrade(self, row): mocked_syncer = self.mox.CreateMock(sps.Syncer) mocked_syncer.default_owner = None mocked_syncer.scomm = cros_test_lib.EasyAttr(ss_key='SomeSSKey') # Replay script for arch in sps.ARCHES: state = sps.Syncer._GetRowValue(mocked_syncer, row, utable.UpgradeTable.COL_STATE, arch) mocked_syncer._GetRowValue(row, utable.UpgradeTable.COL_STATE, arch).AndReturn(state) red_team = sps.Syncer._ReduceTeamName(mocked_syncer, row[sps.COL_TEAM]) mocked_syncer._ReduceTeamName(row[sps.COL_TEAM]).AndReturn(red_team) red_owner = sps.Syncer._ReduceOwnerName(mocked_syncer, row[sps.COL_OWNER]) mocked_syncer._ReduceOwnerName(row[sps.COL_OWNER]).AndReturn(red_owner) for arch in sps.ARCHES: mocked_syncer._GetRowValue(row, utable.UpgradeTable.COL_CURRENT_VER, arch).AndReturn('1') mocked_syncer._GetRowValue( row, utable.UpgradeTable.COL_STABLE_UPSTREAM_VER, arch).AndReturn('2') mocked_syncer._GetRowValue( row, utable.UpgradeTable.COL_LATEST_UPSTREAM_VER, arch).AndReturn('3') self.mox.ReplayAll() # Verify result = sps.Syncer._GenIssueForRow(mocked_syncer, row) self.mox.VerifyAll() return result def testGenIssueForRowNeedsUpgrade1(self): row = { self.col_amd64: utable.UpgradeTable.STATE_NEEDS_UPGRADE, self.col_arm: 'Not important', self.col_x86: 'Not important', sps.COL_TEAM: 'build', sps.COL_OWNER: None, sps.COL_PACKAGE: 'dev/foo', } result = self._TestGenIssueForRowNeedsUpgrade(row) self.assertEquals(None, result.owner) self.assertEquals(0, result.id) self.assertEquals('Untriaged', result.status) def testGenIssueForRowNeedsUpgrade2(self): row = { self.col_amd64: utable.UpgradeTable.STATE_NEEDS_UPGRADE, self.col_arm: utable.UpgradeTable.STATE_NEEDS_UPGRADE_AND_PATCHED, self.col_x86: 'Not important', sps.COL_TEAM: 'build', sps.COL_OWNER: 'joe', sps.COL_PACKAGE: 'dev/foo', } result = self._TestGenIssueForRowNeedsUpgrade(row) self.assertEquals('*****@*****.**', result.owner) self.assertEquals(0, result.id) self.assertEquals('Available', result.status) def testGenIssueForRowNeedsUpgrade3(self): mocked_syncer = self.mox.CreateMock(sps.Syncer) row = { self.col_amd64: utable.UpgradeTable.STATE_NEEDS_UPGRADE, self.col_arm: utable.UpgradeTable.STATE_NEEDS_UPGRADE_AND_PATCHED, self.col_x86: 'Not important', sps.COL_TEAM: None, sps.COL_OWNER: 'joe', sps.COL_PACKAGE: 'dev/foo', } # Replay script for arch in sps.ARCHES: state = sps.Syncer._GetRowValue(mocked_syncer, row, utable.UpgradeTable.COL_STATE, arch) mocked_syncer._GetRowValue(row, utable.UpgradeTable.COL_STATE, arch).AndReturn(state) reduced = sps.Syncer._ReduceTeamName(mocked_syncer, row[sps.COL_TEAM]) mocked_syncer._ReduceTeamName(row[sps.COL_TEAM]).AndReturn(reduced) self.mox.ReplayAll() # Verify with self.OutputCapturer(): self.assertRaises(RuntimeError, sps.Syncer._GenIssueForRow, mocked_syncer, row) self.mox.VerifyAll() def testGenIssueForRowNoUpgrade(self): mocked_syncer = self.mox.CreateMock(sps.Syncer) row = { self.col_amd64: 'Not important', self.col_arm: 'Not important', self.col_x86: 'Not important', sps.COL_TEAM: None, sps.COL_OWNER: 'joe', sps.COL_PACKAGE: 'dev/foo', } # Replay script for arch in sps.ARCHES: state = sps.Syncer._GetRowValue(mocked_syncer, row, utable.UpgradeTable.COL_STATE, arch) mocked_syncer._GetRowValue(row, utable.UpgradeTable.COL_STATE, arch).AndReturn(state) self.mox.ReplayAll() # Verify result = sps.Syncer._GenIssueForRow(mocked_syncer, row) self.mox.VerifyAll() self.assertEquals(None, result) def testGetRowTrackerId(self): mocked_syncer = self.mox.CreateMock(sps.Syncer) row = {sps.COL_TRACKER: '321'} # Replay script self.mox.ReplayAll() # Verify with self.OutputCapturer(): result = sps.Syncer._GetRowTrackerId(mocked_syncer, row) self.mox.VerifyAll() self.assertEquals(321, result) def testCreateRowIssuePretend(self): mocked_syncer = self.mox.CreateMock(sps.Syncer) mocked_syncer.pretend = True row = {sps.COL_PACKAGE: 'dev/foo'} # Replay script self.mox.ReplayAll() # Verify with self.OutputCapturer(): sps.Syncer._CreateRowIssue(mocked_syncer, 5, row, 'some_issue') self.mox.VerifyAll() def testCreateRowIssue(self): mocked_syncer = self.mox.CreateMock(sps.Syncer) mocked_scomm = self.mox.CreateMock(gdata_lib.SpreadsheetComm) mocked_tcomm = self.mox.CreateMock(gdata_lib.TrackerComm) mocked_syncer.scomm = mocked_scomm mocked_syncer.tcomm = mocked_tcomm mocked_syncer.tracker_col_ix = 8 mocked_syncer.pretend = False row_ix = 5 row = {sps.COL_PACKAGE: 'dev/foo'} issue = 'SomeIssue' issue_id = 234 ss_issue_val = 'Hyperlink%d' % issue_id # Replay script mocked_scomm.ClearCellValue(row_ix, mocked_syncer.tracker_col_ix) mocked_tcomm.CreateTrackerIssue(issue).AndReturn(issue_id) mocked_syncer._GenSSLinkToIssue(issue_id).AndReturn(ss_issue_val) mocked_scomm.ReplaceCellValue(row_ix, mocked_syncer.tracker_col_ix, ss_issue_val) self.mox.ReplayAll() # Verify with self.OutputCapturer(): sps.Syncer._CreateRowIssue(mocked_syncer, row_ix, row, issue) self.mox.VerifyAll() def testGenSSLinkToIssue(self): mocked_syncer = self.mox.CreateMock(sps.Syncer) issue_id = 123 # Replay script self.mox.ReplayAll() # Verify result = sps.Syncer._GenSSLinkToIssue(mocked_syncer, issue_id) self.mox.VerifyAll() self.assertEquals('=hyperlink("crosbug.com/123";"123")', result) def testClearRowIssue(self): mocked_syncer = self.mox.CreateMock(sps.Syncer) mocked_scomm = self.mox.CreateMock(gdata_lib.SpreadsheetComm) mocked_syncer.scomm = mocked_scomm mocked_syncer.tracker_col_ix = 8 mocked_syncer.pretend = False row_ix = 44 row = {sps.COL_PACKAGE: 'dev/foo'} # Replay script mocked_scomm.ClearCellValue(row_ix, mocked_syncer.tracker_col_ix) self.mox.ReplayAll() # Verify with self.OutputCapturer(): sps.Syncer._ClearRowIssue(mocked_syncer, row_ix, row) self.mox.VerifyAll() def testClearRowIssuePretend(self): mocked_syncer = self.mox.CreateMock(sps.Syncer) mocked_scomm = self.mox.CreateMock(gdata_lib.SpreadsheetComm) mocked_syncer.scomm = mocked_scomm mocked_syncer.tracker_col_ix = 8 mocked_syncer.pretend = True row_ix = 44 row = {sps.COL_PACKAGE: 'dev/foo'} # Replay script self.mox.ReplayAll() # Verify with self.OutputCapturer(): sps.Syncer._ClearRowIssue(mocked_syncer, row_ix, row) self.mox.VerifyAll()
class UploaderTest(cros_test_lib.MoxOutputTestCase): """Test the functionality of upload_package_status.Uploader class.""" COL_PKG = 'Package' COL_SLOT = 'Slot' COL_OVERLAY = 'Overlay' COL_STATUS = 'Status' COL_VER = 'Current Version' COL_STABLE_UP = 'Stable Upstream Version' COL_LATEST_UP = 'Latest Upstream Version' COL_TARGET = 'Chrome OS Root Target' SS_COL_PKG = gdata_lib.PrepColNameForSS(COL_PKG) SS_COL_SLOT = gdata_lib.PrepColNameForSS(COL_SLOT) SS_COL_OVERLAY = gdata_lib.PrepColNameForSS(COL_OVERLAY) SS_COL_STATUS = gdata_lib.PrepColNameForSS(COL_STATUS) SS_COL_VER = gdata_lib.PrepColNameForSS(COL_VER) SS_COL_STABLE_UP = gdata_lib.PrepColNameForSS(COL_STABLE_UP) SS_COL_LATEST_UP = gdata_lib.PrepColNameForSS(COL_LATEST_UP) SS_COL_TARGET = gdata_lib.PrepColNameForSS(COL_TARGET) COLS = [ COL_PKG, COL_SLOT, COL_OVERLAY, COL_STATUS, COL_VER, COL_STABLE_UP, COL_LATEST_UP, COL_TARGET, ] ROW0 = { COL_PKG: 'lib/foo', COL_SLOT: '0', COL_OVERLAY: 'portage', COL_STATUS: 'needs upgrade', COL_VER: '3.0.2', COL_STABLE_UP: '3.0.9', COL_LATEST_UP: '3.0.11', COL_TARGET: 'chromeos', } ROW1 = { COL_PKG: 'sys-dev/bar', COL_SLOT: '0', COL_OVERLAY: 'chromiumos-overlay', COL_STATUS: 'needs upgrade', COL_VER: '1.2.3-r1', COL_STABLE_UP: '1.2.3-r2', COL_LATEST_UP: '1.2.4', COL_TARGET: 'chromeos-dev', } ROW2 = { COL_PKG: 'sys-dev/raster', COL_SLOT: '1', COL_OVERLAY: 'chromiumos-overlay', COL_STATUS: 'current', COL_VER: '1.2.3', COL_STABLE_UP: '1.2.3', COL_LATEST_UP: '1.2.4', COL_TARGET: 'chromeos-test', } SS_ROW0 = dict([(gdata_lib.PrepColNameForSS(c), v) for c, v in ROW0.items()]) SS_ROW1 = dict([(gdata_lib.PrepColNameForSS(c), v) for c, v in ROW1.items()]) SS_ROW2 = dict([(gdata_lib.PrepColNameForSS(c), v) for c, v in ROW2.items()]) EMAIL = '*****@*****.**' PASSWORD = '******' def _MockUploader(self, table=None): """Set up a mocked Uploader object.""" uploader = self.mox.CreateMock(ups.Uploader) if not table: # Use default table table = self._CreateDefaultTable() for slot in ups.Uploader.__slots__: uploader.__setattr__(slot, None) uploader._csv_table = table uploader._scomm = self.mox.CreateMock(gdata_lib.SpreadsheetComm) uploader._creds = cros_test_lib.EasyAttr(user=self.EMAIL, password=self.PASSWORD) uploader._ss_row_cache = self._CreateRowCache(table) return uploader def _CreateRowCache(self, table): """Recreate the expected row cache (by pkg) from |table|.""" if not table: return None row_cache = {} for rowIx, row in enumerate(table): pkg = row[self.COL_PKG] # Translate column names now. ss_row_dict = {} for col in row: ss_row_dict[gdata_lib.PrepColNameForSS(col)] = row[col] ss_row = gdata_lib.SpreadsheetRow('OrigRow%d' % (rowIx + 2), rowIx + 2, ss_row_dict) entry = row_cache.get(pkg) if not entry: row_cache[pkg] = ss_row elif type(entry) == list: row_cache[pkg] = entry + [ss_row] else: row_cache[pkg] = [entry, ss_row] return row_cache def _CreateDefaultTable(self): return self._CreateTableWithRows(self.COLS, [self.ROW0, self.ROW1]) def _CreateTableWithRows(self, cols, rows): mytable = tablelib.Table(list(cols)) if rows: for row in rows: mytable.AppendRow(dict(row)) return mytable def testLoadTable(self): # Note that this test is not actually for method of Uploader class. self.mox.StubOutWithMock(tablelib.Table, 'LoadFromCSV') csv = 'any.csv' # Replay script tablelib.Table.LoadFromCSV(csv).AndReturn('loaded_table') self.mox.ReplayAll() # Verification steps. with self.OutputCapturer(): loaded_table = ups.LoadTable(csv) self.assertEquals(loaded_table, 'loaded_table') def testGetSSRowForPackage(self): mocked_uploader = self._MockUploader() # No replay script. self.mox.ReplayAll() # Verification steps. result = ups.Uploader._GetSSRowForPackage(mocked_uploader, self.ROW0[self.COL_PKG]) self.assertEquals(result, self.SS_ROW0) self.assertEquals(2, result.ss_row_num) result = ups.Uploader._GetSSRowForPackage(mocked_uploader, self.ROW1[self.COL_PKG]) self.assertEquals(result, self.SS_ROW1) self.assertEquals(3, result.ss_row_num) result = ups.Uploader._GetSSRowForPackage(mocked_uploader, self.ROW2[self.COL_PKG]) self.assertEquals(result, None) self.mox.VerifyAll() def testUploadFirstWorksheet(self): mocked_uploader = self._MockUploader() # Clear ._scomm attribute to simulate uploading first worksheet. mocked_scomm = mocked_uploader._scomm mocked_uploader._scomm = None self.mox.StubOutWithMock(gdata_lib.SpreadsheetComm, '__new__') ss_key = 'Some ss_key' ws_name = 'Some ws_name' # Replay script gdata_lib.SpreadsheetComm.__new__( gdata_lib.SpreadsheetComm).AndReturn(mocked_scomm) mocked_scomm.Connect(mocked_uploader._creds, ss_key, ws_name, source='Upload Package Status') mocked_scomm.GetRowCacheByCol(self.SS_COL_PKG).AndReturn('RowCache') mocked_uploader._UploadChangedRows().AndReturn(tuple([0, 1, 2])) mocked_uploader._DeleteOldRows().AndReturn(tuple([3, 4])) self.mox.ReplayAll() # Verify with self.OutputCapturer(): ups.Uploader.Upload(mocked_uploader, ss_key, ws_name) self.mox.VerifyAll() def testUploadSecondWorksheet(self): mocked_uploader = self._MockUploader() ss_key = 'Some ss_key' ws_name = 'Some ws_name' # Replay script mocked_uploader._scomm.SetCurrentWorksheet(ws_name) mocked_uploader._scomm.GetRowCacheByCol( self.SS_COL_PKG).AndReturn('RCache') mocked_uploader._UploadChangedRows().AndReturn(tuple([0, 1, 2])) mocked_uploader._DeleteOldRows().AndReturn(tuple([3, 4])) self.mox.ReplayAll() # Verify with self.OutputCapturer(): ups.Uploader.Upload(mocked_uploader, ss_key, ws_name) self.mox.VerifyAll() def testUploadChangedRows(self): table = self._CreateTableWithRows(self.COLS, [self.ROW0, self.ROW1, self.ROW2]) mocked_uploader = self._MockUploader(table=table) def RowVerifier(row_delta, golden_col_set, golden_row): if golden_col_set != set(row_delta.keys()): return False for col in row_delta: val = row_delta[col] if val != golden_row[col]: return False return True # First Row. # Pretend first row does not exist already in online spreadsheet # by returning (None, None) from _GetSSRowForPackage. # row0_pkg = self.ROW0[self.COL_PKG] mocked_uploader._GetSSRowForPackage(row0_pkg).AndReturn(None) mocked_uploader._scomm.InsertRow(mox.IgnoreArg()) # Second Row. # Pretend second row does already exist in online spreadsheet, and # pretend that it has a different value that needs to be changed # by an upload. row1_pkg = self.ROW1[self.COL_PKG] row1_reverse_delta = {self.SS_COL_VER: '1.2.3'} ss_row1 = dict(self.SS_ROW1) for col in row1_reverse_delta: ss_row1[col] = row1_reverse_delta[col] ss_row1 = gdata_lib.SpreadsheetRow('OrigRow1', 3, ss_row1) mocked_uploader._GetSSRowForPackage(row1_pkg).AndReturn(ss_row1) # Prepare verfication for row. g_col_set1 = set(row1_reverse_delta.keys()) g_row1 = gdata_lib.PrepRowForSS(self.SS_ROW1) row1_verifier = lambda rdelta: RowVerifier(rdelta, g_col_set1, g_row1) mocked_uploader._scomm.UpdateRowCellByCell(3, mox.Func(row1_verifier)) # Third Row. # Pretend third row does already exist in online spreadsheet, and # pretend that several values need to be changed by an upload. row2_pkg = self.ROW2[self.COL_PKG] row2_reverse_delta = { self.SS_COL_STATUS: 'needs upgrade', self.SS_COL_VER: '0.5', self.SS_COL_TARGET: 'chromeos-foo', } ss_row2 = dict(self.SS_ROW2) for col in row2_reverse_delta: ss_row2[col] = row2_reverse_delta[col] ss_row2 = gdata_lib.SpreadsheetRow('OrigRow2', 4, ss_row2) mocked_uploader._GetSSRowForPackage(row2_pkg).AndReturn(ss_row2) # Prepare verification for row. g_col_set2 = set(row2_reverse_delta.keys()) g_row2 = gdata_lib.PrepRowForSS(self.SS_ROW2) row2_verifier = lambda rdelta: RowVerifier(rdelta, g_col_set2, g_row2) mocked_uploader._scomm.UpdateRowCellByCell(4, mox.Func(row2_verifier)) self.mox.ReplayAll() # Verify with self.OutputCapturer(): ups.Uploader._UploadChangedRows(mocked_uploader) self.mox.VerifyAll() def testDeleteOldRows(self): mocked_uploader = self._MockUploader() # Pretend spreadsheet has 2 rows, one in table and one not. ss_row1 = gdata_lib.SpreadsheetRow('OrigRow1', 2, self.SS_ROW1) ss_row2 = gdata_lib.SpreadsheetRow('OrigRow2', 3, self.SS_ROW2) ss_rows = (ss_row1, ss_row2) mocked_uploader._scomm.GetRows().AndReturn(ss_rows) # We expect ROW2 in spreadsheet to be deleted. mocked_uploader._scomm.DeleteRow('OrigRow2') self.mox.ReplayAll() # Verify with self.OutputCapturer(): ups.Uploader._DeleteOldRows(mocked_uploader) self.mox.VerifyAll()
class Uploader(object): """Uploads portage package status data from csv file to Google spreadsheet.""" __slots__ = ('_creds', # gdata_lib.Creds object '_scomm', # gdata_lib.SpreadsheetComm object '_ss_row_cache', # dict with key=pkg, val=SpreadsheetRow obj '_csv_table', # table.Table of csv rows ) ID_COL = utable.UpgradeTable.COL_PACKAGE SS_ID_COL = gdata_lib.PrepColNameForSS(ID_COL) SOURCE = 'Uploaded from CSV' def __init__(self, creds, table_obj): self._creds = creds self._csv_table = table_obj self._scomm = None self._ss_row_cache = None def _GetSSRowForPackage(self, package): """Return the SpreadsheetRow corresponding to Package=|package|.""" if package in self._ss_row_cache: row = self._ss_row_cache[package] if isinstance(row, list): raise LookupError('More than one row in spreadsheet with Package=%s' % package) return row return None def Upload(self, ss_key, ws_name): """Upload |_csv_table| to the given Google Spreadsheet. The spreadsheet is identified the spreadsheet key |ss_key|. The worksheet within that spreadsheet is identified by the worksheet name |ws_name|. """ if self._scomm: self._scomm.SetCurrentWorksheet(ws_name) else: self._scomm = gdata_lib.SpreadsheetComm() self._scomm.Connect(self._creds, ss_key, ws_name, source='Upload Package Status') oper.Notice('Caching rows for worksheet %r.' % self._scomm.ws_name) self._ss_row_cache = self._scomm.GetRowCacheByCol(self.SS_ID_COL) oper.Notice('Uploading changes to worksheet "%s" of spreadsheet "%s" now.' % (self._scomm.ws_name, self._scomm.ss_key)) oper.Info('Details by package: S=Same, C=Changed, A=Added, D=Deleted') rows_unchanged, rows_updated, rows_inserted = self._UploadChangedRows() rows_deleted, rows_with_owner_deleted = self._DeleteOldRows() oper.Notice('Final row stats for worksheet "%s"' ': %d changed, %d added, %d deleted, %d same.' % (self._scomm.ws_name, rows_updated, rows_inserted, rows_deleted, rows_unchanged)) if rows_with_owner_deleted: oper.Warning('%d rows with owner entry deleted, see above warnings.' % rows_with_owner_deleted) else: oper.Notice('No rows with owner entry were deleted.') def _UploadChangedRows(self): """Upload all rows in table that need to be changed in spreadsheet.""" rows_unchanged, rows_updated, rows_inserted = (0, 0, 0) # Go over all rows in csv table. Identify existing row by the 'Package' # column. Either update existing row or create new one. for csv_row in self._csv_table: # Seed new row values from csv_row values, with column translation. new_row = dict((gdata_lib.PrepColNameForSS(key), csv_row[key]) for key in csv_row) # Retrieve row values already in spreadsheet, along with row index. csv_package = csv_row[self.ID_COL] ss_row = self._GetSSRowForPackage(csv_package) if ss_row: changed = [] # Gather changes for log message. # Check each key/value in new_row to see if it is different from what # is already in spreadsheet (ss_row). Keep only differences to get # the row delta. row_delta = {} for col in new_row: if col in ss_row: ss_val = ss_row[col] new_val = new_row[col] if (ss_val or new_val) and ss_val != new_val: changed.append('%s="%s"->"%s"' % (col, ss_val, new_val)) row_delta[col] = new_val if row_delta: self._scomm.UpdateRowCellByCell(ss_row.ss_row_num, gdata_lib.PrepRowForSS(row_delta)) rows_updated += 1 oper.Info('C %-30s: %s' % (csv_package, ', '.join(changed))) else: rows_unchanged += 1 oper.Info('S %-30s:' % csv_package) else: self._scomm.InsertRow(gdata_lib.PrepRowForSS(new_row)) rows_inserted += 1 row_descr_list = [] for col in sorted(new_row.keys()): if col != self.ID_COL: row_descr_list.append('%s="%s"' % (col, new_row[col])) oper.Info('A %-30s: %s' % (csv_package, ', '.join(row_descr_list))) return (rows_unchanged, rows_updated, rows_inserted) def _DeleteOldRows(self): """Delete all rows from spreadsheet that not found in table.""" oper.Notice('Checking for rows in worksheet that should be deleted now.') rows_deleted, rows_with_owner_deleted = (0, 0) # Also need to delete rows in spreadsheet that are not in csv table. ss_rows = self._scomm.GetRows() for ss_row in ss_rows: ss_package = gdata_lib.ScrubValFromSS(ss_row[self.SS_ID_COL]) # See whether this row is in csv table. csv_rows = self._csv_table.GetRowsByValue({ self.ID_COL: ss_package }) if not csv_rows: # Row needs to be deleted from spreadsheet. owner_val = None owner_notes_val = None row_descr_list = [] for col in sorted(ss_row.keys()): if col == 'owner': owner_val = ss_row[col] if col == 'ownernotes': owner_notes_val = ss_row[col] # Don't include ID_COL value in description, it is in prefix already. if col != self.SS_ID_COL: val = ss_row[col] row_descr_list.append('%s="%s"' % (col, val)) oper.Info('D %-30s: %s' % (ss_package, ', '.join(row_descr_list))) if owner_val or owner_notes_val: rows_with_owner_deleted += 1 oper.Notice('WARNING: Deleting spreadsheet row with owner entry:\n' + ' %-30s: Owner=%s, Owner Notes=%s' % (ss_package, owner_val, owner_notes_val)) self._scomm.DeleteRow(ss_row.ss_row_obj) rows_deleted += 1 return (rows_deleted, rows_with_owner_deleted)
def _GetRowValue(self, row, colName, arch=None): """Get value from |row| at |colName|, adjusted for |arch|""" if arch: colName = utable.UpgradeTable.GetColumnName(colName, arch=arch) colName = gdata_lib.PrepColNameForSS(colName) return row[colName]
from chromite.lib import operation from chromite.lib import upgrade_table as utable from chromite.scripts import upload_package_status as ups # pylint: disable=W0201,R0904 PROJECT_NAME = 'chromium-os' PKGS_WS_NAME = 'Packages' CROS_ORG = 'chromium.org' CHROMIUMOS_SITE = 'http://www.%s/%s' % (CROS_ORG, PROJECT_NAME) PKG_UPGRADE_PAGE = '%s/gentoo-package-upgrade-process' % CHROMIUMOS_SITE DOCS_SITE = 'https://docs.google.com/a' COL_PACKAGE = gdata_lib.PrepColNameForSS(utable.UpgradeTable.COL_PACKAGE) COL_TEAM = gdata_lib.PrepColNameForSS('Team/Lead') COL_OWNER = gdata_lib.PrepColNameForSS('Owner') COL_TRACKER = gdata_lib.PrepColNameForSS('Tracker') ARCHES = ('amd64', 'arm', 'x86') oper = operation.Operation('sync_package_status') def _GetPkgSpreadsheetURL(ss_key): return '%s/%s/spreadsheet/ccc?key=%s' % (DOCS_SITE, CROS_ORG, ss_key) class SyncError(RuntimeError): """Extend RuntimeError for use in this module."""