def _GetCachedCommitId(commit_name): '''Determines which commit ID was last cached. ''' commit_tracker = CommitTracker( ObjectStoreCreator(store_type=PersistentObjectStoreFake, start_empty=False)) return commit_tracker.Get(commit_name).Get()
class AdminServletsTest(unittest.TestCase): def setUp(self): object_store_creator = ObjectStoreCreator(start_empty=True) self._commit_tracker = CommitTracker(object_store_creator) for id in _COMMIT_HISTORY_DATA: self._commit_tracker.Set('master', id).Get() def _ResetCommit(self, commit_name, commit_id): return ResetCommitServlet( Request.ForTest('%s/%s' % (commit_name, commit_id)), _ResetCommitDelegate(self._commit_tracker)).Get() def _AssertBadRequest(self, commit_name, commit_id): response = self._ResetCommit(commit_name, commit_id) self.assertEqual( response.status, 400, 'Should have failed to reset to commit %s to %s.' % (commit_name, commit_id)) def _AssertOk(self, commit_name, commit_id): response = self._ResetCommit(commit_name, commit_id) self.assertEqual( response.status, 200, 'Failed to reset commit %s to %s.' % (commit_name, commit_id)) def testResetCommitServlet(self): # Make sure all the valid commits can be used for reset. for id in _COMMIT_HISTORY_DATA: self._AssertOk('master', id) # Non-existent commit should fail to update self._AssertBadRequest('master', 'b000000000000000000000000000000000000000') # Commit 'master' should still point to the last valid entry self.assertEqual( self._commit_tracker.Get('master').Get(), _COMMIT_HISTORY_DATA[-1]) # Reset to a valid commit but older self._AssertOk('master', _COMMIT_HISTORY_DATA[0]) # Commit 'master' should point to the first history entry self.assertEqual( self._commit_tracker.Get('master').Get(), _COMMIT_HISTORY_DATA[0]) # Add a new entry to the history and validate that it can be used for reset. _NEW_ENTRY = '9999999999999999999999999999999999999999' self._commit_tracker.Set('master', _NEW_ENTRY).Get() self._AssertOk('master', _NEW_ENTRY) # Add a bunch (> 50) of entries to ensure that _NEW_ENTRY has been flushed # out of the history. for i in xrange(0, 20): for id in _COMMIT_HISTORY_DATA: self._commit_tracker.Set('master', id).Get() # Verify that _NEW_ENTRY is no longer valid for reset. self._AssertBadRequest('master', _NEW_ENTRY)
def _UpdateCommitId(commit_name, commit_id): '''Sets the commit ID for a named commit. This is the final step performed during update. Once all the appropriate datastore entries have been populated for a new commit ID, the 'master' commit entry is updated to that ID and the frontend will begin serving the new data. Note that this requires an access token identifying the main service account for the chrome-apps-doc project. VM instances will get this automatically from their environment, but if you want to do a local push to prod you will need to set the DOCSERVER_ACCESS_TOKEN environment variable appropriately. ''' commit_tracker = CommitTracker( ObjectStoreCreator(store_type=PersistentObjectStoreFake, start_empty=False)) commit_tracker.Set(commit_name, commit_id).Get() logging.info('Commit "%s" updated to %s.' % (commit_name, commit_id))
def Get(self): object_store_creator = ObjectStoreCreator(start_empty=False) commit_tracker = CommitTracker(object_store_creator) def generate_response(result): commit_id, history = result history_log = ''.join('%s: %s<br>' % (entry.datetime, entry.commit_id) for entry in reversed(history)) response = 'Current commit: %s<br><br>Most recent commits:<br>%s' % ( commit_id, history_log) return response commit_name = self._request.path id_future = commit_tracker.Get(commit_name) history_future = commit_tracker.GetHistory(commit_name) return Response.Ok( All((id_future, history_future)).Then(generate_response).Get())
def CreateServerInstance(self): object_store_creator = ObjectStoreCreator(start_empty=False) branch_utility = self._delegate.CreateBranchUtility( object_store_creator) commit_tracker = CommitTracker(object_store_creator) # In production have offline=True so that we can catch cron errors. In # development it's annoying to have to run the cron job, so offline=False. # Note that offline=True if running on any appengine server due to # http://crbug.com/345361. host_file_system_provider = self._delegate.CreateHostFileSystemProvider( object_store_creator, offline=not (IsDevServer() or IsReleaseServer()), pinned_commit=commit_tracker.Get('master').Get(), cache_only=True) return ServerInstance( object_store_creator, CompiledFileSystem.Factory(object_store_creator), branch_utility, host_file_system_provider, CloudStorageFileSystemProvider(object_store_creator))
def CreateCommitTracker(self): return CommitTracker(ObjectStoreCreator(start_empty=False))
def setUp(self): object_store_creator = ObjectStoreCreator(start_empty=True) self._commit_tracker = CommitTracker(object_store_creator) for id in _COMMIT_HISTORY_DATA: self._commit_tracker.Set('master', id).Get()
def _GetImpl(self): path = self._request.path.strip('/') parts = self._request.path.split('/', 1) source_name = parts[0] if len(parts) == 2: source_path = parts[1] else: source_path = None _log.info( 'starting refresh of %s DataSource %s' % (source_name, '' if source_path is None else '[%s]' % source_path)) if 'commit' in self._request.arguments: commit = self._request.arguments['commit'] else: _log.warning('No commit given; refreshing from master. ' 'This is probably NOT what you want.') commit = None server_instance = self._CreateServerInstance(commit) commit_tracker = CommitTracker(server_instance.object_store_creator) refresh_tracker = RefreshTracker(server_instance.object_store_creator) # If no commit was given, use the ID of the last cached master commit. # This allows sources external to the chromium repository to be updated # independently from individual refresh cycles. if commit is None: commit = commit_tracker.Get('master').Get() success = True try: if source_name == 'platform_bundle': data_source = server_instance.platform_bundle elif source_name == 'content_providers': data_source = server_instance.content_providers else: data_source = CreateDataSource(source_name, server_instance) class_name = data_source.__class__.__name__ refresh_future = data_source.Refresh(source_path) assert isinstance( refresh_future, Future), ('%s.Refresh() did not return a Future' % class_name) timer = Timer() try: refresh_future.Get() # Mark this (commit, task) pair as completed and then see if this # concludes the full cache refresh. The list of tasks required to # complete a cache refresh is registered (and keyed on commit ID) by the # CronServlet before kicking off all the refresh tasks. (refresh_tracker.MarkTaskComplete( commit, path).Then(lambda _: refresh_tracker.GetRefreshComplete( commit)).Then(lambda is_complete: commit_tracker.Set( 'master', commit) if is_complete else None).Get()) except Exception as e: _log.error('%s: error %s' % (class_name, traceback.format_exc())) success = False if IsFileSystemThrottledError(e): return Response.ThrottledError('Throttled') raise finally: _log.info('Refreshing %s took %s' % (class_name, timer.Stop().FormatElapsed())) except: success = False # This should never actually happen. _log.error('uncaught error: %s' % traceback.format_exc()) raise finally: _log.info('finished (%s)', 'success' if success else 'FAILED') return (Response.Ok('Success') if success else Response.InternalError('Failure'))
def Get(self): object_store_creator = ObjectStoreCreator(start_empty=False) commit_tracker = CommitTracker(object_store_creator) return Response.Ok(commit_tracker.Get(self._request.path).Get())