def _GetCachedCommitId(commit_name):
'''Determines which commit ID was last cached.
'''
commit_tracker = CommitTracker(
ObjectStoreCreator(store_type=PersistentObjectStoreFake,
start_empty=False))
return commit_tracker.Get(commit_name).Get()
def _UpdateCommitId(commit_name, commit_id):
'''Sets the commit ID for a named commit. This is the final step performed
during update. Once all the appropriate datastore entries have been populated
for a new commit ID, the 'master' commit entry is updated to that ID and the
frontend will begin serving the new data.
Note that this requires an access token identifying the main service account
for the chrome-apps-doc project. VM instances will get this automatically
from their environment, but if you want to do a local push to prod you will
need to set the DOCSERVER_ACCESS_TOKEN environment variable appropriately.
'''
commit_tracker = CommitTracker(
ObjectStoreCreator(store_type=PersistentObjectStoreFake,
start_empty=False))
commit_tracker.Set(commit_name, commit_id).Get()
logging.info('Commit "%s" updated to %s.' % (commit_name, commit_id))
def Get(self):
object_store_creator = ObjectStoreCreator(start_empty=False)
commit_tracker = CommitTracker(object_store_creator)
def generate_response(result):
commit_id, history = result
history_log = ''.join('%s: %s<br>' %
(entry.datetime, entry.commit_id)
for entry in reversed(history))
response = 'Current commit: %s<br><br>Most recent commits:<br>%s' % (
commit_id, history_log)
return response
commit_name = self._request.path
id_future = commit_tracker.Get(commit_name)
history_future = commit_tracker.GetHistory(commit_name)
return Response.Ok(
All((id_future, history_future)).Then(generate_response).Get())
def CreateServerInstance(self):
object_store_creator = ObjectStoreCreator(start_empty=False)
branch_utility = self._delegate.CreateBranchUtility(
object_store_creator)
commit_tracker = CommitTracker(object_store_creator)
# In production have offline=True so that we can catch cron errors. In
# development it's annoying to have to run the cron job, so offline=False.
# Note that offline=True if running on any appengine server due to
# http://crbug.com/345361.
host_file_system_provider = self._delegate.CreateHostFileSystemProvider(
object_store_creator,
offline=not (IsDevServer() or IsReleaseServer()),
pinned_commit=commit_tracker.Get('master').Get(),
cache_only=True)
return ServerInstance(
object_store_creator,
CompiledFileSystem.Factory(object_store_creator), branch_utility,
host_file_system_provider,
CloudStorageFileSystemProvider(object_store_creator))
def CreateCommitTracker(self):
return CommitTracker(ObjectStoreCreator(start_empty=False))
def setUp(self):
object_store_creator = ObjectStoreCreator(start_empty=True)
self._commit_tracker = CommitTracker(object_store_creator)
for id in _COMMIT_HISTORY_DATA:
self._commit_tracker.Set('master', id).Get()
def _GetImpl(self):
path = self._request.path.strip('/')
parts = self._request.path.split('/', 1)
source_name = parts[0]
if len(parts) == 2:
source_path = parts[1]
else:
source_path = None
_log.info(
'starting refresh of %s DataSource %s' %
(source_name, '' if source_path is None else '[%s]' % source_path))
if 'commit' in self._request.arguments:
commit = self._request.arguments['commit']
else:
_log.warning('No commit given; refreshing from master. '
'This is probably NOT what you want.')
commit = None
server_instance = self._CreateServerInstance(commit)
commit_tracker = CommitTracker(server_instance.object_store_creator)
refresh_tracker = RefreshTracker(server_instance.object_store_creator)
# If no commit was given, use the ID of the last cached master commit.
# This allows sources external to the chromium repository to be updated
# independently from individual refresh cycles.
if commit is None:
commit = commit_tracker.Get('master').Get()
success = True
try:
if source_name == 'platform_bundle':
data_source = server_instance.platform_bundle
elif source_name == 'content_providers':
data_source = server_instance.content_providers
else:
data_source = CreateDataSource(source_name, server_instance)
class_name = data_source.__class__.__name__
refresh_future = data_source.Refresh(source_path)
assert isinstance(
refresh_future,
Future), ('%s.Refresh() did not return a Future' % class_name)
timer = Timer()
try:
refresh_future.Get()
# Mark this (commit, task) pair as completed and then see if this
# concludes the full cache refresh. The list of tasks required to
# complete a cache refresh is registered (and keyed on commit ID) by the
# CronServlet before kicking off all the refresh tasks.
(refresh_tracker.MarkTaskComplete(
commit,
path).Then(lambda _: refresh_tracker.GetRefreshComplete(
commit)).Then(lambda is_complete: commit_tracker.Set(
'master', commit) if is_complete else None).Get())
except Exception as e:
_log.error('%s: error %s' %
(class_name, traceback.format_exc()))
success = False
if IsFileSystemThrottledError(e):
return Response.ThrottledError('Throttled')
raise
finally:
_log.info('Refreshing %s took %s' %
(class_name, timer.Stop().FormatElapsed()))
except:
success = False
# This should never actually happen.
_log.error('uncaught error: %s' % traceback.format_exc())
raise
finally:
_log.info('finished (%s)', 'success' if success else 'FAILED')
return (Response.Ok('Success')
if success else Response.InternalError('Failure'))
def Get(self):
object_store_creator = ObjectStoreCreator(start_empty=False)
commit_tracker = CommitTracker(object_store_creator)
return Response.Ok(commit_tracker.Get(self._request.path).Get())