def CreateServerInstanceForChannel(self, channel):
base_object_store_creator = ObjectStoreCreator(channel,
start_empty=False)
# TODO(fj): Use OfflineFileSystem here once all json/idl files in api/
# are pulled into data store by cron jobs.
base_file_system = CachingFileSystem(
self._delegate.CreateHostFileSystemForBranch(channel),
base_object_store_creator)
base_compiled_fs_factory = CompiledFileSystem.Factory(
base_file_system, base_object_store_creator)
object_store_creator = ObjectStoreCreator('trunk@%s' % self._issue,
start_empty=False)
rietveld_patcher = CachingRietveldPatcher(
RietveldPatcher(
svn_constants.EXTENSIONS_PATH, self._issue,
AppEngineUrlFetcher(url_constants.CODEREVIEW_SERVER)),
object_store_creator)
patched_file_system = PatchedFileSystem(base_file_system,
rietveld_patcher)
patched_compiled_fs_factory = CompiledFileSystem.Factory(
patched_file_system, object_store_creator)
compiled_fs_factory = ChainedCompiledFileSystem.Factory([
(patched_compiled_fs_factory, patched_file_system),
(base_compiled_fs_factory, base_file_system)
])
return ServerInstance(
channel, object_store_creator, patched_file_system,
self._delegate.CreateAppSamplesFileSystem(
base_object_store_creator), '/_patch/%s' % self._issue,
compiled_fs_factory)
def _CreateFactory():
return CompiledFileSystem.Factory(
TestFileSystem(deepcopy(_TEST_DATA)),
ObjectStoreCreator('test',
start_empty=False,
store_type=TestObjectStore,
disable_wrappers=True))
def CreateServerInstance(self):
object_store_creator = ObjectStoreCreator(start_empty=False)
branch_utility = self._delegate.CreateBranchUtility(
object_store_creator)
host_file_system_creator = self._delegate.CreateHostFileSystemCreator(
object_store_creator)
# offline=False because a patch can rely on files that are already in SVN
# repository but not yet pulled into data store by cron jobs (a typical
# example is to add documentation for an existing API).
base_file_system = CachingFileSystem(
host_file_system_creator.Create(offline=False),
object_store_creator)
base_compiled_fs_factory = CompiledFileSystem.Factory(
base_file_system, object_store_creator)
rietveld_patcher = CachingRietveldPatcher(
RietveldPatcher(
svn_constants.EXTENSIONS_PATH, self._issue,
AppEngineUrlFetcher(url_constants.CODEREVIEW_SERVER)),
object_store_creator)
patched_file_system = PatchedFileSystem(base_file_system,
rietveld_patcher)
patched_compiled_fs_factory = CompiledFileSystem.Factory(
patched_file_system, object_store_creator)
compiled_fs_factory = ChainedCompiledFileSystem.Factory([
(patched_compiled_fs_factory, patched_file_system),
(base_compiled_fs_factory, base_file_system)
])
return ServerInstance(
object_store_creator, patched_file_system,
self._delegate.CreateAppSamplesFileSystem(object_store_creator),
'/_patch/%s' % self._issue, compiled_fs_factory, branch_utility,
host_file_system_creator)
def _GetCachedCommitId(commit_name):
'''Determines which commit ID was last cached.
'''
commit_tracker = CommitTracker(
ObjectStoreCreator(store_type=PersistentObjectStoreFake,
start_empty=False))
return commit_tracker.Get(commit_name).Get()
def setUp(self):
self._datetime = FakeDateTime()
self._test_patcher = TestPatcher(_TEST_PATCH_VERSION,
_TEST_PATCH_FILES, _TEST_PATCH_DATA)
self._patcher = CachingRietveldPatcher(
self._test_patcher, ObjectStoreCreator(start_empty=False),
self._datetime)
def setUp(self):
self._object_store_creator = ObjectStoreCreator('chained',
start_empty=False)
self._base_object_store_creator = ObjectStoreCreator('base',
start_empty=False)
base_file_system = TestFileSystem(_TEST_DATA_BASE)
self._base_factory = CompiledFileSystem.Factory(
base_file_system, self._base_object_store_creator)
self._file_system = TestFileSystem(_TEST_DATA_NEW)
self._patched_factory = CompiledFileSystem.Factory(
self._file_system, self._object_store_creator)
self._chained_factory = ChainedCompiledFileSystem.Factory([
(self._patched_factory, self._file_system),
(self._base_factory, base_file_system)
])
self._base_compiled_fs = self._base_factory.CreateIdentity(
TestFileSystem)
self._chained_compiled_fs = self._chained_factory.CreateIdentity(
TestFileSystem)
def _GetTestCompiledFsCreator():
'''Returns a function which creates CompiledFileSystem views of
TestFileSystems backed by _TEST_DATA.
'''
return functools.partial(
CompiledFileSystem.Factory(
ObjectStoreCreator(start_empty=False,
store_type=TestObjectStore,
disable_wrappers=True), ).Create,
TestFileSystem(deepcopy(_TEST_DATA)))
def ForLocal():
channel = 'trunk'
object_store_creator = ObjectStoreCreator(channel,
start_empty=False,
store_type=TestObjectStore)
file_system = CachingFileSystem(LocalFileSystem.Create(),
object_store_creator)
return ServerInstance(
channel, object_store_creator, file_system, EmptyDirFileSystem(),
'', CompiledFileSystem.Factory(file_system, object_store_creator))
def ForLocal():
object_store_creator = ObjectStoreCreator(start_empty=False,
store_type=TestObjectStore)
host_file_system_provider = HostFileSystemProvider.ForLocal(
object_store_creator)
return ServerInstance(object_store_creator,
CompiledFileSystem.Factory(object_store_creator),
TestBranchUtility.CreateWithCannedData(),
host_file_system_provider,
GithubFileSystemProvider.ForEmpty())
def setUp(self):
self._datetime = FakeDateTime()
# CachingRietveldPatcher should always call Apply with binary=True.
self._test_patcher = TestPatcher(_TEST_PATCH_VERSION,
_TEST_PATCH_FILES,
_TEST_PATCH_DATA,
assert_binary=True)
self._patcher = CachingRietveldPatcher(
self._test_patcher, ObjectStoreCreator('test', start_empty=False),
self._datetime)
def ForLocal():
object_store_creator = ObjectStoreCreator(start_empty=False,
store_type=TestObjectStore)
host_file_system_creator = HostFileSystemCreator.ForLocal(
object_store_creator)
trunk_file_system = host_file_system_creator.Create()
return ServerInstance(
object_store_creator, trunk_file_system, EmptyDirFileSystem(), '',
CompiledFileSystem.Factory(trunk_file_system,
object_store_creator),
TestBranchUtility.CreateWithCannedData(), host_file_system_creator)
def ForLocal():
object_store_creator = ObjectStoreCreator(start_empty=False,
store_type=TestObjectStore)
host_file_system_provider = HostFileSystemProvider.ForLocal(
object_store_creator)
return ServerInstance(
object_store_creator,
CompiledFileSystem.Factory(object_store_creator),
BranchUtility.Create(object_store_creator),
host_file_system_provider,
CloudStorageFileSystemProvider(object_store_creator))
def setUp(self):
object_store_creator = ObjectStoreCreator(start_empty=False)
base_file_system = TestFileSystem(_TEST_DATA_BASE, identity='base')
self._base_compiled_fs = CompiledFileSystem.Factory(
object_store_creator).Create(base_file_system, identity,
ChainedCompiledFileSystemTest)
chained_factory = ChainedCompiledFileSystem.Factory(
[base_file_system], object_store_creator)
self._new_file_system = TestFileSystem(_TEST_DATA_NEW, identity='new')
self._chained_compiled_fs = chained_factory.Create(
self._new_file_system, identity, ChainedCompiledFileSystemTest)
def _CreateServerInstance(commit):
'''Creates a ServerInstance based on origin/master.
'''
object_store_creator = ObjectStoreCreator(
start_empty=False, store_type=PersistentObjectStoreFake)
branch_utility = BranchUtility.Create(object_store_creator)
host_file_system_provider = HostFileSystemProvider(object_store_creator,
pinned_commit=commit)
gcs_file_system_provider = CloudStorageFileSystemProvider(
object_store_creator)
return ServerInstance(object_store_creator,
CompiledFileSystem.Factory(object_store_creator),
branch_utility, host_file_system_provider,
gcs_file_system_provider)
def CreateServerInstance(self):
object_store_creator = ObjectStoreCreator(start_empty=False)
branch_utility = self._delegate.CreateBranchUtility(
object_store_creator)
# In production have offline=True so that we can catch cron errors. In
# development it's annoying to have to run the cron job, so offline=False.
host_file_system_provider = self._delegate.CreateHostFileSystemProvider(
object_store_creator, offline=not IsDevServer())
github_file_system_provider = self._delegate.CreateGithubFileSystemProvider(
object_store_creator)
return ServerInstance(object_store_creator,
CompiledFileSystem.Factory(object_store_creator),
branch_utility, host_file_system_provider,
github_file_system_provider)
def _CreateCachingFileSystem(self, fs, start_empty=False):
def store_type_constructor(namespace, start_empty=False):
'''Returns an ObjectStore backed onto test-lifetime-persistent objects
in |_object_store_dbs|.
'''
if namespace not in self._object_store_dbs:
self._object_store_dbs[namespace] = {}
db = self._object_store_dbs[namespace]
if start_empty:
db.clear()
return TestObjectStore(namespace, init=db)
object_store_creator = ObjectStoreCreator(start_empty=start_empty,
store_type=store_type_constructor)
return CachingFileSystem(fs, object_store_creator)
def CreateServerInstance(self):
# start_empty=False because a patch can rely on files that are already in
# SVN repository but not yet pulled into data store by cron jobs (a typical
# example is to add documentation for an existing API).
object_store_creator = ObjectStoreCreator(start_empty=False)
unpatched_file_system = self._delegate.CreateHostFileSystemProvider(
object_store_creator).GetTrunk()
rietveld_patcher = CachingRietveldPatcher(
RietveldPatcher(
self._issue,
AppEngineUrlFetcher(url_constants.CODEREVIEW_SERVER)),
object_store_creator)
patched_file_system = PatchedFileSystem(unpatched_file_system,
rietveld_patcher)
patched_host_file_system_provider = (
self._delegate.CreateHostFileSystemProvider(
object_store_creator,
# The patched file system needs to be online otherwise it'd be
# impossible to add files in the patches.
offline=False,
# The trunk file system for this creator should be the patched one.
default_trunk_instance=patched_file_system))
combined_compiled_fs_factory = ChainedCompiledFileSystem.Factory(
[unpatched_file_system], object_store_creator)
branch_utility = self._delegate.CreateBranchUtility(
object_store_creator)
server_instance = ServerInstance(
object_store_creator,
combined_compiled_fs_factory,
branch_utility,
patched_host_file_system_provider,
self._delegate.CreateGithubFileSystemProvider(
object_store_creator),
CloudStorageFileSystemProvider(object_store_creator),
base_path='/_patch/%s/' % self._issue)
# HACK: if content_providers.json changes in this patch then the cron needs
# to be re-run to pull in the new configuration.
_, _, modified = rietveld_patcher.GetPatchedFiles()
if CONTENT_PROVIDERS in modified:
server_instance.content_providers.Cron().Get()
return server_instance
def Get(self):
object_store_creator = ObjectStoreCreator(start_empty=False)
refresh_tracker = RefreshTracker(object_store_creator)
commit_id = self._request.path
work_order = refresh_tracker._GetWorkOrder(commit_id).Get()
task_names = ['%s@%s' % (commit_id, task) for task in work_order.tasks]
completions = refresh_tracker._task_completions.GetMulti(
task_names).Get()
missing = []
for task in task_names:
if task not in completions:
missing.append(task)
response = 'Missing:<br>%s' % ''.join('%s<br>' % task
for task in missing)
return Response.Ok(response)
def _CreateServerInstance(self, revision):
object_store_creator = ObjectStoreCreator(start_empty=True)
branch_utility = self._delegate.CreateBranchUtility(
object_store_creator)
host_file_system_creator = self._delegate.CreateHostFileSystemCreator(
object_store_creator)
host_file_system = host_file_system_creator.Create(revision=revision)
app_samples_file_system = self._delegate.CreateAppSamplesFileSystem(
object_store_creator)
compiled_host_fs_factory = CompiledFileSystem.Factory(
host_file_system, object_store_creator)
return ServerInstance(object_store_creator, host_file_system,
app_samples_file_system, '',
compiled_host_fs_factory, branch_utility,
host_file_system_creator)
def _UpdateCommitId(commit_name, commit_id):
'''Sets the commit ID for a named commit. This is the final step performed
during update. Once all the appropriate datastore entries have been populated
for a new commit ID, the 'master' commit entry is updated to that ID and the
frontend will begin serving the new data.
Note that this requires an access token identifying the main service account
for the chrome-apps-doc project. VM instances will get this automatically
from their environment, but if you want to do a local push to prod you will
need to set the DOCSERVER_ACCESS_TOKEN environment variable appropriately.
'''
commit_tracker = CommitTracker(
ObjectStoreCreator(store_type=PersistentObjectStoreFake,
start_empty=False))
commit_tracker.Set(commit_name, commit_id).Get()
logging.info('Commit "%s" updated to %s.' % (commit_name, commit_id))
def _CreateServerInstance(self, revision):
'''Creates a ServerInstance pinned to |revision|, or HEAD if None.
NOTE: If passed None it's likely that during the cron run patches will be
submitted at HEAD, which may change data underneath the cron run.
'''
object_store_creator = ObjectStoreCreator(start_empty=True)
branch_utility = self._delegate.CreateBranchUtility(
object_store_creator)
host_file_system_provider = self._delegate.CreateHostFileSystemProvider(
object_store_creator, max_trunk_revision=revision)
github_file_system_provider = self._delegate.CreateGithubFileSystemProvider(
object_store_creator)
return ServerInstance(object_store_creator,
CompiledFileSystem.Factory(object_store_creator),
branch_utility, host_file_system_provider,
github_file_system_provider)
def Get(self):
object_store_creator = ObjectStoreCreator(start_empty=False)
commit_tracker = CommitTracker(object_store_creator)
def generate_response(result):
commit_id, history = result
history_log = ''.join('%s: %s<br>' %
(entry.datetime, entry.commit_id)
for entry in reversed(history))
response = 'Current commit: %s<br><br>Most recent commits:<br>%s' % (
commit_id, history_log)
return response
commit_name = self._request.path
id_future = commit_tracker.Get(commit_name)
history_future = commit_tracker.GetHistory(commit_name)
return Response.Ok(
All((id_future, history_future)).Then(generate_response).Get())
def CreateServerInstance(self):
object_store_creator = ObjectStoreCreator(start_empty=False)
branch_utility = self._delegate.CreateBranchUtility(
object_store_creator)
# In production have offline=True so that we can catch cron errors. In
# development it's annoying to have to run the cron job, so offline=False.
# Note that offline=True if running on any appengine server due to
# http://crbug.com/345361.
host_file_system_provider = self._delegate.CreateHostFileSystemProvider(
object_store_creator,
offline=not (IsDevServer() or IsReleaseServer()))
github_file_system_provider = self._delegate.CreateGithubFileSystemProvider(
object_store_creator)
return ServerInstance(
object_store_creator,
CompiledFileSystem.Factory(object_store_creator), branch_utility,
host_file_system_provider, github_file_system_provider,
CloudStorageFileSystemProvider(object_store_creator))
def CreateServerInstance(self):
object_store_creator = ObjectStoreCreator(start_empty=False)
branch_utility = self._delegate.CreateBranchUtility(
object_store_creator)
# In production have offline=True so that we can catch cron errors. In
# development it's annoying to have to run the cron job, so offline=False.
#
# XXX(kalman): The above comment is not true, I have temporarily disabled
# this while the cron is running out of memory and not reliably finishing.
# In the meantime, live dangerously and fetch content if it's not there.
# I.e. never offline. See http://crbug.com/345361.
host_file_system_provider = self._delegate.CreateHostFileSystemProvider(
object_store_creator, offline=False
) # XXX(kalman): condition should be "not IsDevServer()"
github_file_system_provider = self._delegate.CreateGithubFileSystemProvider(
object_store_creator)
return ServerInstance(
object_store_creator,
CompiledFileSystem.Factory(object_store_creator), branch_utility,
host_file_system_provider, github_file_system_provider,
CloudStorageFileSystemProvider(object_store_creator))
def testDetectInline(self):
schema = {
'types': [
{
'id': 'Key',
'items': {
'$ref': 'Value'
}
},
{
'id': 'Value',
'marker': True
}
]
}
expected_schema = {
'types': [
{
'id': 'Key',
'items': {
'marker': True,
}
}
]
}
object_store_creator = ObjectStoreCreator(start_empty=False)
host_file_system_provider = HostFileSystemProvider(object_store_creator)
schema_processor = SchemaProcessor(_FakeReferenceResolver(),
_FakeAPIModels(),
_FakeFeaturesBundle(),
CompiledFileSystem.Factory(
object_store_creator),
host_file_system_provider.GetMaster(),
False)
schema_processor._DetectInlineableTypes(schema)
schema_processor._InlineDocs(schema)
self.assertEqual(expected_schema, schema)
def _CreateObjectStoreCreator(self, channel):
return ObjectStoreCreator(channel, start_empty=True)
def setUp(self):
object_store_creator = ObjectStoreCreator(start_empty=True)
self._commit_tracker = CommitTracker(object_store_creator)
for id in _COMMIT_HISTORY_DATA:
self._commit_tracker.Set('master', id).Get()
def setUp(self):
self.creator = ObjectStoreCreator(_FooClass,
'3-0',
'test',
store_type=TestObjectStore)
def setUp(self):
self._creator = ObjectStoreCreator('trunk',
start_empty=False,
store_type=TestObjectStore,
disable_wrappers=True)
def CreateCommitTracker(self):
return CommitTracker(ObjectStoreCreator(start_empty=False))