def GetSamplesModel(self, platform):
if self._platform_data[platform].samples_model is None:
# Note: samples are super slow in the dev server because it doesn't
# support async fetch, so disable them.
if IsDevServer():
extension_samples_fs = EmptyDirFileSystem()
app_samples_fs = EmptyDirFileSystem()
else:
extension_samples_fs = self._host_fs_at_master
# TODO(kalman): Re-enable the apps samples, see http://crbug.com/344097.
app_samples_fs = EmptyDirFileSystem()
self._platform_data[platform].samples_model = SamplesModel(
extension_samples_fs,
app_samples_fs, self._compiled_fs_factory,
self.GetReferenceResolver(platform), self._base_path, platform)
return self._platform_data[platform].samples_model
def ForTest(file_system):
object_store_creator = ObjectStoreCreator.ForTest()
return ServerInstance(
object_store_creator, file_system, EmptyDirFileSystem(), '',
CompiledFileSystem.Factory(file_system, object_store_creator),
TestBranchUtility.CreateWithCannedData(),
HostFileSystemCreator.ForTest(file_system, object_store_creator))
def __init__(self, channel, object_store_creator_factory, svn_file_system,
github_file_system):
self.svn_file_system = svn_file_system
self.github_file_system = github_file_system
self.compiled_fs_factory = CompiledFileSystem.Factory(
svn_file_system, object_store_creator_factory)
self.api_list_data_source_factory = APIListDataSource.Factory(
self.compiled_fs_factory, svn_constants.API_PATH,
svn_constants.PUBLIC_TEMPLATE_PATH)
self.api_data_source_factory = APIDataSource.Factory(
self.compiled_fs_factory, svn_constants.API_PATH)
self.ref_resolver_factory = ReferenceResolver.Factory(
self.api_data_source_factory, self.api_list_data_source_factory,
object_store_creator_factory)
self.api_data_source_factory.SetReferenceResolverFactory(
self.ref_resolver_factory)
# Note: samples are super slow in the dev server because it doesn't support
# async fetch, so disable them. If you actually want to test samples, then
# good luck, and modify _IsSamplesDisabled at the top.
if _IsSamplesDisabled():
svn_fs_for_samples = EmptyDirFileSystem()
else:
svn_fs_for_samples = self.svn_file_system
self.samples_data_source_factory = SamplesDataSource.Factory(
channel, svn_fs_for_samples, self.github_file_system,
self.ref_resolver_factory, object_store_creator_factory,
svn_constants.EXAMPLES_PATH)
self.api_data_source_factory.SetSamplesDataSourceFactory(
self.samples_data_source_factory)
self.intro_data_source_factory = IntroDataSource.Factory(
self.compiled_fs_factory, self.ref_resolver_factory,
[svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH])
self.sidenav_data_source_factory = SidenavDataSource.Factory(
self.compiled_fs_factory, svn_constants.JSON_PATH)
self.template_data_source_factory = TemplateDataSource.Factory(
channel, self.api_data_source_factory,
self.api_list_data_source_factory, self.intro_data_source_factory,
self.samples_data_source_factory, self.sidenav_data_source_factory,
self.compiled_fs_factory, self.ref_resolver_factory,
svn_constants.PUBLIC_TEMPLATE_PATH,
svn_constants.PRIVATE_TEMPLATE_PATH)
self.example_zipper = ExampleZipper(self.compiled_fs_factory,
svn_constants.DOCS_PATH)
self.path_canonicalizer = PathCanonicalizer(channel,
self.compiled_fs_factory)
self.content_cache = self.compiled_fs_factory.GetOrCreateIdentity()
def ForLocal():
channel = 'trunk'
object_store_creator = ObjectStoreCreator(channel,
start_empty=False,
store_type=TestObjectStore)
file_system = CachingFileSystem(LocalFileSystem.Create(),
object_store_creator)
return ServerInstance(
channel, object_store_creator, file_system, EmptyDirFileSystem(),
'', CompiledFileSystem.Factory(file_system, object_store_creator))
def ForLocal():
object_store_creator = ObjectStoreCreator(start_empty=False,
store_type=TestObjectStore)
host_file_system_creator = HostFileSystemCreator.ForLocal(
object_store_creator)
trunk_file_system = host_file_system_creator.Create()
return ServerInstance(
object_store_creator, trunk_file_system, EmptyDirFileSystem(), '',
CompiledFileSystem.Factory(trunk_file_system,
object_store_creator),
TestBranchUtility.CreateWithCannedData(), host_file_system_creator)
def _GetOrCreateGithubFileSystem():
# Initialising github is pointless if samples are disabled, since it's only
# used for apps samples.
if ServerInstance.github_file_system is None:
if _IsSamplesDisabled():
ServerInstance.github_file_system = EmptyDirFileSystem()
else:
ServerInstance.github_file_system = GithubFileSystem(
AppEngineUrlFetcher(url_constants.GITHUB_URL),
AppEngineBlobstore())
return ServerInstance.github_file_system
def Create(self, bucket):
'''Creates a CloudStorageFileSystemProvider.
|bucket| is the name of GCS bucket, eg devtools-docs. It is expected
that this bucket has Read permission for this app in its ACLs.
Optional configuration can be set in a local_debug/gcs_debug.conf file:
use_local_fs=True|False
remote_bucket_prefix=<prefix>
If running in Preview mode or in Development mode with use_local_fs set to
True, buckets and files are looked for inside the local_debug folder instead
of in the real GCS server.
'''
if IsTest():
return EmptyDirFileSystem()
debug_bucket_prefix = None
use_local_fs = False
if os.path.exists(LOCAL_GCS_DEBUG_CONF):
with open(LOCAL_GCS_DEBUG_CONF, "r") as token_file:
properties = dict(line.strip().split('=', 1)
for line in token_file)
use_local_fs = properties.get('use_local_fs', 'False') == 'True'
debug_bucket_prefix = properties.get('remote_bucket_prefix', None)
logging.debug('gcs: prefixing all bucket names with %s' %
debug_bucket_prefix)
if use_local_fs:
return LocalFileSystem(
ToDirectory(os.path.join(LOCAL_GCS_DIR, bucket)))
if debug_bucket_prefix:
bucket = debug_bucket_prefix + bucket
return CachingFileSystem(CloudStorageFileSystem(bucket),
self._object_store_creator)
def Create(self, owner, repo):
return EmptyDirFileSystem()
def __init__(self, object_store_creator, host_file_system,
app_samples_file_system, base_path, compiled_fs_factory,
branch_utility, host_file_system_creator):
self.object_store_creator = object_store_creator
self.host_file_system = host_file_system
self.app_samples_file_system = app_samples_file_system
self.compiled_host_fs_factory = compiled_fs_factory
self.host_file_system_creator = host_file_system_creator
self.availability_finder_factory = AvailabilityFinder.Factory(
object_store_creator, self.compiled_host_fs_factory,
branch_utility, host_file_system_creator)
self.api_list_data_source_factory = APIListDataSource.Factory(
self.compiled_host_fs_factory, self.host_file_system,
svn_constants.API_PATH, svn_constants.PUBLIC_TEMPLATE_PATH)
self.api_data_source_factory = APIDataSource.Factory(
self.compiled_host_fs_factory, svn_constants.API_PATH,
self.availability_finder_factory)
self.ref_resolver_factory = ReferenceResolver.Factory(
self.api_data_source_factory, self.api_list_data_source_factory,
object_store_creator)
self.api_data_source_factory.SetReferenceResolverFactory(
self.ref_resolver_factory)
# Note: samples are super slow in the dev server because it doesn't support
# async fetch, so disable them.
if IsDevServer():
extension_samples_fs = EmptyDirFileSystem()
else:
extension_samples_fs = self.host_file_system
self.samples_data_source_factory = SamplesDataSource.Factory(
extension_samples_fs,
CompiledFileSystem.Factory(extension_samples_fs,
object_store_creator),
self.app_samples_file_system,
CompiledFileSystem.Factory(self.app_samples_file_system,
object_store_creator),
self.ref_resolver_factory, svn_constants.EXAMPLES_PATH, base_path)
self.api_data_source_factory.SetSamplesDataSourceFactory(
self.samples_data_source_factory)
self.intro_data_source_factory = IntroDataSource.Factory(
self.compiled_host_fs_factory, self.ref_resolver_factory,
[svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH])
self.sidenav_data_source_factory = SidenavDataSource.Factory(
self.compiled_host_fs_factory, svn_constants.JSON_PATH)
self.manifest_data_source = ManifestDataSource(
self.compiled_host_fs_factory, host_file_system, '/'.join(
(svn_constants.JSON_PATH, 'manifest.json')), '/'.join(
(svn_constants.API_PATH, '_manifest_features.json')))
self.template_data_source_factory = TemplateDataSource.Factory(
self.api_data_source_factory, self.api_list_data_source_factory,
self.intro_data_source_factory, self.samples_data_source_factory,
self.sidenav_data_source_factory, self.compiled_host_fs_factory,
self.ref_resolver_factory, self.manifest_data_source,
svn_constants.PUBLIC_TEMPLATE_PATH,
svn_constants.PRIVATE_TEMPLATE_PATH, base_path)
self.api_data_source_factory.SetTemplateDataSource(
self.template_data_source_factory)
self.example_zipper = ExampleZipper(self.compiled_host_fs_factory,
self.host_file_system,
svn_constants.DOCS_PATH)
self.path_canonicalizer = PathCanonicalizer(
self.compiled_host_fs_factory)
self.redirector = Redirector(self.compiled_host_fs_factory,
self.host_file_system,
svn_constants.PUBLIC_TEMPLATE_PATH)
def __init__(self, channel, object_store_creator, host_file_system,
app_samples_file_system, base_path, compiled_fs_factory):
self.channel = channel
self.object_store_creator = object_store_creator
self.host_file_system = host_file_system
self.app_samples_file_system = app_samples_file_system
self.compiled_host_fs_factory = compiled_fs_factory
self.api_list_data_source_factory = APIListDataSource.Factory(
self.compiled_host_fs_factory, svn_constants.API_PATH,
svn_constants.PUBLIC_TEMPLATE_PATH)
self.api_data_source_factory = APIDataSource.Factory(
self.compiled_host_fs_factory, svn_constants.API_PATH)
self.ref_resolver_factory = ReferenceResolver.Factory(
self.api_data_source_factory, self.api_list_data_source_factory,
object_store_creator)
self.api_data_source_factory.SetReferenceResolverFactory(
self.ref_resolver_factory)
# Note: samples are super slow in the dev server because it doesn't support
# async fetch, so disable them.
if IsDevServer():
extension_samples_fs = EmptyDirFileSystem()
else:
extension_samples_fs = self.host_file_system
self.samples_data_source_factory = SamplesDataSource.Factory(
channel, extension_samples_fs,
CompiledFileSystem.Factory(extension_samples_fs,
object_store_creator),
self.app_samples_file_system,
CompiledFileSystem.Factory(self.app_samples_file_system,
object_store_creator),
self.ref_resolver_factory, svn_constants.EXAMPLES_PATH)
self.api_data_source_factory.SetSamplesDataSourceFactory(
self.samples_data_source_factory)
self.intro_data_source_factory = IntroDataSource.Factory(
self.compiled_host_fs_factory, self.ref_resolver_factory,
[svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH])
self.sidenav_data_source_factory = SidenavDataSource.Factory(
self.compiled_host_fs_factory, svn_constants.JSON_PATH, base_path)
self.template_data_source_factory = TemplateDataSource.Factory(
channel, self.api_data_source_factory,
self.api_list_data_source_factory, self.intro_data_source_factory,
self.samples_data_source_factory, self.sidenav_data_source_factory,
self.compiled_host_fs_factory, self.ref_resolver_factory,
svn_constants.PUBLIC_TEMPLATE_PATH,
svn_constants.PRIVATE_TEMPLATE_PATH, base_path)
self.example_zipper = ExampleZipper(self.compiled_host_fs_factory,
svn_constants.DOCS_PATH)
self.path_canonicalizer = PathCanonicalizer(
channel, self.compiled_host_fs_factory)
self.content_cache = self.compiled_host_fs_factory.CreateIdentity(
ServerInstance)
def CreateAppSamplesFileSystem(self, object_store_creator):
# TODO(kalman): CachingFileSystem wrapper for GithubFileSystem, but it's
# not supported yet (see comment there).
return (EmptyDirFileSystem() if IsDevServer() else
GithubFileSystem.Create(object_store_creator))
def ForTest(file_system):
object_store_creator = ObjectStoreCreator.ForTest()
return ServerInstance(
'test', object_store_creator, file_system, EmptyDirFileSystem(),
'', CompiledFileSystem.Factory(file_system, object_store_creator))
def __init__(self,
object_store_creator,
compiled_fs_factory,
branch_utility,
host_file_system_provider,
github_file_system_provider,
base_path='/'):
'''
|object_store_creator|
The ObjectStoreCreator used to create almost all caches.
|compiled_fs_factory|
Factory used to create CompiledFileSystems, a higher-level cache type
than ObjectStores. This can usually be derived from just
|object_store_creator| but under special circumstances a different
implementation needs to be passed in.
|branch_utility|
Has knowledge of Chrome branches, channels, and versions.
|host_file_system_provider|
Creates FileSystem instances which host the server at alternative
revisions.
|github_file_system_provider|
Creates FileSystem instances backed by GitHub.
|base_path|
The path which all HTML is generated relative to. Usually this is /
but some servlets need to override this.
'''
self.object_store_creator = object_store_creator
self.compiled_fs_factory = compiled_fs_factory
self.host_file_system_provider = host_file_system_provider
host_fs_at_trunk = host_file_system_provider.GetTrunk()
self.github_file_system_provider = github_file_system_provider
assert base_path.startswith('/') and base_path.endswith('/')
self.base_path = base_path
self.host_file_system_iterator = HostFileSystemIterator(
host_file_system_provider, branch_utility)
self.features_bundle = FeaturesBundle(host_fs_at_trunk,
self.compiled_fs_factory,
self.object_store_creator)
self.api_models = APIModels(self.features_bundle,
self.compiled_fs_factory, host_fs_at_trunk)
self.availability_finder = AvailabilityFinder(
branch_utility, compiled_fs_factory,
self.host_file_system_iterator, host_fs_at_trunk,
object_store_creator)
self.api_list_data_source_factory = APIListDataSource.Factory(
self.compiled_fs_factory, host_fs_at_trunk, self.features_bundle,
self.object_store_creator)
self.api_data_source_factory = APIDataSource.Factory(
self.compiled_fs_factory, host_fs_at_trunk, svn_constants.API_PATH,
self.availability_finder, branch_utility)
self.ref_resolver_factory = ReferenceResolver.Factory(
self.api_data_source_factory, self.api_models,
object_store_creator)
self.api_data_source_factory.SetReferenceResolverFactory(
self.ref_resolver_factory)
# Note: samples are super slow in the dev server because it doesn't support
# async fetch, so disable them.
if IsDevServer():
extension_samples_fs = EmptyDirFileSystem()
app_samples_fs = EmptyDirFileSystem()
else:
extension_samples_fs = host_fs_at_trunk
app_samples_fs = github_file_system_provider.Create(
'GoogleChrome', 'chrome-app-samples')
self.samples_data_source_factory = SamplesDataSource.Factory(
extension_samples_fs, app_samples_fs,
CompiledFileSystem.Factory(object_store_creator),
self.ref_resolver_factory, svn_constants.EXAMPLES_PATH, base_path)
self.api_data_source_factory.SetSamplesDataSourceFactory(
self.samples_data_source_factory)
self.intro_data_source_factory = IntroDataSource.Factory(
self.compiled_fs_factory, host_fs_at_trunk,
self.ref_resolver_factory,
[svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH])
self.path_canonicalizer = PathCanonicalizer(self.compiled_fs_factory,
host_fs_at_trunk)
self.content_providers = ContentProviders(
self.compiled_fs_factory, host_fs_at_trunk,
self.github_file_system_provider)
# TODO(kalman): Move all the remaining DataSources into DataSourceRegistry,
# then factor out the DataSource creation into a factory method, so that
# the entire ServerInstance doesn't need to be passed in here.
self.template_renderer = TemplateRenderer(self)
self.strings_json_path = svn_constants.STRINGS_JSON_PATH
self.manifest_json_path = svn_constants.MANIFEST_JSON_PATH
self.manifest_features_path = svn_constants.MANIFEST_FEATURES_PATH
def __init__(self,
object_store_creator,
compiled_fs_factory,
branch_utility,
host_file_system_provider,
github_file_system_provider,
gcs_file_system_provider,
base_path='/'):
'''
|object_store_creator|
The ObjectStoreCreator used to create almost all caches.
|compiled_fs_factory|
Factory used to create CompiledFileSystems, a higher-level cache type
than ObjectStores. This can usually be derived from just
|object_store_creator| but under special circumstances a different
implementation needs to be passed in.
|branch_utility|
Has knowledge of Chrome branches, channels, and versions.
|host_file_system_provider|
Creates FileSystem instances which host the server at alternative
revisions.
|github_file_system_provider|
Creates FileSystem instances backed by GitHub.
|base_path|
The path which all HTML is generated relative to. Usually this is /
but some servlets need to override this.
'''
self.object_store_creator = object_store_creator
self.compiled_fs_factory = compiled_fs_factory
self.host_file_system_provider = host_file_system_provider
host_fs_at_trunk = host_file_system_provider.GetTrunk()
self.github_file_system_provider = github_file_system_provider
self.gcs_file_system_provider = gcs_file_system_provider
assert base_path.startswith('/') and base_path.endswith('/')
self.base_path = base_path
self.host_file_system_iterator = HostFileSystemIterator(
host_file_system_provider,
branch_utility)
self.features_bundle = FeaturesBundle(
host_fs_at_trunk,
self.compiled_fs_factory,
self.object_store_creator)
self.api_models = APIModels(
self.features_bundle,
self.compiled_fs_factory,
host_fs_at_trunk)
self.availability_finder = AvailabilityFinder(
branch_utility,
compiled_fs_factory,
self.host_file_system_iterator,
host_fs_at_trunk,
object_store_creator)
self.api_categorizer = APICategorizer(
host_fs_at_trunk,
compiled_fs_factory)
self.api_data_source_factory = APIDataSource.Factory(
self.compiled_fs_factory,
host_fs_at_trunk,
self.availability_finder,
self.api_models,
self.features_bundle,
self.object_store_creator)
self.ref_resolver_factory = ReferenceResolver.Factory(
self.api_data_source_factory,
self.api_models,
object_store_creator)
self.api_data_source_factory.SetReferenceResolverFactory(
self.ref_resolver_factory)
# Note: samples are super slow in the dev server because it doesn't support
# async fetch, so disable them.
if IsDevServer():
extension_samples_fs = EmptyDirFileSystem()
app_samples_fs = EmptyDirFileSystem()
else:
extension_samples_fs = host_fs_at_trunk
# TODO(kalman): Re-enable the apps samples, see http://crbug.com/344097.
app_samples_fs = EmptyDirFileSystem()
#app_samples_fs = github_file_system_provider.Create(
# 'GoogleChrome', 'chrome-app-samples')
self.samples_data_source_factory = SamplesDataSource.Factory(
extension_samples_fs,
app_samples_fs,
CompiledFileSystem.Factory(object_store_creator),
self.ref_resolver_factory,
base_path)
self.api_data_source_factory.SetSamplesDataSourceFactory(
self.samples_data_source_factory)
self.content_providers = ContentProviders(
object_store_creator,
self.compiled_fs_factory,
host_fs_at_trunk,
self.github_file_system_provider,
self.gcs_file_system_provider)
# TODO(kalman): Move all the remaining DataSources into DataSourceRegistry,
# then factor out the DataSource creation into a factory method, so that
# the entire ServerInstance doesn't need to be passed in here.
self.template_renderer = TemplateRenderer(self)
# TODO(kalman): It may be better for |document_renderer| to construct a
# TemplateDataSource itself rather than depending on template_renderer, but
# for that the above todo should be addressed.
self.document_renderer = DocumentRenderer(
TableOfContentsRenderer(host_fs_at_trunk,
compiled_fs_factory,
self.template_renderer),
self.ref_resolver_factory.Create())
def CreateAppSamplesFileSystem(self, object_store_creator):
return EmptyDirFileSystem()
def CreateAppSamplesFileSystem(self, object_store_creator):
# TODO(kalman): OfflineServerInstance wrapper for GithubFileSystem, but
# the cron job doesn't crawl the samples yet.
return (EmptyDirFileSystem() if IsDevServer() else
GithubFileSystem.Create(object_store_creator))
def Create(self, bucket):
'''Creates a CloudStorageFileSystemProvider.
|bucket| is the name of GCS bucket, eg devtools-docs. It is expected
that this bucket has Read permission for this app in its ACLs.
Optional configuration can be set in a local_debug/gcs_debug.conf file:
use_local_fs=True|False
access_token=<token>
remote_bucket_prefix=<prefix>
If running in Preview mode or in Development mode with use_local_fs set to
True, buckets and files are looked inside the local_debug folder instead
of in the real GCS server. Preview server does not support direct GCS
access, so it is always forced to use a LocalFileSystem.
For real GCS access in the Development mode (dev_appserver.py),
access_token and remote_bucket_prefix options can be
used to change the way GCS files are accessed. Both are ignored in a real
appengine instance.
"access_token" is always REQUIRED on dev_appengine, otherwise you will
get 404 (auth) errors. You can get one access_token valid for a few minutes
by typing:
gsutil -d ls 2>&1 | grep "Bearer" |
sed "s/.*Bearer \(.*\).r.nUser-Agent.*/access_token=\1/" )"
A sample output would be:
access_token=ya29.1.AADtN_VW5ibbfLHV5cMIK5ss4bHtVzBXpa4byjd
Now add this line to the local_debug/gcs_debug.conf file and restart the
appengine development server.
Remember that you will need a new access_token every ten minutes or
so. If you get 404 errors on log, update it. Access token is not
used for a deployed appengine app, only if you use dev_appengine.py.
remote_bucket_prefix is useful if you want to test on your own GCS buckets
before using the real GCS buckets.
'''
if not environment.IsReleaseServer() and not environment.IsDevServer():
bucket_local_path = os.path.join(LOCAL_GCS_DIR, bucket)
if IsDirectory(bucket_local_path):
return LocalFileSystem(bucket_local_path)
else:
return EmptyDirFileSystem()
debug_access_token = None
debug_bucket_prefix = None
use_local_fs = False
if environment.IsDevServer() and os.path.exists(LOCAL_GCS_DEBUG_CONF):
with open(LOCAL_GCS_DEBUG_CONF, "r") as token_file:
properties = dict(line.strip().split('=', 1)
for line in token_file)
use_local_fs = properties.get('use_local_fs', 'False') == 'True'
debug_access_token = properties.get('access_token', None)
debug_bucket_prefix = properties.get('remote_bucket_prefix', None)
if environment.IsDevServer() and use_local_fs:
return LocalFileSystem(os.path.join(LOCAL_GCS_DIR, bucket))
# gcs_file_system has strong dependencies on runtime appengine APIs,
# so we only import it when we are sure we are not on preview.py or tests.
from gcs_file_system import CloudStorageFileSystem
return CachingFileSystem(
CloudStorageFileSystem(bucket, debug_access_token,
debug_bucket_prefix),
self._object_store_creator)
def Create(self, bucket):
return EmptyDirFileSystem()
