def GetSamplesModel(self, platform): if self._platform_data[platform].samples_model is None: # Note: samples are super slow in the dev server because it doesn't # support async fetch, so disable them. if IsDevServer(): extension_samples_fs = EmptyDirFileSystem() app_samples_fs = EmptyDirFileSystem() else: extension_samples_fs = self._host_fs_at_master # TODO(kalman): Re-enable the apps samples, see http://crbug.com/344097. app_samples_fs = EmptyDirFileSystem() self._platform_data[platform].samples_model = SamplesModel( extension_samples_fs, app_samples_fs, self._compiled_fs_factory, self.GetReferenceResolver(platform), self._base_path, platform) return self._platform_data[platform].samples_model
def ForTest(file_system): object_store_creator = ObjectStoreCreator.ForTest() return ServerInstance( object_store_creator, file_system, EmptyDirFileSystem(), '', CompiledFileSystem.Factory(file_system, object_store_creator), TestBranchUtility.CreateWithCannedData(), HostFileSystemCreator.ForTest(file_system, object_store_creator))
def __init__(self, channel, object_store_creator_factory, svn_file_system, github_file_system): self.svn_file_system = svn_file_system self.github_file_system = github_file_system self.compiled_fs_factory = CompiledFileSystem.Factory( svn_file_system, object_store_creator_factory) self.api_list_data_source_factory = APIListDataSource.Factory( self.compiled_fs_factory, svn_constants.API_PATH, svn_constants.PUBLIC_TEMPLATE_PATH) self.api_data_source_factory = APIDataSource.Factory( self.compiled_fs_factory, svn_constants.API_PATH) self.ref_resolver_factory = ReferenceResolver.Factory( self.api_data_source_factory, self.api_list_data_source_factory, object_store_creator_factory) self.api_data_source_factory.SetReferenceResolverFactory( self.ref_resolver_factory) # Note: samples are super slow in the dev server because it doesn't support # async fetch, so disable them. If you actually want to test samples, then # good luck, and modify _IsSamplesDisabled at the top. if _IsSamplesDisabled(): svn_fs_for_samples = EmptyDirFileSystem() else: svn_fs_for_samples = self.svn_file_system self.samples_data_source_factory = SamplesDataSource.Factory( channel, svn_fs_for_samples, self.github_file_system, self.ref_resolver_factory, object_store_creator_factory, svn_constants.EXAMPLES_PATH) self.api_data_source_factory.SetSamplesDataSourceFactory( self.samples_data_source_factory) self.intro_data_source_factory = IntroDataSource.Factory( self.compiled_fs_factory, self.ref_resolver_factory, [svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH]) self.sidenav_data_source_factory = SidenavDataSource.Factory( self.compiled_fs_factory, svn_constants.JSON_PATH) self.template_data_source_factory = TemplateDataSource.Factory( channel, self.api_data_source_factory, self.api_list_data_source_factory, self.intro_data_source_factory, self.samples_data_source_factory, self.sidenav_data_source_factory, self.compiled_fs_factory, self.ref_resolver_factory, svn_constants.PUBLIC_TEMPLATE_PATH, svn_constants.PRIVATE_TEMPLATE_PATH) self.example_zipper = ExampleZipper(self.compiled_fs_factory, svn_constants.DOCS_PATH) self.path_canonicalizer = PathCanonicalizer(channel, self.compiled_fs_factory) self.content_cache = self.compiled_fs_factory.GetOrCreateIdentity()
def ForLocal(): channel = 'trunk' object_store_creator = ObjectStoreCreator(channel, start_empty=False, store_type=TestObjectStore) file_system = CachingFileSystem(LocalFileSystem.Create(), object_store_creator) return ServerInstance( channel, object_store_creator, file_system, EmptyDirFileSystem(), '', CompiledFileSystem.Factory(file_system, object_store_creator))
def ForLocal(): object_store_creator = ObjectStoreCreator(start_empty=False, store_type=TestObjectStore) host_file_system_creator = HostFileSystemCreator.ForLocal( object_store_creator) trunk_file_system = host_file_system_creator.Create() return ServerInstance( object_store_creator, trunk_file_system, EmptyDirFileSystem(), '', CompiledFileSystem.Factory(trunk_file_system, object_store_creator), TestBranchUtility.CreateWithCannedData(), host_file_system_creator)
def _GetOrCreateGithubFileSystem(): # Initialising github is pointless if samples are disabled, since it's only # used for apps samples. if ServerInstance.github_file_system is None: if _IsSamplesDisabled(): ServerInstance.github_file_system = EmptyDirFileSystem() else: ServerInstance.github_file_system = GithubFileSystem( AppEngineUrlFetcher(url_constants.GITHUB_URL), AppEngineBlobstore()) return ServerInstance.github_file_system
def Create(self, bucket): '''Creates a CloudStorageFileSystemProvider. |bucket| is the name of GCS bucket, eg devtools-docs. It is expected that this bucket has Read permission for this app in its ACLs. Optional configuration can be set in a local_debug/gcs_debug.conf file: use_local_fs=True|False remote_bucket_prefix=<prefix> If running in Preview mode or in Development mode with use_local_fs set to True, buckets and files are looked for inside the local_debug folder instead of in the real GCS server. ''' if IsTest(): return EmptyDirFileSystem() debug_bucket_prefix = None use_local_fs = False if os.path.exists(LOCAL_GCS_DEBUG_CONF): with open(LOCAL_GCS_DEBUG_CONF, "r") as token_file: properties = dict(line.strip().split('=', 1) for line in token_file) use_local_fs = properties.get('use_local_fs', 'False') == 'True' debug_bucket_prefix = properties.get('remote_bucket_prefix', None) logging.debug('gcs: prefixing all bucket names with %s' % debug_bucket_prefix) if use_local_fs: return LocalFileSystem( ToDirectory(os.path.join(LOCAL_GCS_DIR, bucket))) if debug_bucket_prefix: bucket = debug_bucket_prefix + bucket return CachingFileSystem(CloudStorageFileSystem(bucket), self._object_store_creator)
def Create(self, owner, repo): return EmptyDirFileSystem()
def __init__(self, object_store_creator, host_file_system, app_samples_file_system, base_path, compiled_fs_factory, branch_utility, host_file_system_creator): self.object_store_creator = object_store_creator self.host_file_system = host_file_system self.app_samples_file_system = app_samples_file_system self.compiled_host_fs_factory = compiled_fs_factory self.host_file_system_creator = host_file_system_creator self.availability_finder_factory = AvailabilityFinder.Factory( object_store_creator, self.compiled_host_fs_factory, branch_utility, host_file_system_creator) self.api_list_data_source_factory = APIListDataSource.Factory( self.compiled_host_fs_factory, self.host_file_system, svn_constants.API_PATH, svn_constants.PUBLIC_TEMPLATE_PATH) self.api_data_source_factory = APIDataSource.Factory( self.compiled_host_fs_factory, svn_constants.API_PATH, self.availability_finder_factory) self.ref_resolver_factory = ReferenceResolver.Factory( self.api_data_source_factory, self.api_list_data_source_factory, object_store_creator) self.api_data_source_factory.SetReferenceResolverFactory( self.ref_resolver_factory) # Note: samples are super slow in the dev server because it doesn't support # async fetch, so disable them. if IsDevServer(): extension_samples_fs = EmptyDirFileSystem() else: extension_samples_fs = self.host_file_system self.samples_data_source_factory = SamplesDataSource.Factory( extension_samples_fs, CompiledFileSystem.Factory(extension_samples_fs, object_store_creator), self.app_samples_file_system, CompiledFileSystem.Factory(self.app_samples_file_system, object_store_creator), self.ref_resolver_factory, svn_constants.EXAMPLES_PATH, base_path) self.api_data_source_factory.SetSamplesDataSourceFactory( self.samples_data_source_factory) self.intro_data_source_factory = IntroDataSource.Factory( self.compiled_host_fs_factory, self.ref_resolver_factory, [svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH]) self.sidenav_data_source_factory = SidenavDataSource.Factory( self.compiled_host_fs_factory, svn_constants.JSON_PATH) self.manifest_data_source = ManifestDataSource( self.compiled_host_fs_factory, host_file_system, '/'.join( (svn_constants.JSON_PATH, 'manifest.json')), '/'.join( (svn_constants.API_PATH, '_manifest_features.json'))) self.template_data_source_factory = TemplateDataSource.Factory( self.api_data_source_factory, self.api_list_data_source_factory, self.intro_data_source_factory, self.samples_data_source_factory, self.sidenav_data_source_factory, self.compiled_host_fs_factory, self.ref_resolver_factory, self.manifest_data_source, svn_constants.PUBLIC_TEMPLATE_PATH, svn_constants.PRIVATE_TEMPLATE_PATH, base_path) self.api_data_source_factory.SetTemplateDataSource( self.template_data_source_factory) self.example_zipper = ExampleZipper(self.compiled_host_fs_factory, self.host_file_system, svn_constants.DOCS_PATH) self.path_canonicalizer = PathCanonicalizer( self.compiled_host_fs_factory) self.redirector = Redirector(self.compiled_host_fs_factory, self.host_file_system, svn_constants.PUBLIC_TEMPLATE_PATH)
def __init__(self, channel, object_store_creator, host_file_system, app_samples_file_system, base_path, compiled_fs_factory): self.channel = channel self.object_store_creator = object_store_creator self.host_file_system = host_file_system self.app_samples_file_system = app_samples_file_system self.compiled_host_fs_factory = compiled_fs_factory self.api_list_data_source_factory = APIListDataSource.Factory( self.compiled_host_fs_factory, svn_constants.API_PATH, svn_constants.PUBLIC_TEMPLATE_PATH) self.api_data_source_factory = APIDataSource.Factory( self.compiled_host_fs_factory, svn_constants.API_PATH) self.ref_resolver_factory = ReferenceResolver.Factory( self.api_data_source_factory, self.api_list_data_source_factory, object_store_creator) self.api_data_source_factory.SetReferenceResolverFactory( self.ref_resolver_factory) # Note: samples are super slow in the dev server because it doesn't support # async fetch, so disable them. if IsDevServer(): extension_samples_fs = EmptyDirFileSystem() else: extension_samples_fs = self.host_file_system self.samples_data_source_factory = SamplesDataSource.Factory( channel, extension_samples_fs, CompiledFileSystem.Factory(extension_samples_fs, object_store_creator), self.app_samples_file_system, CompiledFileSystem.Factory(self.app_samples_file_system, object_store_creator), self.ref_resolver_factory, svn_constants.EXAMPLES_PATH) self.api_data_source_factory.SetSamplesDataSourceFactory( self.samples_data_source_factory) self.intro_data_source_factory = IntroDataSource.Factory( self.compiled_host_fs_factory, self.ref_resolver_factory, [svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH]) self.sidenav_data_source_factory = SidenavDataSource.Factory( self.compiled_host_fs_factory, svn_constants.JSON_PATH, base_path) self.template_data_source_factory = TemplateDataSource.Factory( channel, self.api_data_source_factory, self.api_list_data_source_factory, self.intro_data_source_factory, self.samples_data_source_factory, self.sidenav_data_source_factory, self.compiled_host_fs_factory, self.ref_resolver_factory, svn_constants.PUBLIC_TEMPLATE_PATH, svn_constants.PRIVATE_TEMPLATE_PATH, base_path) self.example_zipper = ExampleZipper(self.compiled_host_fs_factory, svn_constants.DOCS_PATH) self.path_canonicalizer = PathCanonicalizer( channel, self.compiled_host_fs_factory) self.content_cache = self.compiled_host_fs_factory.CreateIdentity( ServerInstance)
def CreateAppSamplesFileSystem(self, object_store_creator): # TODO(kalman): CachingFileSystem wrapper for GithubFileSystem, but it's # not supported yet (see comment there). return (EmptyDirFileSystem() if IsDevServer() else GithubFileSystem.Create(object_store_creator))
def ForTest(file_system): object_store_creator = ObjectStoreCreator.ForTest() return ServerInstance( 'test', object_store_creator, file_system, EmptyDirFileSystem(), '', CompiledFileSystem.Factory(file_system, object_store_creator))
def __init__(self, object_store_creator, compiled_fs_factory, branch_utility, host_file_system_provider, github_file_system_provider, base_path='/'): ''' |object_store_creator| The ObjectStoreCreator used to create almost all caches. |compiled_fs_factory| Factory used to create CompiledFileSystems, a higher-level cache type than ObjectStores. This can usually be derived from just |object_store_creator| but under special circumstances a different implementation needs to be passed in. |branch_utility| Has knowledge of Chrome branches, channels, and versions. |host_file_system_provider| Creates FileSystem instances which host the server at alternative revisions. |github_file_system_provider| Creates FileSystem instances backed by GitHub. |base_path| The path which all HTML is generated relative to. Usually this is / but some servlets need to override this. ''' self.object_store_creator = object_store_creator self.compiled_fs_factory = compiled_fs_factory self.host_file_system_provider = host_file_system_provider host_fs_at_trunk = host_file_system_provider.GetTrunk() self.github_file_system_provider = github_file_system_provider assert base_path.startswith('/') and base_path.endswith('/') self.base_path = base_path self.host_file_system_iterator = HostFileSystemIterator( host_file_system_provider, branch_utility) self.features_bundle = FeaturesBundle(host_fs_at_trunk, self.compiled_fs_factory, self.object_store_creator) self.api_models = APIModels(self.features_bundle, self.compiled_fs_factory, host_fs_at_trunk) self.availability_finder = AvailabilityFinder( branch_utility, compiled_fs_factory, self.host_file_system_iterator, host_fs_at_trunk, object_store_creator) self.api_list_data_source_factory = APIListDataSource.Factory( self.compiled_fs_factory, host_fs_at_trunk, self.features_bundle, self.object_store_creator) self.api_data_source_factory = APIDataSource.Factory( self.compiled_fs_factory, host_fs_at_trunk, svn_constants.API_PATH, self.availability_finder, branch_utility) self.ref_resolver_factory = ReferenceResolver.Factory( self.api_data_source_factory, self.api_models, object_store_creator) self.api_data_source_factory.SetReferenceResolverFactory( self.ref_resolver_factory) # Note: samples are super slow in the dev server because it doesn't support # async fetch, so disable them. if IsDevServer(): extension_samples_fs = EmptyDirFileSystem() app_samples_fs = EmptyDirFileSystem() else: extension_samples_fs = host_fs_at_trunk app_samples_fs = github_file_system_provider.Create( 'GoogleChrome', 'chrome-app-samples') self.samples_data_source_factory = SamplesDataSource.Factory( extension_samples_fs, app_samples_fs, CompiledFileSystem.Factory(object_store_creator), self.ref_resolver_factory, svn_constants.EXAMPLES_PATH, base_path) self.api_data_source_factory.SetSamplesDataSourceFactory( self.samples_data_source_factory) self.intro_data_source_factory = IntroDataSource.Factory( self.compiled_fs_factory, host_fs_at_trunk, self.ref_resolver_factory, [svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH]) self.path_canonicalizer = PathCanonicalizer(self.compiled_fs_factory, host_fs_at_trunk) self.content_providers = ContentProviders( self.compiled_fs_factory, host_fs_at_trunk, self.github_file_system_provider) # TODO(kalman): Move all the remaining DataSources into DataSourceRegistry, # then factor out the DataSource creation into a factory method, so that # the entire ServerInstance doesn't need to be passed in here. self.template_renderer = TemplateRenderer(self) self.strings_json_path = svn_constants.STRINGS_JSON_PATH self.manifest_json_path = svn_constants.MANIFEST_JSON_PATH self.manifest_features_path = svn_constants.MANIFEST_FEATURES_PATH
def __init__(self, object_store_creator, compiled_fs_factory, branch_utility, host_file_system_provider, github_file_system_provider, gcs_file_system_provider, base_path='/'): ''' |object_store_creator| The ObjectStoreCreator used to create almost all caches. |compiled_fs_factory| Factory used to create CompiledFileSystems, a higher-level cache type than ObjectStores. This can usually be derived from just |object_store_creator| but under special circumstances a different implementation needs to be passed in. |branch_utility| Has knowledge of Chrome branches, channels, and versions. |host_file_system_provider| Creates FileSystem instances which host the server at alternative revisions. |github_file_system_provider| Creates FileSystem instances backed by GitHub. |base_path| The path which all HTML is generated relative to. Usually this is / but some servlets need to override this. ''' self.object_store_creator = object_store_creator self.compiled_fs_factory = compiled_fs_factory self.host_file_system_provider = host_file_system_provider host_fs_at_trunk = host_file_system_provider.GetTrunk() self.github_file_system_provider = github_file_system_provider self.gcs_file_system_provider = gcs_file_system_provider assert base_path.startswith('/') and base_path.endswith('/') self.base_path = base_path self.host_file_system_iterator = HostFileSystemIterator( host_file_system_provider, branch_utility) self.features_bundle = FeaturesBundle( host_fs_at_trunk, self.compiled_fs_factory, self.object_store_creator) self.api_models = APIModels( self.features_bundle, self.compiled_fs_factory, host_fs_at_trunk) self.availability_finder = AvailabilityFinder( branch_utility, compiled_fs_factory, self.host_file_system_iterator, host_fs_at_trunk, object_store_creator) self.api_categorizer = APICategorizer( host_fs_at_trunk, compiled_fs_factory) self.api_data_source_factory = APIDataSource.Factory( self.compiled_fs_factory, host_fs_at_trunk, self.availability_finder, self.api_models, self.features_bundle, self.object_store_creator) self.ref_resolver_factory = ReferenceResolver.Factory( self.api_data_source_factory, self.api_models, object_store_creator) self.api_data_source_factory.SetReferenceResolverFactory( self.ref_resolver_factory) # Note: samples are super slow in the dev server because it doesn't support # async fetch, so disable them. if IsDevServer(): extension_samples_fs = EmptyDirFileSystem() app_samples_fs = EmptyDirFileSystem() else: extension_samples_fs = host_fs_at_trunk # TODO(kalman): Re-enable the apps samples, see http://crbug.com/344097. app_samples_fs = EmptyDirFileSystem() #app_samples_fs = github_file_system_provider.Create( # 'GoogleChrome', 'chrome-app-samples') self.samples_data_source_factory = SamplesDataSource.Factory( extension_samples_fs, app_samples_fs, CompiledFileSystem.Factory(object_store_creator), self.ref_resolver_factory, base_path) self.api_data_source_factory.SetSamplesDataSourceFactory( self.samples_data_source_factory) self.content_providers = ContentProviders( object_store_creator, self.compiled_fs_factory, host_fs_at_trunk, self.github_file_system_provider, self.gcs_file_system_provider) # TODO(kalman): Move all the remaining DataSources into DataSourceRegistry, # then factor out the DataSource creation into a factory method, so that # the entire ServerInstance doesn't need to be passed in here. self.template_renderer = TemplateRenderer(self) # TODO(kalman): It may be better for |document_renderer| to construct a # TemplateDataSource itself rather than depending on template_renderer, but # for that the above todo should be addressed. self.document_renderer = DocumentRenderer( TableOfContentsRenderer(host_fs_at_trunk, compiled_fs_factory, self.template_renderer), self.ref_resolver_factory.Create())
def CreateAppSamplesFileSystem(self, object_store_creator): return EmptyDirFileSystem()
def CreateAppSamplesFileSystem(self, object_store_creator): # TODO(kalman): OfflineServerInstance wrapper for GithubFileSystem, but # the cron job doesn't crawl the samples yet. return (EmptyDirFileSystem() if IsDevServer() else GithubFileSystem.Create(object_store_creator))
def Create(self, bucket): '''Creates a CloudStorageFileSystemProvider. |bucket| is the name of GCS bucket, eg devtools-docs. It is expected that this bucket has Read permission for this app in its ACLs. Optional configuration can be set in a local_debug/gcs_debug.conf file: use_local_fs=True|False access_token=<token> remote_bucket_prefix=<prefix> If running in Preview mode or in Development mode with use_local_fs set to True, buckets and files are looked inside the local_debug folder instead of in the real GCS server. Preview server does not support direct GCS access, so it is always forced to use a LocalFileSystem. For real GCS access in the Development mode (dev_appserver.py), access_token and remote_bucket_prefix options can be used to change the way GCS files are accessed. Both are ignored in a real appengine instance. "access_token" is always REQUIRED on dev_appengine, otherwise you will get 404 (auth) errors. You can get one access_token valid for a few minutes by typing: gsutil -d ls 2>&1 | grep "Bearer" | sed "s/.*Bearer \(.*\).r.nUser-Agent.*/access_token=\1/" )" A sample output would be: access_token=ya29.1.AADtN_VW5ibbfLHV5cMIK5ss4bHtVzBXpa4byjd Now add this line to the local_debug/gcs_debug.conf file and restart the appengine development server. Remember that you will need a new access_token every ten minutes or so. If you get 404 errors on log, update it. Access token is not used for a deployed appengine app, only if you use dev_appengine.py. remote_bucket_prefix is useful if you want to test on your own GCS buckets before using the real GCS buckets. ''' if not environment.IsReleaseServer() and not environment.IsDevServer(): bucket_local_path = os.path.join(LOCAL_GCS_DIR, bucket) if IsDirectory(bucket_local_path): return LocalFileSystem(bucket_local_path) else: return EmptyDirFileSystem() debug_access_token = None debug_bucket_prefix = None use_local_fs = False if environment.IsDevServer() and os.path.exists(LOCAL_GCS_DEBUG_CONF): with open(LOCAL_GCS_DEBUG_CONF, "r") as token_file: properties = dict(line.strip().split('=', 1) for line in token_file) use_local_fs = properties.get('use_local_fs', 'False') == 'True' debug_access_token = properties.get('access_token', None) debug_bucket_prefix = properties.get('remote_bucket_prefix', None) if environment.IsDevServer() and use_local_fs: return LocalFileSystem(os.path.join(LOCAL_GCS_DIR, bucket)) # gcs_file_system has strong dependencies on runtime appengine APIs, # so we only import it when we are sure we are not on preview.py or tests. from gcs_file_system import CloudStorageFileSystem return CachingFileSystem( CloudStorageFileSystem(bucket, debug_access_token, debug_bucket_prefix), self._object_store_creator)
def Create(self, bucket): return EmptyDirFileSystem()