def __init__(self, channel, object_store_creator_factory, svn_file_system,
github_file_system):
self.svn_file_system = svn_file_system
self.github_file_system = github_file_system
self.compiled_fs_factory = CompiledFileSystem.Factory(
svn_file_system, object_store_creator_factory)
self.api_list_data_source_factory = APIListDataSource.Factory(
self.compiled_fs_factory, svn_constants.API_PATH,
svn_constants.PUBLIC_TEMPLATE_PATH)
self.api_data_source_factory = APIDataSource.Factory(
self.compiled_fs_factory, svn_constants.API_PATH)
self.ref_resolver_factory = ReferenceResolver.Factory(
self.api_data_source_factory, self.api_list_data_source_factory,
object_store_creator_factory)
self.api_data_source_factory.SetReferenceResolverFactory(
self.ref_resolver_factory)
# Note: samples are super slow in the dev server because it doesn't support
# async fetch, so disable them. If you actually want to test samples, then
# good luck, and modify _IsSamplesDisabled at the top.
if _IsSamplesDisabled():
svn_fs_for_samples = EmptyDirFileSystem()
else:
svn_fs_for_samples = self.svn_file_system
self.samples_data_source_factory = SamplesDataSource.Factory(
channel, svn_fs_for_samples, self.github_file_system,
self.ref_resolver_factory, object_store_creator_factory,
svn_constants.EXAMPLES_PATH)
self.api_data_source_factory.SetSamplesDataSourceFactory(
self.samples_data_source_factory)
self.intro_data_source_factory = IntroDataSource.Factory(
self.compiled_fs_factory, self.ref_resolver_factory,
[svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH])
self.sidenav_data_source_factory = SidenavDataSource.Factory(
self.compiled_fs_factory, svn_constants.JSON_PATH)
self.template_data_source_factory = TemplateDataSource.Factory(
channel, self.api_data_source_factory,
self.api_list_data_source_factory, self.intro_data_source_factory,
self.samples_data_source_factory, self.sidenav_data_source_factory,
self.compiled_fs_factory, self.ref_resolver_factory,
svn_constants.PUBLIC_TEMPLATE_PATH,
svn_constants.PRIVATE_TEMPLATE_PATH)
self.example_zipper = ExampleZipper(self.compiled_fs_factory,
svn_constants.DOCS_PATH)
self.path_canonicalizer = PathCanonicalizer(channel,
self.compiled_fs_factory)
self.content_cache = self.compiled_fs_factory.GetOrCreateIdentity()
def setUp(self):
object_store_creator = ObjectStoreCreator.ForTest()
self._file_system = CachingFileSystem(
LocalFileSystem(os.path.join(sys.path[0], 'test_data')),
object_store_creator)
self._example_zipper = ExampleZipper(
CompiledFileSystem.Factory(self._file_system,
object_store_creator), 'example_zipper')
def setUp(self):
object_store = InMemoryObjectStore('')
self._file_system = MemcacheFileSystem(
LocalFileSystem(os.path.join(sys.path[0], 'test_data')),
object_store)
self._example_zipper = ExampleZipper(
self._file_system,
CompiledFileSystem.Factory(self._file_system, object_store),
'example_zipper')
def __init__(self, channel, object_store_creator_factory, svn_file_system,
github_file_system):
self.svn_file_system = svn_file_system
self.github_file_system = github_file_system
self.compiled_fs_factory = CompiledFileSystem.Factory(
svn_file_system, object_store_creator_factory)
self.api_list_data_source_factory = APIListDataSource.Factory(
self.compiled_fs_factory, svn_constants.API_PATH,
svn_constants.PUBLIC_TEMPLATE_PATH)
self.api_data_source_factory = APIDataSource.Factory(
self.compiled_fs_factory, svn_constants.API_PATH)
self.ref_resolver_factory = ReferenceResolver.Factory(
self.api_data_source_factory, self.api_list_data_source_factory,
object_store_creator_factory)
self.api_data_source_factory.SetReferenceResolverFactory(
self.ref_resolver_factory)
self.samples_data_source_factory = SamplesDataSource.Factory(
channel, self.svn_file_system, ServerInstance.github_file_system,
self.ref_resolver_factory, object_store_creator_factory,
svn_constants.EXAMPLES_PATH)
self.api_data_source_factory.SetSamplesDataSourceFactory(
self.samples_data_source_factory)
self.intro_data_source_factory = IntroDataSource.Factory(
self.compiled_fs_factory, self.ref_resolver_factory,
[svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH])
self.sidenav_data_source_factory = SidenavDataSource.Factory(
self.compiled_fs_factory, svn_constants.JSON_PATH)
self.template_data_source_factory = TemplateDataSource.Factory(
channel, self.api_data_source_factory,
self.api_list_data_source_factory, self.intro_data_source_factory,
self.samples_data_source_factory, self.sidenav_data_source_factory,
self.compiled_fs_factory, self.ref_resolver_factory,
svn_constants.PUBLIC_TEMPLATE_PATH,
svn_constants.PRIVATE_TEMPLATE_PATH)
self.example_zipper = ExampleZipper(self.compiled_fs_factory,
svn_constants.DOCS_PATH)
self.path_canonicalizer = PathCanonicalizer(channel,
self.compiled_fs_factory)
self.content_cache = self.compiled_fs_factory.GetOrCreateIdentity()
def _GetInstanceForBranch(self, branch):
if branch in SERVER_INSTANCES:
return SERVER_INSTANCES[branch]
if branch == 'local':
fetcher = LocalFetcher(EXTENSIONS_PATH)
# No cache for local doc server.
cache_timeout_seconds = 0
else:
fetcher = SubversionFetcher(branch, EXTENSIONS_PATH, urlfetch)
cache_timeout_seconds = 300
cache_builder = FetcherCache.Builder(fetcher, cache_timeout_seconds)
api_data_source = APIDataSource(cache_builder, API_PATH)
intro_data_source = IntroDataSource(cache_builder,
[INTRO_PATH, ARTICLE_PATH])
samples_data_source = SamplesDataSource(fetcher, cache_builder,
EXAMPLES_PATH)
template_data_source = TemplateDataSource(
branch, api_data_source, intro_data_source, samples_data_source,
cache_builder, [PUBLIC_TEMPLATE_PATH, PRIVATE_TEMPLATE_PATH])
example_zipper = ExampleZipper(fetcher, cache_builder, DOCS_PATH,
EXAMPLES_PATH)
SERVER_INSTANCES[branch] = ServerInstance(template_data_source,
example_zipper,
cache_builder)
return SERVER_INSTANCES[branch]
def _GetInstanceForBranch(channel_name, local_path):
branch = BRANCH_UTILITY.GetBranchNumberForChannelName(channel_name)
# The key for the server is a tuple of |channel_name| with |branch|, since
# sometimes stable and beta point to the same branch.
instance_key = _MakeInstanceKey(channel_name, branch)
instance = SERVER_INSTANCES.get(instance_key, None)
if instance is not None:
return instance
branch_memcache = InMemoryObjectStore(branch)
if branch == 'local':
file_system = LocalFileSystem(local_path)
else:
file_system = _CreateMemcacheFileSystem(branch, branch_memcache)
cache_factory = CompiledFileSystem.Factory(file_system, branch_memcache)
api_list_data_source_factory = APIListDataSource.Factory(cache_factory,
file_system,
API_PATH,
PUBLIC_TEMPLATE_PATH)
intro_data_source_factory = IntroDataSource.Factory(
cache_factory,
[INTRO_PATH, ARTICLE_PATH])
samples_data_source_factory = SamplesDataSource.Factory(
channel_name,
file_system,
GITHUB_FILE_SYSTEM,
cache_factory,
GITHUB_COMPILED_FILE_SYSTEM,
api_list_data_source_factory,
EXAMPLES_PATH)
api_data_source_factory = APIDataSource.Factory(cache_factory,
API_PATH,
samples_data_source_factory)
template_data_source_factory = TemplateDataSource.Factory(
channel_name,
api_data_source_factory,
api_list_data_source_factory,
intro_data_source_factory,
samples_data_source_factory,
KNOWN_ISSUES_DATA_SOURCE,
cache_factory,
PUBLIC_TEMPLATE_PATH,
PRIVATE_TEMPLATE_PATH)
example_zipper = ExampleZipper(file_system,
cache_factory,
DOCS_PATH)
instance = ServerInstance(template_data_source_factory,
example_zipper,
cache_factory)
SERVER_INSTANCES[instance_key] = instance
return instance
class ExampleZipperTest(unittest.TestCase):
def setUp(self):
object_store_creator = ObjectStoreCreator.ForTest()
self._file_system = CachingFileSystem(
LocalFileSystem(os.path.join(sys.path[0], 'test_data')),
object_store_creator)
self._example_zipper = ExampleZipper(
CompiledFileSystem.Factory(self._file_system,
object_store_creator), 'example_zipper')
def testCreateZip(self):
# Cache manifest.json as unicode and make sure ExampleZipper doesn't error.
self._file_system.ReadSingle('example_zipper/basic/manifest.json')
self.assertTrue(len(self._example_zipper.Create('basic')) > 0)
class ServerInstance(object):
# Lazily create so we don't create github file systems unnecessarily in
# tests.
branch_utility = None
github_file_system = None
@staticmethod
@memoize
def GetOrCreateOffline(channel):
'''Gets/creates a local ServerInstance, meaning that only resources local to
the server - memcache, object store, etc, are queried. This amounts to not
setting up the subversion nor github file systems.
'''
branch_utility = ServerInstance._GetOrCreateBranchUtility()
branch = branch_utility.GetBranchNumberForChannelName(channel)
object_store_creator_factory = ObjectStoreCreator.Factory(
GetAppVersion(), branch)
# No svn nor github file systems. Rely on the crons to fill the caches, and
# for the caches to exist.
return ServerInstance(
channel,
object_store_creator_factory,
CachingFileSystem(OfflineFileSystem(SubversionFileSystem),
object_store_creator_factory),
# TODO(kalman): convert GithubFileSystem to be wrappable in a
# CachingFileSystem so that it can be replaced with an
# OfflineFileSystem. Currently GFS doesn't set the child versions of
# stat requests so it doesn't.
ServerInstance._GetOrCreateGithubFileSystem())
@staticmethod
def CreateOnline(channel):
'''Creates/creates an online server instance, meaning that both local and
subversion/github resources are queried.
'''
branch_utility = ServerInstance._GetOrCreateBranchUtility()
branch = branch_utility.GetBranchNumberForChannelName(channel)
if branch == 'trunk':
svn_url = '/'.join((url_constants.SVN_TRUNK_URL, 'src',
svn_constants.EXTENSIONS_PATH))
else:
svn_url = '/'.join((url_constants.SVN_BRANCH_URL, branch, 'src',
svn_constants.EXTENSIONS_PATH))
viewvc_url = svn_url.replace(url_constants.SVN_URL,
url_constants.VIEWVC_URL)
object_store_creator_factory = ObjectStoreCreator.Factory(
GetAppVersion(), branch, start_empty=True)
svn_file_system = CachingFileSystem(
SubversionFileSystem(AppEngineUrlFetcher(svn_url),
AppEngineUrlFetcher(viewvc_url)),
object_store_creator_factory)
return ServerInstance(channel, object_store_creator_factory,
svn_file_system,
ServerInstance._GetOrCreateGithubFileSystem())
@staticmethod
def CreateForTest(file_system):
return ServerInstance('test', ObjectStoreCreator.TestFactory(),
file_system, None)
@staticmethod
def _GetOrCreateBranchUtility():
if ServerInstance.branch_utility is None:
ServerInstance.branch_utility = BranchUtility(
url_constants.OMAHA_PROXY_URL, AppEngineUrlFetcher())
return ServerInstance.branch_utility
@staticmethod
def _GetOrCreateGithubFileSystem():
# Initialising github is pointless if samples are disabled, since it's only
# used for apps samples.
if ServerInstance.github_file_system is None:
if _IsSamplesDisabled():
ServerInstance.github_file_system = EmptyDirFileSystem()
else:
ServerInstance.github_file_system = GithubFileSystem(
AppEngineUrlFetcher(url_constants.GITHUB_URL),
AppEngineBlobstore())
return ServerInstance.github_file_system
def __init__(self, channel, object_store_creator_factory, svn_file_system,
github_file_system):
self.svn_file_system = svn_file_system
self.github_file_system = github_file_system
self.compiled_fs_factory = CompiledFileSystem.Factory(
svn_file_system, object_store_creator_factory)
self.api_list_data_source_factory = APIListDataSource.Factory(
self.compiled_fs_factory, svn_constants.API_PATH,
svn_constants.PUBLIC_TEMPLATE_PATH)
self.api_data_source_factory = APIDataSource.Factory(
self.compiled_fs_factory, svn_constants.API_PATH)
self.ref_resolver_factory = ReferenceResolver.Factory(
self.api_data_source_factory, self.api_list_data_source_factory,
object_store_creator_factory)
self.api_data_source_factory.SetReferenceResolverFactory(
self.ref_resolver_factory)
# Note: samples are super slow in the dev server because it doesn't support
# async fetch, so disable them. If you actually want to test samples, then
# good luck, and modify _IsSamplesDisabled at the top.
if _IsSamplesDisabled():
svn_fs_for_samples = EmptyDirFileSystem()
else:
svn_fs_for_samples = self.svn_file_system
self.samples_data_source_factory = SamplesDataSource.Factory(
channel, svn_fs_for_samples, self.github_file_system,
self.ref_resolver_factory, object_store_creator_factory,
svn_constants.EXAMPLES_PATH)
self.api_data_source_factory.SetSamplesDataSourceFactory(
self.samples_data_source_factory)
self.intro_data_source_factory = IntroDataSource.Factory(
self.compiled_fs_factory, self.ref_resolver_factory,
[svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH])
self.sidenav_data_source_factory = SidenavDataSource.Factory(
self.compiled_fs_factory, svn_constants.JSON_PATH)
self.template_data_source_factory = TemplateDataSource.Factory(
channel, self.api_data_source_factory,
self.api_list_data_source_factory, self.intro_data_source_factory,
self.samples_data_source_factory, self.sidenav_data_source_factory,
self.compiled_fs_factory, self.ref_resolver_factory,
svn_constants.PUBLIC_TEMPLATE_PATH,
svn_constants.PRIVATE_TEMPLATE_PATH)
self.example_zipper = ExampleZipper(self.compiled_fs_factory,
svn_constants.DOCS_PATH)
self.path_canonicalizer = PathCanonicalizer(channel,
self.compiled_fs_factory)
self.content_cache = self.compiled_fs_factory.GetOrCreateIdentity()
def _FetchStaticResource(self, path, response):
"""Fetch a resource in the 'static' directory.
"""
mimetype = mimetypes.guess_type(path)[0] or 'text/plain'
result = self.content_cache.GetFromFile(
svn_constants.DOCS_PATH + '/' + path,
binary=_IsBinaryMimetype(mimetype))
response.headers['content-type'] = mimetype
return result
def Get(self, path, request, response):
templates = self.template_data_source_factory.Create(request, path)
content = None
try:
if fnmatch(path, 'extensions/examples/*.zip'):
content = self.example_zipper.Create(
path[len('extensions/'):-len('.zip')])
response.headers['content-type'] = 'application/zip'
elif path.startswith('extensions/examples/'):
mimetype = mimetypes.guess_type(path)[0] or 'text/plain'
content = self.content_cache.GetFromFile(
'%s/%s' %
(svn_constants.DOCS_PATH, path[len('extensions/'):]),
binary=_IsBinaryMimetype(mimetype))
response.headers['content-type'] = 'text/plain'
elif path.startswith('static/'):
content = self._FetchStaticResource(path, response)
elif path.endswith('.html'):
content = templates.Render(path)
except FileNotFoundError as e:
logging.warning(traceback.format_exc())
response.headers['x-frame-options'] = 'sameorigin'
if content is None:
response.set_status(404)
response.out.write(templates.Render('404'))
else:
if not content:
logging.error('%s had empty content' % path)
response.headers['cache-control'] = 'max-age=300'
response.out.write(content)
def __init__(self, channel, object_store_creator, host_file_system,
app_samples_file_system, base_path, compiled_fs_factory):
self.channel = channel
self.object_store_creator = object_store_creator
self.host_file_system = host_file_system
self.app_samples_file_system = app_samples_file_system
self.compiled_host_fs_factory = compiled_fs_factory
self.api_list_data_source_factory = APIListDataSource.Factory(
self.compiled_host_fs_factory, svn_constants.API_PATH,
svn_constants.PUBLIC_TEMPLATE_PATH)
self.api_data_source_factory = APIDataSource.Factory(
self.compiled_host_fs_factory, svn_constants.API_PATH)
self.ref_resolver_factory = ReferenceResolver.Factory(
self.api_data_source_factory, self.api_list_data_source_factory,
object_store_creator)
self.api_data_source_factory.SetReferenceResolverFactory(
self.ref_resolver_factory)
# Note: samples are super slow in the dev server because it doesn't support
# async fetch, so disable them.
if IsDevServer():
extension_samples_fs = EmptyDirFileSystem()
else:
extension_samples_fs = self.host_file_system
self.samples_data_source_factory = SamplesDataSource.Factory(
channel, extension_samples_fs,
CompiledFileSystem.Factory(extension_samples_fs,
object_store_creator),
self.app_samples_file_system,
CompiledFileSystem.Factory(self.app_samples_file_system,
object_store_creator),
self.ref_resolver_factory, svn_constants.EXAMPLES_PATH)
self.api_data_source_factory.SetSamplesDataSourceFactory(
self.samples_data_source_factory)
self.intro_data_source_factory = IntroDataSource.Factory(
self.compiled_host_fs_factory, self.ref_resolver_factory,
[svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH])
self.sidenav_data_source_factory = SidenavDataSource.Factory(
self.compiled_host_fs_factory, svn_constants.JSON_PATH, base_path)
self.template_data_source_factory = TemplateDataSource.Factory(
channel, self.api_data_source_factory,
self.api_list_data_source_factory, self.intro_data_source_factory,
self.samples_data_source_factory, self.sidenav_data_source_factory,
self.compiled_host_fs_factory, self.ref_resolver_factory,
svn_constants.PUBLIC_TEMPLATE_PATH,
svn_constants.PRIVATE_TEMPLATE_PATH, base_path)
self.example_zipper = ExampleZipper(self.compiled_host_fs_factory,
svn_constants.DOCS_PATH)
self.path_canonicalizer = PathCanonicalizer(
channel, self.compiled_host_fs_factory)
self.content_cache = self.compiled_host_fs_factory.CreateIdentity(
ServerInstance)
def __init__(self, object_store_creator, host_file_system,
app_samples_file_system, base_path, compiled_fs_factory,
branch_utility, host_file_system_creator):
self.object_store_creator = object_store_creator
self.host_file_system = host_file_system
self.app_samples_file_system = app_samples_file_system
self.compiled_host_fs_factory = compiled_fs_factory
self.host_file_system_creator = host_file_system_creator
self.availability_finder_factory = AvailabilityFinder.Factory(
object_store_creator, self.compiled_host_fs_factory,
branch_utility, host_file_system_creator)
self.api_list_data_source_factory = APIListDataSource.Factory(
self.compiled_host_fs_factory, self.host_file_system,
svn_constants.API_PATH, svn_constants.PUBLIC_TEMPLATE_PATH)
self.api_data_source_factory = APIDataSource.Factory(
self.compiled_host_fs_factory, svn_constants.API_PATH,
self.availability_finder_factory)
self.ref_resolver_factory = ReferenceResolver.Factory(
self.api_data_source_factory, self.api_list_data_source_factory,
object_store_creator)
self.api_data_source_factory.SetReferenceResolverFactory(
self.ref_resolver_factory)
# Note: samples are super slow in the dev server because it doesn't support
# async fetch, so disable them.
if IsDevServer():
extension_samples_fs = EmptyDirFileSystem()
else:
extension_samples_fs = self.host_file_system
self.samples_data_source_factory = SamplesDataSource.Factory(
extension_samples_fs,
CompiledFileSystem.Factory(extension_samples_fs,
object_store_creator),
self.app_samples_file_system,
CompiledFileSystem.Factory(self.app_samples_file_system,
object_store_creator),
self.ref_resolver_factory, svn_constants.EXAMPLES_PATH, base_path)
self.api_data_source_factory.SetSamplesDataSourceFactory(
self.samples_data_source_factory)
self.intro_data_source_factory = IntroDataSource.Factory(
self.compiled_host_fs_factory, self.ref_resolver_factory,
[svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH])
self.sidenav_data_source_factory = SidenavDataSource.Factory(
self.compiled_host_fs_factory, svn_constants.JSON_PATH)
self.manifest_data_source = ManifestDataSource(
self.compiled_host_fs_factory, host_file_system, '/'.join(
(svn_constants.JSON_PATH, 'manifest.json')), '/'.join(
(svn_constants.API_PATH, '_manifest_features.json')))
self.template_data_source_factory = TemplateDataSource.Factory(
self.api_data_source_factory, self.api_list_data_source_factory,
self.intro_data_source_factory, self.samples_data_source_factory,
self.sidenav_data_source_factory, self.compiled_host_fs_factory,
self.ref_resolver_factory, self.manifest_data_source,
svn_constants.PUBLIC_TEMPLATE_PATH,
svn_constants.PRIVATE_TEMPLATE_PATH, base_path)
self.api_data_source_factory.SetTemplateDataSource(
self.template_data_source_factory)
self.example_zipper = ExampleZipper(self.compiled_host_fs_factory,
self.host_file_system,
svn_constants.DOCS_PATH)
self.path_canonicalizer = PathCanonicalizer(
self.compiled_host_fs_factory)
self.redirector = Redirector(self.compiled_host_fs_factory,
self.host_file_system,
svn_constants.PUBLIC_TEMPLATE_PATH)
class ServerInstance(object):
# Lazily create so we don't create github file systems unnecessarily in
# tests.
branch_utility = None
github_file_system = None
@staticmethod
@memoize
def GetOrCreateOffline(channel):
'''Gets/creates a local ServerInstance, meaning that only resources local to
the server - memcache, object store, etc, are queried. This amounts to not
setting up the subversion nor github file systems.
'''
branch_utility = ServerInstance._GetOrCreateBranchUtility()
branch = branch_utility.GetBranchNumberForChannelName(channel)
object_store_creator_factory = ObjectStoreCreator.Factory(branch)
# No svn nor github file systems. Rely on the crons to fill the caches, and
# for the caches to exist.
return ServerInstance(
channel,
object_store_creator_factory,
CachingFileSystem(OfflineFileSystem(SubversionFileSystem),
object_store_creator_factory),
# TODO(kalman): convert GithubFileSystem to be wrappable in a
# CachingFileSystem so that it can be replaced with an
# OfflineFileSystem. Currently GFS doesn't set the child versions of
# stat requests so it doesn't.
ServerInstance._GetOrCreateGithubFileSystem())
@staticmethod
@memoize
def GetOrCreateOnline(channel):
'''Creates/creates an online server instance, meaning that both local and
subversion/github resources are queried.
'''
branch_utility = ServerInstance._GetOrCreateBranchUtility()
branch = branch_utility.GetBranchNumberForChannelName(channel)
if branch == 'trunk':
svn_url = '/'.join((url_constants.SVN_TRUNK_URL, 'src',
svn_constants.EXTENSIONS_PATH))
else:
svn_url = '/'.join((url_constants.SVN_BRANCH_URL, branch, 'src',
svn_constants.EXTENSIONS_PATH))
viewvc_url = svn_url.replace(url_constants.SVN_URL,
url_constants.VIEWVC_URL)
object_store_creator_factory = ObjectStoreCreator.Factory(branch)
svn_file_system = CachingFileSystem(
SubversionFileSystem(AppEngineUrlFetcher(svn_url),
AppEngineUrlFetcher(viewvc_url)),
object_store_creator_factory)
return ServerInstance(channel, object_store_creator_factory,
svn_file_system,
ServerInstance._GetOrCreateGithubFileSystem())
@staticmethod
def CreateForTest(file_system):
return ServerInstance('test', ObjectStoreCreator.Factory('test'),
file_system, None)
@staticmethod
def _GetOrCreateBranchUtility():
if ServerInstance.branch_utility is None:
ServerInstance.branch_utility = BranchUtility(
url_constants.OMAHA_PROXY_URL, AppEngineUrlFetcher())
return ServerInstance.branch_utility
@staticmethod
def _GetOrCreateGithubFileSystem():
if ServerInstance.github_file_system is None:
ServerInstance.github_file_system = GithubFileSystem(
AppEngineUrlFetcher(url_constants.GITHUB_URL),
AppEngineBlobstore())
return ServerInstance.github_file_system
def __init__(self, channel, object_store_creator_factory, svn_file_system,
github_file_system):
self.svn_file_system = svn_file_system
self.github_file_system = github_file_system
self.compiled_fs_factory = CompiledFileSystem.Factory(
svn_file_system, object_store_creator_factory)
self.api_list_data_source_factory = APIListDataSource.Factory(
self.compiled_fs_factory, svn_constants.API_PATH,
svn_constants.PUBLIC_TEMPLATE_PATH)
self.api_data_source_factory = APIDataSource.Factory(
self.compiled_fs_factory, svn_constants.API_PATH)
self.ref_resolver_factory = ReferenceResolver.Factory(
self.api_data_source_factory, self.api_list_data_source_factory,
object_store_creator_factory)
self.api_data_source_factory.SetReferenceResolverFactory(
self.ref_resolver_factory)
self.samples_data_source_factory = SamplesDataSource.Factory(
channel, self.svn_file_system, ServerInstance.github_file_system,
self.ref_resolver_factory, object_store_creator_factory,
svn_constants.EXAMPLES_PATH)
self.api_data_source_factory.SetSamplesDataSourceFactory(
self.samples_data_source_factory)
self.intro_data_source_factory = IntroDataSource.Factory(
self.compiled_fs_factory, self.ref_resolver_factory,
[svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH])
self.sidenav_data_source_factory = SidenavDataSource.Factory(
self.compiled_fs_factory, svn_constants.JSON_PATH)
self.template_data_source_factory = TemplateDataSource.Factory(
channel, self.api_data_source_factory,
self.api_list_data_source_factory, self.intro_data_source_factory,
self.samples_data_source_factory, self.sidenav_data_source_factory,
self.compiled_fs_factory, self.ref_resolver_factory,
svn_constants.PUBLIC_TEMPLATE_PATH,
svn_constants.PRIVATE_TEMPLATE_PATH)
self.example_zipper = ExampleZipper(self.compiled_fs_factory,
svn_constants.DOCS_PATH)
self.path_canonicalizer = PathCanonicalizer(channel,
self.compiled_fs_factory)
self.content_cache = self.compiled_fs_factory.GetOrCreateIdentity()
def _FetchStaticResource(self, path, response):
"""Fetch a resource in the 'static' directory.
"""
mimetype = mimetypes.guess_type(path)[0] or 'text/plain'
result = self.content_cache.GetFromFile(
svn_constants.DOCS_PATH + '/' + path,
binary=_IsBinaryMimetype(mimetype))
response.headers['content-type'] = mimetype
return result
def Get(self, path, request, response):
if os.environ.get('CRXDOCZH_SLAVE_TYPE') == 'samples':
logging.error('NOTREACHED: slave-samples: server_instance: Get')
elif os.environ.get('CRXDOCZH_SLAVE_TYPE') == 'docs':
if not self._GetDocsAPI(path, request, response):
self._OriginalGet(path, request, response)
else:
self._OriginalGet(path, request, response)
def _GetSamplesJSON(self, request, key, update=False):
templates = self.template_data_source_factory.Create(
request, key + '/samples.html')
if key == 'apps' or key == 'extensions':
cache_url = None
if key == 'apps':
cache_url = '/trunk/' + key
else:
cache_url = '/' + templates._branch_info['current'] + '/' + key
logging.info('Serving API request %s' % cache_url)
data = ResponseCache.Get(cache_url)
age = ResponseCache.GetAge(cache_url)
need_update = (age > datetime.timedelta(1))
logging.info('Cache age: %s' % str(age))
if data is not None and not (update == True
and need_update == True):
return data
else:
content = []
try:
logging.info('Trying to generate samples.json for %s' %
cache_url)
content = templates._samples_data_source.GetAsJSON(key)
except:
logging.getLogger('slave-samples-api').exception(
'Error generating samples!')
pass
if len(content) > 0:
logging.info('samples.json saved for %s' % cache_url)
ResponseCache.Set(cache_url, content)
return content
def _GetDocsAPI(self, path, request, response):
if path.startswith(url_constants.SLAVE_DOCS_API_BASE_URL):
new_path = path[len(url_constants.SLAVE_DOCS_API_BASE_URL):]
if new_path.startswith('static/'):
response.headers['content-type'] = 'text/plain'
response.out.write(
self._FetchStaticResource(new_path, response))
return True
if not new_path.endswith('.html'):
return False
content = self._GetDocsHTML(request, new_path)
if len(content) > 0:
response.headers['content-type'] = 'text/plain'
response.out.write(content)
else:
response.set_status(503)
return True
else:
return False
def _GetDocsHTML(self, request, path, update=False):
templates = self.template_data_source_factory.Create(request, path)
cache_url = '/' + templates._branch_info['current'] + '/' + path
logging.info('Serving API request %s' % cache_url)
data = ResponseCache.Get(cache_url)
age = ResponseCache.GetAge(cache_url)
need_update = (age > datetime.timedelta(1))
logging.info('Cache age: %s' % str(age))
if data is not None and not (update == True and need_update == True):
return data
else:
content = ''
try:
logging.info('Trying to render %s' % cache_url)
content = templates.Render(path)
except:
logging.getLogger('slave-docs-api').exception(
'Error rendering HTML!')
pass
if len(content) > 0:
logging.info('HTML saved for %s' % cache_url)
ResponseCache.Set(cache_url, content.encode('utf-8'))
return content
def _OriginalGet(self, path, request, response):
templates = self.template_data_source_factory.Create(request, path)
content = None
try:
if fnmatch(path, 'extensions/examples/*.zip'):
content = self.example_zipper.Create(
path[len('extensions/'):-len('.zip')])
response.headers['content-type'] = 'application/zip'
elif path.startswith('extensions/examples/'):
mimetype = mimetypes.guess_type(path)[0] or 'text/plain'
content = self.content_cache.GetFromFile(
'%s/%s' %
(svn_constants.DOCS_PATH, path[len('extensions/'):]),
binary=_IsBinaryMimetype(mimetype))
response.headers['content-type'] = 'text/plain'
elif path.startswith('static/'):
content = self._FetchStaticResource(path, response)
elif path.endswith('.html'):
content = templates.Render(path)
except FileNotFoundError as e:
logging.warning(e)
response.headers['x-frame-options'] = 'sameorigin'
if content is None:
response.set_status(404)
response.out.write(templates.Render('404'))
else:
if not content:
logging.error('%s had empty content' % path)
response.headers['cache-control'] = 'max-age=300'
response.out.write(content)
cache_url = '/' + templates._branch_info['current'] + '/' + path
if isinstance(content, unicode):
ResponseCache.Set(cache_url, content.encode('utf-8'))
else:
ResponseCache.Set(cache_url, content)