def __init__(self, object_store_creator, compiled_fs_factory, host_file_system, github_file_system_provider, gcs_file_system_provider): self._object_store_creator = object_store_creator self._compiled_fs_factory = compiled_fs_factory self._host_file_system = host_file_system self._github_file_system_provider = github_file_system_provider self._gcs_file_system_provider = gcs_file_system_provider self._cache = None # If running the devserver and there is a LOCAL_DEBUG_DIR, we # will read the content_provider configuration from there instead # of fetching it from SVN trunk or patch. if environment.IsDevServer() and os.path.exists(LOCAL_DEBUG_DIR): local_fs = LocalFileSystem(LOCAL_DEBUG_DIR) conf_stat = None try: conf_stat = local_fs.Stat(CONTENT_PROVIDERS) except: pass if conf_stat: logging.warn( ("Using local debug folder (%s) for " "content_provider.json configuration") % LOCAL_DEBUG_DIR) self._cache = compiled_fs_factory.ForJson(local_fs) if not self._cache: self._cache = compiled_fs_factory.ForJson(host_file_system)
def ForLocal(object_store_creator, **optargs): '''Used in creating a server instance on localhost. ''' return HostFileSystemProvider( object_store_creator, constructor_for_test=lambda **_: LocalFileSystem.Create(), **optargs)
def testSimple(self): self._base_path = os.path.join(self._base_path, 'simple') fetcher = LocalFileSystem(self._base_path) compiled_fs_factory = CompiledFileSystem.Factory( fetcher, ObjectStoreCreator.Factory()) t_data_source = self._CreateTemplateDataSource( compiled_fs_factory, ObjectStoreCreator.Factory()) template_a1 = Handlebar(self._ReadLocalFile('test1.html')) self.assertEqual( template_a1.render({}, { 'templates': {} }).text, t_data_source.get('test1').render({}, { 'templates': {} }).text) template_a2 = Handlebar(self._ReadLocalFile('test2.html')) self.assertEqual( template_a2.render({}, { 'templates': {} }).text, t_data_source.get('test2').render({}, { 'templates': {} }).text) self.assertEqual(None, t_data_source.get('junk.html'))
def testEverything(self): # All these tests are dependent (see above comment) so lump everything in # the one test. delegate = _TestDelegate( lambda _: MockFileSystem(LocalFileSystem.Create())) # Test that the cron runs successfully. response = CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get() self.assertEqual(200, response.status) # Save the file systems created, start with a fresh set for the next run. first_run_file_systems = delegate.file_systems[:] delegate.file_systems[:] = [] # When re-running, all file systems should be Stat()d the same number of # times, but the second round shouldn't have been re-Read() since the # Stats haven't changed. response = CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get() self.assertEqual(200, response.status) self.assertEqual(len(first_run_file_systems), len(delegate.file_systems)) for i, second_run_file_system in enumerate(delegate.file_systems): self.assertTrue(*second_run_file_system.CheckAndReset( read_count=0, stat_count=first_run_file_systems[i].GetStatCount()))
def testNotFound(self): self._base_path = os.path.join(self._base_path, 'simple') fetcher = LocalFileSystem(self._base_path) compiled_fs_factory = CompiledFileSystem.Factory( fetcher, ObjectStoreCreator.TestFactory()) t_data_source = self._CreateTemplateDataSource( compiled_fs_factory, ObjectStoreCreator.TestFactory()) self.assertEqual(None, t_data_source.get('junk.html'))
def _MakeShaJson(self, hash_value): commit_json = json.loads( deepcopy( LocalFileSystem('').ReadSingle( 'test_data/github_file_system/test_owner/repo/commits/HEAD' ).Get())) commit_json['sha'] = hash_value return json.dumps(commit_json)
def setUp(self): object_store_creator = ObjectStoreCreator.ForTest() self._file_system = CachingFileSystem( LocalFileSystem(os.path.join(sys.path[0], 'test_data')), object_store_creator) self._example_zipper = ExampleZipper( CompiledFileSystem.Factory(self._file_system, object_store_creator), 'example_zipper')
def setUp(self): object_store = InMemoryObjectStore('') self._file_system = MemcacheFileSystem( LocalFileSystem(os.path.join(sys.path[0], 'test_data')), object_store) self._example_zipper = ExampleZipper( self._file_system, CompiledFileSystem.Factory(self._file_system, object_store), 'example_zipper')
def testPartials(self): self._base_path = os.path.join(self._base_path, 'partials') fetcher = LocalFileSystem(self._base_path) cache_factory = CompiledFileSystem.Factory(fetcher, self._object_store) t_data_source = self._CreateTemplateDataSource(cache_factory) self.assertEqual( self._ReadLocalFile('test_expected.html'), t_data_source.get('test_tmpl').render( json.loads(self._ReadLocalFile('input.json')), t_data_source).text)
def setUp(self): self._base_path = os.path.join(sys.path[0], 'test_data', 'template_data_source') self._fake_api_list_data_source_factory = _FakeFactory() self._fake_intro_data_source_factory = _FakeFactory() self._fake_samples_data_source_factory = _FakeFactory() self._fake_sidenav_data_source_factory = _FakeFactory() self._manifest_data_source = ManifestDataSource( _FakeFactory(), LocalFileSystem.Create(), '', '')
def ForLocal(): channel = 'trunk' object_store_creator = ObjectStoreCreator(channel, start_empty=False, store_type=TestObjectStore) file_system = CachingFileSystem(LocalFileSystem.Create(), object_store_creator) return ServerInstance( channel, object_store_creator, file_system, EmptyDirFileSystem(), '', CompiledFileSystem.Factory(file_system, object_store_creator))
def make_sha_json(hash_value): from copy import deepcopy commit_json = json.loads( deepcopy( LocalFileSystem('').ReadSingle( 'test_data/github_file_system/test_owner/repo/commits/HEAD' ).Get())) commit_json['commit']['tree']['sha'] = 'hash' + FAKE_HASH[4:] return json.dumps(commit_json)
def _GetInstanceForBranch(channel_name, local_path): branch = BRANCH_UTILITY.GetBranchNumberForChannelName(channel_name) # The key for the server is a tuple of |channel_name| with |branch|, since # sometimes stable and beta point to the same branch. instance_key = _MakeInstanceKey(channel_name, branch) instance = SERVER_INSTANCES.get(instance_key, None) if instance is not None: return instance branch_memcache = InMemoryObjectStore(branch) if branch == 'local': file_system = LocalFileSystem(local_path) else: file_system = _CreateMemcacheFileSystem(branch, branch_memcache) cache_factory = CompiledFileSystem.Factory(file_system, branch_memcache) api_list_data_source_factory = APIListDataSource.Factory(cache_factory, file_system, API_PATH, PUBLIC_TEMPLATE_PATH) intro_data_source_factory = IntroDataSource.Factory( cache_factory, [INTRO_PATH, ARTICLE_PATH]) samples_data_source_factory = SamplesDataSource.Factory( channel_name, file_system, GITHUB_FILE_SYSTEM, cache_factory, GITHUB_COMPILED_FILE_SYSTEM, api_list_data_source_factory, EXAMPLES_PATH) api_data_source_factory = APIDataSource.Factory(cache_factory, API_PATH, samples_data_source_factory) template_data_source_factory = TemplateDataSource.Factory( channel_name, api_data_source_factory, api_list_data_source_factory, intro_data_source_factory, samples_data_source_factory, KNOWN_ISSUES_DATA_SOURCE, cache_factory, PUBLIC_TEMPLATE_PATH, PRIVATE_TEMPLATE_PATH) example_zipper = ExampleZipper(file_system, cache_factory, DOCS_PATH) instance = ServerInstance(template_data_source_factory, example_zipper, cache_factory) SERVER_INSTANCES[instance_key] = instance return instance
class LocalFileSystemTest(unittest.TestCase): def setUp(self): self._file_system = LocalFileSystem( os.path.join(sys.path[0], 'test_data', 'file_system')) def testReadFiles(self): expected = { 'test1.txt': 'test1\n', 'test2.txt': 'test2\n', 'test3.txt': 'test3\n', } self.assertEqual( expected, self._file_system.Read(['test1.txt', 'test2.txt', 'test3.txt']).Get()) def testListDir(self): expected = ['dir/'] for i in range(7): expected.append('file%d.html' % i) self.assertEqual(expected, sorted(self._file_system.ReadSingle('list/')))
def testRender(self): self._base_path = os.path.join(self._base_path, 'render') fetcher = LocalFileSystem(self._base_path) context = json.loads(self._ReadLocalFile('test1.json')) cache_factory = CompiledFileSystem.Factory(fetcher, self._object_store) self._RenderTest( 'test1', self._CreateTemplateDataSource( json.loads(self._ReadLocalFile('test1.json')), cache_factory)) self._RenderTest( 'test2', self._CreateTemplateDataSource( json.loads(self._ReadLocalFile('test2.json')), cache_factory))
def testRender(self): self._base_path = os.path.join(self._base_path, 'render') fetcher = LocalFileSystem(self._base_path) context = json.loads(self._ReadLocalFile('test1.json')) compiled_fs_factory = CompiledFileSystem.Factory( fetcher, ObjectStoreCreator.Factory()) self._RenderTest( 'test1', self._CreateTemplateDataSource( compiled_fs_factory, api_data=json.loads(self._ReadLocalFile('test1.json')))) self._RenderTest( 'test2', self._CreateTemplateDataSource( compiled_fs_factory, api_data=json.loads(self._ReadLocalFile('test2.json'))))
def testEverything(self): # All these tests are dependent (see above comment) so lump everything in # the one test. delegate = _TestDelegate( lambda _: MockFileSystem(LocalFileSystem.Create())) # Test that the cron runs successfully. response = CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get() self.assertEqual(1, len(delegate.file_systems)) self.assertEqual(200, response.status) # When re-running, all file systems should be Stat()d the same number of # times, but the second round shouldn't have been re-Read() since the # Stats haven't changed. response = CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get() self.assertEqual(2, len(delegate.file_systems)) self.assertTrue(*delegate.file_systems[1].CheckAndReset( read_count=0, stat_count=delegate.file_systems[0].GetStatCount()))
def DISABLED_testSimple(self): cache_factory = CompiledFileSystem.Factory( LocalFileSystem(self._base_path), InMemoryObjectStore('fake_branch')) data_source_factory = APIDataSource.Factory(cache_factory, '.', FakeSamplesDataSource()) data_source = data_source_factory.Create({}) # Take the dict out of the list. expected = json.loads(self._ReadLocalFile('expected_test_file.json')) expected['permissions'] = None test1 = data_source['test_file'] test1.pop('samples') self.assertEqual(expected, test1) test2 = data_source['testFile'] test2.pop('samples') self.assertEqual(expected, test2) test3 = data_source['testFile.html'] test3.pop('samples') self.assertEqual(expected, test3) self.assertRaises(FileNotFoundError, data_source.get, 'junk')
def testSimple(self): self._base_path = os.path.join(self._base_path, 'simple') fetcher = LocalFileSystem(self._base_path) cache_factory = CompiledFileSystem.Factory(fetcher, self._object_store) t_data_source = self._CreateTemplateDataSource( self._fake_api_data_source_factory, cache_factory) template_a1 = Handlebar(self._ReadLocalFile('test1.html')) self.assertEqual( template_a1.render({}, { 'templates': {} }).text, t_data_source['test1'].render({}, { 'templates': {} }).text) template_a2 = Handlebar(self._ReadLocalFile('test2.html')) self.assertEqual( template_a2.render({}, { 'templates': {} }).text, t_data_source['test2'].render({}, { 'templates': {} }).text) self.assertEqual(None, t_data_source['junk.html'])
def Create(self, bucket): '''Creates a CloudStorageFileSystemProvider. |bucket| is the name of GCS bucket, eg devtools-docs. It is expected that this bucket has Read permission for this app in its ACLs. Optional configuration can be set in a local_debug/gcs_debug.conf file: use_local_fs=True|False remote_bucket_prefix=<prefix> If running in Preview mode or in Development mode with use_local_fs set to True, buckets and files are looked for inside the local_debug folder instead of in the real GCS server. ''' if IsTest(): return EmptyDirFileSystem() debug_bucket_prefix = None use_local_fs = False if os.path.exists(LOCAL_GCS_DEBUG_CONF): with open(LOCAL_GCS_DEBUG_CONF, "r") as token_file: properties = dict(line.strip().split('=', 1) for line in token_file) use_local_fs = properties.get('use_local_fs', 'False') == 'True' debug_bucket_prefix = properties.get('remote_bucket_prefix', None) logging.debug('gcs: prefixing all bucket names with %s' % debug_bucket_prefix) if use_local_fs: return LocalFileSystem( ToDirectory(os.path.join(LOCAL_GCS_DIR, bucket))) if debug_bucket_prefix: bucket = debug_bucket_prefix + bucket return CachingFileSystem(CloudStorageFileSystem(bucket), self._object_store_creator)
def setUp(self): self._base_path = os.path.join(sys.path[0], 'test_data', 'sidenav_data_source') self._compiled_fs_factory = CompiledFileSystem.Factory( LocalFileSystem(self._base_path), ObjectStoreCreator.TestFactory())
def CreateServerInstance(self): return ServerInstance.ForTest(LocalFileSystem.Create())
def setUp(self): self._path_canonicalizer = PathCanonicalizer( LocalFileSystem.Create(PUBLIC_TEMPLATES), ObjectStoreCreator.ForTest(), ('.html', '.md'))
def Create(cls, branch='master', commit=None): if IsTest(): return LocalFileSystem.Create('') return LocalGitFileSystem(branch, commit)
def CreateServerInstanceForChannel(self, channel): return ServerInstance.ForTest(LocalFileSystem.Create())
def setUp(self): self._json_path = 'docs/server2/test_data/sidenav_data_source' self._compiled_fs_factory = CompiledFileSystem.Factory( LocalFileSystem.Create(), ObjectStoreCreator.ForTest())
def testCronAndPublicFiles(self): '''Runs cron then requests every public file. Cron needs to be run first because the public file requests are offline. ''' if _EXPLICIT_TEST_FILES is not None: return print('Running cron...') start_time = time.time() try: response = Handler(Request.ForTest('/_cron/stable')).Get() self.assertEqual(200, response.status) self.assertEqual('Success', response.content.ToString()) finally: print('Took %s seconds' % (time.time() - start_time)) print("Checking for broken links...") start_time = time.time() link_error_detector = LinkErrorDetector( LocalFileSystem(os.path.join(sys.path[0], os.pardir, os.pardir)), lambda path: Handler(Request.ForTest(path)).Get(), 'templates/public', ('extensions/index.html', 'apps/about_apps.html')) broken_links = link_error_detector.GetBrokenLinks() if broken_links: # TODO(jshumway): Test should fail when broken links are detected. print('Warning: Found %d broken links:' % (len(broken_links))) print(StringifyBrokenLinks(broken_links)) print('Took %s seconds.' % (time.time() - start_time)) print('Searching for orphaned pages...') start_time = time.time() orphaned_pages = link_error_detector.GetOrphanedPages() if orphaned_pages: # TODO(jshumway): Test should fail when orphaned pages are detected. print('Warning: Found %d orphaned pages:' % len(orphaned_pages)) for page in orphaned_pages: print(page) print('Took %s seconds.' % (time.time() - start_time)) public_files = _GetPublicFiles() print('Rendering %s public files...' % len(public_files.keys())) start_time = time.time() try: for path, content in public_files.iteritems(): if path.endswith('redirects.json'): continue def check_result(response): self.assertEqual( 200, response.status, 'Got %s when rendering %s' % (response.status, path)) # This is reaaaaally rough since usually these will be tiny templates # that render large files. At least it'll catch zero-length responses. self.assertTrue( len(response.content) >= len(content), 'Content was "%s" when rendering %s' % (response.content, path)) check_result(Handler(Request.ForTest(path)).Get()) # Make sure that leaving out the .html will temporarily redirect to the # path with the .html. if path.startswith(('apps/', 'extensions/')): redirect_result = Handler( Request.ForTest(posixpath.splitext(path)[0])).Get() self.assertEqual((path, False), redirect_result.GetRedirect()) # Make sure including a channel will permanently redirect to the same # path without a channel. for channel in BranchUtility.GetAllChannelNames(): redirect_result = Handler( Request.ForTest('%s/%s' % (channel, path))).Get() self.assertEqual((path, True), redirect_result.GetRedirect()) # Samples are internationalized, test some locales. if path.endswith('/samples.html'): for lang in ['en-US', 'es', 'ar']: check_result( Handler( Request.ForTest(path, headers={ 'Accept-Language': '%s;q=0.8' % lang })).Get()) finally: print('Took %s seconds' % (time.time() - start_time))
def Create(self, bucket): '''Creates a CloudStorageFileSystemProvider. |bucket| is the name of GCS bucket, eg devtools-docs. It is expected that this bucket has Read permission for this app in its ACLs. Optional configuration can be set in a local_debug/gcs_debug.conf file: use_local_fs=True|False access_token=<token> remote_bucket_prefix=<prefix> If running in Preview mode or in Development mode with use_local_fs set to True, buckets and files are looked inside the local_debug folder instead of in the real GCS server. Preview server does not support direct GCS access, so it is always forced to use a LocalFileSystem. For real GCS access in the Development mode (dev_appserver.py), access_token and remote_bucket_prefix options can be used to change the way GCS files are accessed. Both are ignored in a real appengine instance. "access_token" is always REQUIRED on dev_appengine, otherwise you will get 404 (auth) errors. You can get one access_token valid for a few minutes by typing: gsutil -d ls 2>&1 | grep "Bearer" | sed "s/.*Bearer \(.*\).r.nUser-Agent.*/access_token=\1/" )" A sample output would be: access_token=ya29.1.AADtN_VW5ibbfLHV5cMIK5ss4bHtVzBXpa4byjd Now add this line to the local_debug/gcs_debug.conf file and restart the appengine development server. Remember that you will need a new access_token every ten minutes or so. If you get 404 errors on log, update it. Access token is not used for a deployed appengine app, only if you use dev_appengine.py. remote_bucket_prefix is useful if you want to test on your own GCS buckets before using the real GCS buckets. ''' if not environment.IsReleaseServer() and not environment.IsDevServer(): bucket_local_path = os.path.join(LOCAL_GCS_DIR, bucket) if IsDirectory(bucket_local_path): return LocalFileSystem(bucket_local_path) else: return EmptyDirFileSystem() debug_access_token = None debug_bucket_prefix = None use_local_fs = False if environment.IsDevServer() and os.path.exists(LOCAL_GCS_DEBUG_CONF): with open(LOCAL_GCS_DEBUG_CONF, "r") as token_file: properties = dict(line.strip().split('=', 1) for line in token_file) use_local_fs = properties.get('use_local_fs', 'False') == 'True' debug_access_token = properties.get('access_token', None) debug_bucket_prefix = properties.get('remote_bucket_prefix', None) if environment.IsDevServer() and use_local_fs: return LocalFileSystem(os.path.join(LOCAL_GCS_DIR, bucket)) # gcs_file_system has strong dependencies on runtime appengine APIs, # so we only import it when we are sure we are not on preview.py or tests. from gcs_file_system import CloudStorageFileSystem return CachingFileSystem( CloudStorageFileSystem(bucket, debug_access_token, debug_bucket_prefix), self._object_store_creator)
def _CreateLocalFs(): return LocalFileSystem( os.path.join(sys.path[0], 'test_data', 'file_system'))
def CreateHostFileSystemForBranch(self, channel): return LocalFileSystem.Create()