def setUp(self): test_fs = TestFileSystem(_TEST_DATA) compiled_fs_factory = CompiledFileSystem.Factory( test_fs, ObjectStoreCreator.TestFactory()) self._path_canonicalizer = PathCanonicalizer('stable', compiled_fs_factory)
def _CreateFactory(): return CompiledFileSystem.Factory( TestFileSystem(deepcopy(_TEST_DATA)), ObjectStoreCreator(start_empty=False, store_type=TestObjectStore, disable_wrappers=True))
def setUp(self): self._path_canonicalizer = PathCanonicalizer( LocalFileSystem.Create(PUBLIC_TEMPLATES), ObjectStoreCreator.ForTest(), ('.html', '.md'))
def __init__(self, fetcher, blobstore): self._fetcher = fetcher self._stat_object_store = ObjectStoreCreator(GithubFileSystem).Create() self._blobstore = blobstore self._version = None self._GetZip(self.Stat(ZIP_KEY).version)
def _CreateFactory(): return CompiledFileSystem.Factory(TestFileSystem(deepcopy(_TEST_DATA)), ObjectStoreCreator.Factory(), store_type=TestObjectStore)
def testRemoveNoDocs(self): expected_nodoc = [ { 'name': 'B', 'list': [ { 'name': 'B2' } ] }, { 'name': 'D', 'nodoc': False }, { 'name': 'E', 'items1': [ { 'name': 'E1', 'items': [ { 'name': 'E1.3' } ] }, { 'name': 'E2' } ] } ] nodoc_data = [ { 'name': 'A', 'nodoc': True }, { 'name': 'B', 'list': [ { 'name': 'B1', 'nodoc': True }, { 'name': 'B2' }, { 'name': 'B3', 'nodoc': True } ] }, { 'name': 'C', 'nodoc': True }, { 'name': 'D', 'nodoc': False }, { 'name': 'E', 'dict': { 'name': 'Ed', 'nodoc': True }, 'items1': [ { 'name': 'E1', 'items': [ { 'name': 'E1.1', 'nodoc': True }, { 'name': 'E1.2', 'nodoc': True }, { 'name': 'E1.3' } ] }, { 'name': 'E2' }, { 'name': 'E3', 'nodoc': True } ] } ] object_store_creator = ObjectStoreCreator(start_empty=False) host_file_system_provider = HostFileSystemProvider(object_store_creator) schema_processor = SchemaProcessor(_FakeReferenceResolver(), _FakeAPIModels(), _FakeFeaturesBundle(), CompiledFileSystem.Factory( object_store_creator), host_file_system_provider.GetMaster(), True) schema_processor._RemoveNoDocs(nodoc_data) self.assertEquals(expected_nodoc, nodoc_data)
def _CreateRefResolver(self, filename): test_data = self._LoadJSON(filename) return ReferenceResolver.Factory(_FakeAPIDataSource(test_data), _FakeAPIModels(test_data), ObjectStoreCreator.ForTest()).Create()
'extensions': { 'redirects.json': json.dumps({ 'manifest': 'manifest.html' }), 'manifest': { 'redirects.json': json.dumps({ '': '../manifest.html', 'more-info': 'http://lmgtfy.com' }) } } } }) redirector = Redirector( CompiledFileSystem.Factory(file_system, ObjectStoreCreator.ForTest()), None, 'public') class RedirectorTest(unittest.TestCase): def testExternalRedirection(self): self.assertEqual( 'http://something.absolute.com/', redirector.Redirect(HOST, 'index.html')) self.assertEqual( 'http://lmgtfy.com', redirector.Redirect(HOST, 'extensions/manifest/more-info')) def testAbsoluteRedirection(self): self.assertEqual( '/apps/about_apps.html', redirector.Redirect(HOST, 'apps/index.html'))
def setUp(self): self._redirector = Redirector( CompiledFileSystem.Factory(ObjectStoreCreator.ForTest()), file_system)
def setUp(self): self._gfs = GithubFileSystem.ForTest( 'repo', partial(FakeURLFSFetcher, LocalFileSystem(''))) # Start and finish the repository load. self._cgfs = CachingFileSystem(self._gfs, ObjectStoreCreator.ForTest())
def setUp(self): self._branch_util = BranchUtility( os.path.join('branch_utility', 'first.json'), os.path.join('branch_utility', 'second.json'), FakeUrlFetcher(Server2Path('test_data')), ObjectStoreCreator.ForTest())
def __init__(self, fetch_url, fetcher, object_store=None): self._fetch_url = fetch_url self._fetcher = fetcher if object_store is None: object_store = ObjectStoreCreator(BranchUtility).Create() self._object_store = object_store
def setUp(self): self._refresh_tracker = RefreshTracker(ObjectStoreCreator.ForTest())
def testCaching(self): fake_fs = TestFileSystem({ 'bob': { 'bob0': 'bob/bob0 contents', 'bob1': 'bob/bob1 contents', 'bob2': 'bob/bob2 contents', 'bob3': 'bob/bob3 contents', } }) file_system = CachingFileSystem(fake_fs, ObjectStoreCreator.TestFactory()) self.assertEqual('bob/bob0 contents', file_system.ReadSingle('bob/bob0')) self.assertTrue(fake_fs.CheckAndReset(read_count=1, stat_count=1)) # Resource has been cached, so test resource is not re-fetched. self.assertEqual('bob/bob0 contents', file_system.ReadSingle('bob/bob0')) self.assertTrue(fake_fs.CheckAndReset()) # Test if the Stat version is the same the resource is not re-fetched. file_system._stat_object_store.Del('bob/bob0') self.assertEqual('bob/bob0 contents', file_system.ReadSingle('bob/bob0')) self.assertTrue(fake_fs.CheckAndReset(stat_count=1)) # Test if there is a newer version, the resource is re-fetched. file_system._stat_object_store.Del('bob/bob0') fake_fs.IncrementStat() self.assertEqual('bob/bob0 contents', file_system.ReadSingle('bob/bob0')) self.assertTrue(fake_fs.CheckAndReset(read_count=1, stat_count=1)) # Test directory and subdirectory stats are cached. file_system._stat_object_store.Del('bob/bob0') file_system._read_object_store.Del('bob/bob0') file_system._stat_object_store.Del('bob/bob1') fake_fs.IncrementStat() self.assertEqual('bob/bob1 contents', file_system.ReadSingle('bob/bob1')) self.assertEqual('bob/bob0 contents', file_system.ReadSingle('bob/bob0')) self.assertTrue(fake_fs.CheckAndReset(read_count=2, stat_count=1)) self.assertEqual('bob/bob1 contents', file_system.ReadSingle('bob/bob1')) self.assertTrue(fake_fs.CheckAndReset()) # Test a more recent parent directory doesn't force a refetch of children. file_system._read_object_store.Del('bob/bob0') file_system._read_object_store.Del('bob/bob1') self.assertEqual('bob/bob1 contents', file_system.ReadSingle('bob/bob1')) self.assertEqual('bob/bob2 contents', file_system.ReadSingle('bob/bob2')) self.assertEqual('bob/bob3 contents', file_system.ReadSingle('bob/bob3')) self.assertTrue(fake_fs.CheckAndReset(read_count=3)) fake_fs.IncrementStat(path='bob/') self.assertEqual('bob/bob1 contents', file_system.ReadSingle('bob/bob1')) self.assertEqual('bob/bob2 contents', file_system.ReadSingle('bob/bob2')) self.assertEqual('bob/bob3 contents', file_system.ReadSingle('bob/bob3')) self.assertTrue(fake_fs.CheckAndReset()) file_system._stat_object_store.Del('bob/bob0') self.assertEqual('bob/bob0 contents', file_system.ReadSingle('bob/bob0')) self.assertTrue(fake_fs.CheckAndReset(read_count=1, stat_count=1)) self.assertEqual('bob/bob0 contents', file_system.ReadSingle('bob/bob0')) self.assertTrue(fake_fs.CheckAndReset())
def _CreateRefResolver(self, filename): data_source = FakeAPIAndListDataSource(self._LoadJSON(filename)) return ReferenceResolver.Factory( data_source, data_source, ObjectStoreCreator.ForTest()).Create()
def testInstanceMethods(self): test_data = { 'app.yaml': _GenerateAppYaml('1-0'), 'app_yaml_helper.py': 'Copyright notice etc' } updates = [] # Pass a specific file system at head to the HostFileSystemProvider so that # we know it's always going to be backed by a MockFileSystem. The Provider # may decide to wrap it in caching etc. file_system_at_head = MockFileSystem( TestFileSystem(test_data, relative_to=SERVER2)) def apply_update(update): update = MoveTo(SERVER2, update) file_system_at_head.Update(update) updates.append(update) def host_file_system_constructor(branch, commit=None): self.assertEqual('master', branch) self.assertTrue(commit is not None) return MockFileSystem.Create( TestFileSystem(test_data, relative_to=SERVER2), updates[:commit]) object_store_creator = ObjectStoreCreator.ForTest() host_file_system_provider = HostFileSystemProvider( object_store_creator, default_master_instance=file_system_at_head, constructor_for_test=host_file_system_constructor) helper = AppYamlHelper(object_store_creator, host_file_system_provider) def assert_is_up_to_date(version): self.assertTrue(helper.IsUpToDate(version), '%s is not up to date' % version) self.assertRaises(ValueError, helper.GetFirstRevisionGreaterThan, version) self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0')) assert_is_up_to_date('1-0-0') assert_is_up_to_date('1-5-0') # Revision 1. apply_update({ 'app.yaml': _GenerateAppYaml('1-5-0') }) self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0')) self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0')) assert_is_up_to_date('1-5-0') assert_is_up_to_date('2-5-0') # Revision 2. apply_update({ 'app_yaml_helper.py': 'fixed a bug' }) self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0')) self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0')) assert_is_up_to_date('1-5-0') assert_is_up_to_date('2-5-0') # Revision 3. apply_update({ 'app.yaml': _GenerateAppYaml('1-6-0') }) self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0')) self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0')) self.assertEqual(3, helper.GetFirstRevisionGreaterThan('1-5-0')) assert_is_up_to_date('2-5-0') # Revision 4. apply_update({ 'app.yaml': _GenerateAppYaml('1-8-0') }) # Revision 5. apply_update({ 'app.yaml': _GenerateAppYaml('2-0-0') }) # Revision 6. apply_update({ 'app.yaml': _GenerateAppYaml('2-2-0') }) # Revision 7. apply_update({ 'app.yaml': _GenerateAppYaml('2-4-0') }) # Revision 8. apply_update({ 'app.yaml': _GenerateAppYaml('2-6-0') }) self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0')) self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0')) self.assertEqual(3, helper.GetFirstRevisionGreaterThan('1-5-0')) self.assertEqual(5, helper.GetFirstRevisionGreaterThan('1-8-0')) self.assertEqual(6, helper.GetFirstRevisionGreaterThan('2-0-0')) self.assertEqual(6, helper.GetFirstRevisionGreaterThan('2-1-0')) self.assertEqual(7, helper.GetFirstRevisionGreaterThan('2-2-0')) self.assertEqual(7, helper.GetFirstRevisionGreaterThan('2-3-0')) self.assertEqual(8, helper.GetFirstRevisionGreaterThan('2-4-0')) self.assertEqual(8, helper.GetFirstRevisionGreaterThan('2-5-0')) assert_is_up_to_date('2-6-0') assert_is_up_to_date('2-7-0')
def testInlineDocs(self): schema = { 'namespace': 'storage', 'properties': { 'key2': { 'description': 'second key', '$ref': 'Key' }, 'key1': { 'description': 'first key', '$ref': 'Key' } }, 'types': [ { 'inline_doc': True, 'type': 'string', 'id': 'Key', # Should be inlined into both properties and be removed # from types. 'description': 'This is a key.', # This description should disappear. 'marker': True # This should appear three times in the output. }, { 'items': { '$ref': 'Key' }, 'type': 'array', 'id': 'KeyList', 'description': 'A list of keys' } ] } expected_schema = { 'namespace': 'storage', 'properties': { 'key2': { 'marker': True, 'type': 'string', 'description': 'second key' }, 'key1': { 'marker': True, 'type': 'string', 'description': 'first key' } }, 'types': [ { 'items': { 'marker': True, 'type': 'string' }, 'type': 'array', 'id': 'KeyList', 'description': 'A list of keys' } ] } object_store_creator = ObjectStoreCreator(start_empty=False) host_file_system_provider = HostFileSystemProvider(object_store_creator) schema_processor = SchemaProcessor(_FakeReferenceResolver(), _FakeAPIModels(), _FakeFeaturesBundle(), CompiledFileSystem.Factory( object_store_creator), host_file_system_provider.GetMaster(), False) inlined_schema = deepcopy(schema) schema_processor._InlineDocs(inlined_schema) self.assertEqual(expected_schema, inlined_schema)
def setUp(self): self._base_path = os.path.join(sys.path[0], 'test_data', 'sidenav_data_source') self._compiled_fs_factory = CompiledFileSystem.Factory( LocalFileSystem(self._base_path), ObjectStoreCreator.Factory())
def testInstanceMethods(self): test_data = { 'server2': { 'app.yaml': _GenerateAppYaml('1-0'), 'app_yaml_helper.py': 'Copyright notice etc' } } updates = [] file_system_at_head = MockFileSystem(TestFileSystem(test_data)) def apply_update(update): file_system_at_head.Update(update) updates.append(update) assert_true = self.assertTrue class TestDelegate(AppYamlHelper.Delegate): def GetHostFileSystemForRevision(self, revision): assert_true(revision is not None) assert_true(revision >= 0) return MockFileSystem.Create(TestFileSystem(test_data), updates[:revision]) helper = AppYamlHelper( 'server2/app.yaml', file_system_at_head, TestDelegate(), ObjectStoreCreator.ForTest(disable_wrappers=False)) def assert_is_up_to_date(version): self.assertTrue(helper.IsUpToDate(version), '%s is not up to date' % version) self.assertRaises(ValueError, helper.GetFirstRevisionGreaterThan, version) self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0')) assert_is_up_to_date('1-0-0') assert_is_up_to_date('1-5-0') # Revision 1. apply_update({'server2': {'app.yaml': _GenerateAppYaml('1-5-0')}}) self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0')) self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0')) assert_is_up_to_date('1-5-0') assert_is_up_to_date('2-5-0') # Revision 2. apply_update({'server2': {'app_yaml_helper.py': 'fixed a bug'}}) self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0')) self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0')) assert_is_up_to_date('1-5-0') assert_is_up_to_date('2-5-0') # Revision 3. apply_update({'server2': {'app.yaml': _GenerateAppYaml('1-6-0')}}) self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0')) self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0')) self.assertEqual(3, helper.GetFirstRevisionGreaterThan('1-5-0')) assert_is_up_to_date('2-5-0') # Revision 4. apply_update({'server2': {'app.yaml': _GenerateAppYaml('1-8-0')}}) # Revision 5. apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-0-0')}}) # Revision 6. apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-2-0')}}) # Revision 7. apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-4-0')}}) # Revision 8. apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-6-0')}}) self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0')) self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0')) self.assertEqual(3, helper.GetFirstRevisionGreaterThan('1-5-0')) self.assertEqual(5, helper.GetFirstRevisionGreaterThan('1-8-0')) self.assertEqual(6, helper.GetFirstRevisionGreaterThan('2-0-0')) self.assertEqual(6, helper.GetFirstRevisionGreaterThan('2-1-0')) self.assertEqual(7, helper.GetFirstRevisionGreaterThan('2-2-0')) self.assertEqual(7, helper.GetFirstRevisionGreaterThan('2-3-0')) self.assertEqual(8, helper.GetFirstRevisionGreaterThan('2-4-0')) self.assertEqual(8, helper.GetFirstRevisionGreaterThan('2-5-0')) assert_is_up_to_date('2-6-0') assert_is_up_to_date('2-7-0')
def Get(self): object_store_creator = ObjectStoreCreator(start_empty=False) commit_tracker = CommitTracker(object_store_creator) return Response.Ok(commit_tracker.Get(self._request.path).Get())
def setUp(self): self._creator = ObjectStoreCreator(start_empty=False, store_type=TestObjectStore, disable_wrappers=True)
class GithubFileSystem(FileSystem): """FileSystem implementation which fetches resources from github. """ def __init__(self, fetcher, blobstore): self._fetcher = fetcher self._stat_object_store = ObjectStoreCreator(GithubFileSystem).Create() self._blobstore = blobstore self._version = None self._GetZip(self.Stat(ZIP_KEY).version) def _GetZip(self, version): blob = self._blobstore.Get(_MakeBlobstoreKey(version), blobstore.BLOBSTORE_GITHUB) if blob is not None: try: self._zip_file = Future(value=ZipFile(StringIO(blob))) except BadZipfile as e: self._blobstore.Delete(_MakeBlobstoreKey(version), blobstore.BLOBSTORE_GITHUB) logging.error('Bad github zip file: %s' % e) self._zip_file = Future(value=None) else: self._zip_file = Future( delegate=_AsyncFetchFutureZip(self._fetcher, self._blobstore, version, key_to_delete=self._version)) self._version = version def _ReadFile(self, path): try: zip_file = self._zip_file.Get() except Exception as e: logging.error('Github ReadFile error: %s' % e) return '' if zip_file is None: logging.error('Bad github zip file.') return '' prefix = zip_file.namelist()[0][:-1] return zip_file.read(prefix + path) def _ListDir(self, path): try: zip_file = self._zip_file.Get() except Exception as e: logging.error('Github ListDir error: %s' % e) return [] if zip_file is None: logging.error('Bad github zip file.') return [] filenames = zip_file.namelist() # Take out parent directory name (GoogleChrome-chrome-app-samples-c78a30f) filenames = [f[len(filenames[0]) - 1:] for f in filenames] # Remove the path of the directory we're listing from the filenames. filenames = [f[len(path):] for f in filenames if f != path and f.startswith(path)] # Remove all files not directly in this directory. return [f for f in filenames if f[:-1].count('/') == 0] def Read(self, paths, binary=False): version = self.Stat(ZIP_KEY).version if version != self._version: self._GetZip(version) result = {} for path in paths: if path.endswith('/'): result[path] = self._ListDir(path) else: result[path] = self._ReadFile(path) return Future(value=result) def _DefaultStat(self, path): version = 0 # TODO(kalman): we should replace all of this by wrapping the # GithubFileSystem in a CachingFileSystem. A lot of work has been put into # CFS to be robust, and GFS is missing out. # For example: the following line is wrong, but it could be moot. self._stat_object_store.Set(path, version) return StatInfo(version) def Stat(self, path): version = self._stat_object_store.Get(path).Get() if version is not None: return StatInfo(version) try: result = self._fetcher.Fetch('commits/HEAD', username=USERNAME, password=PASSWORD) except urlfetch.DownloadError as e: logging.error('GithubFileSystem Stat: %s' % e) return self._DefaultStat(path) # Check if Github authentication failed. if result.status_code == 401: logging.error('Github authentication failed for %s, falling back to ' 'unauthenticated.' % USERNAME) try: result = self._fetcher.Fetch('commits/HEAD') except urlfetch.DownloadError as e: logging.error('GithubFileSystem Stat: %s' % e) return self._DefaultStat(path) version = (json.loads(result.content).get('commit', {}) .get('tree', {}) .get('sha', None)) # Check if the JSON was valid, and set to 0 if not. if version is not None: self._stat_object_store.Set(path, version) else: logging.warning('Problem fetching commit hash from github.') return self._DefaultStat(path) return StatInfo(version)