def setUp(self):
test_fs = TestFileSystem(_TEST_DATA)
compiled_fs_factory = CompiledFileSystem.Factory(
test_fs, ObjectStoreCreator.TestFactory())
self._path_canonicalizer = PathCanonicalizer('stable',
compiled_fs_factory)
def _CreateFactory():
return CompiledFileSystem.Factory(
TestFileSystem(deepcopy(_TEST_DATA)),
ObjectStoreCreator(start_empty=False,
store_type=TestObjectStore,
disable_wrappers=True))
def setUp(self):
self._path_canonicalizer = PathCanonicalizer(
LocalFileSystem.Create(PUBLIC_TEMPLATES),
ObjectStoreCreator.ForTest(), ('.html', '.md'))
def __init__(self, fetcher, blobstore): self._fetcher = fetcher self._stat_object_store = ObjectStoreCreator(GithubFileSystem).Create() self._blobstore = blobstore self._version = None self._GetZip(self.Stat(ZIP_KEY).version)
def _CreateFactory():
return CompiledFileSystem.Factory(TestFileSystem(deepcopy(_TEST_DATA)),
ObjectStoreCreator.Factory(),
store_type=TestObjectStore)
def testRemoveNoDocs(self):
expected_nodoc = [
{
'name': 'B',
'list': [
{
'name': 'B2'
}
]
},
{
'name': 'D',
'nodoc': False
},
{
'name': 'E',
'items1': [
{
'name': 'E1',
'items': [
{
'name': 'E1.3'
}
]
},
{
'name': 'E2'
}
]
}
]
nodoc_data = [
{
'name': 'A',
'nodoc': True
},
{
'name': 'B',
'list': [
{
'name': 'B1',
'nodoc': True
},
{
'name': 'B2'
},
{
'name': 'B3',
'nodoc': True
}
]
},
{
'name': 'C',
'nodoc': True
},
{
'name': 'D',
'nodoc': False
},
{
'name': 'E',
'dict': {
'name': 'Ed',
'nodoc': True
},
'items1': [
{
'name': 'E1',
'items': [
{
'name': 'E1.1',
'nodoc': True
},
{
'name': 'E1.2',
'nodoc': True
},
{
'name': 'E1.3'
}
]
},
{
'name': 'E2'
},
{
'name': 'E3',
'nodoc': True
}
]
}
]
object_store_creator = ObjectStoreCreator(start_empty=False)
host_file_system_provider = HostFileSystemProvider(object_store_creator)
schema_processor = SchemaProcessor(_FakeReferenceResolver(),
_FakeAPIModels(),
_FakeFeaturesBundle(),
CompiledFileSystem.Factory(
object_store_creator),
host_file_system_provider.GetMaster(),
True)
schema_processor._RemoveNoDocs(nodoc_data)
self.assertEquals(expected_nodoc, nodoc_data)
def _CreateRefResolver(self, filename):
test_data = self._LoadJSON(filename)
return ReferenceResolver.Factory(_FakeAPIDataSource(test_data),
_FakeAPIModels(test_data),
ObjectStoreCreator.ForTest()).Create()
'extensions': {
'redirects.json': json.dumps({
'manifest': 'manifest.html'
}),
'manifest': {
'redirects.json': json.dumps({
'': '../manifest.html',
'more-info': 'http://lmgtfy.com'
})
}
}
}
})
redirector = Redirector(
CompiledFileSystem.Factory(file_system, ObjectStoreCreator.ForTest()),
None,
'public')
class RedirectorTest(unittest.TestCase):
def testExternalRedirection(self):
self.assertEqual(
'http://something.absolute.com/',
redirector.Redirect(HOST, 'index.html'))
self.assertEqual(
'http://lmgtfy.com',
redirector.Redirect(HOST, 'extensions/manifest/more-info'))
def testAbsoluteRedirection(self):
self.assertEqual(
'/apps/about_apps.html', redirector.Redirect(HOST, 'apps/index.html'))
def setUp(self):
self._redirector = Redirector(
CompiledFileSystem.Factory(ObjectStoreCreator.ForTest()),
file_system)
def setUp(self):
self._gfs = GithubFileSystem.ForTest(
'repo', partial(FakeURLFSFetcher, LocalFileSystem('')))
# Start and finish the repository load.
self._cgfs = CachingFileSystem(self._gfs, ObjectStoreCreator.ForTest())
def setUp(self):
self._branch_util = BranchUtility(
os.path.join('branch_utility', 'first.json'),
os.path.join('branch_utility', 'second.json'),
FakeUrlFetcher(Server2Path('test_data')),
ObjectStoreCreator.ForTest())
def __init__(self, fetch_url, fetcher, object_store=None):
self._fetch_url = fetch_url
self._fetcher = fetcher
if object_store is None:
object_store = ObjectStoreCreator(BranchUtility).Create()
self._object_store = object_store
def setUp(self):
self._refresh_tracker = RefreshTracker(ObjectStoreCreator.ForTest())
def testCaching(self):
fake_fs = TestFileSystem({
'bob': {
'bob0': 'bob/bob0 contents',
'bob1': 'bob/bob1 contents',
'bob2': 'bob/bob2 contents',
'bob3': 'bob/bob3 contents',
}
})
file_system = CachingFileSystem(fake_fs,
ObjectStoreCreator.TestFactory())
self.assertEqual('bob/bob0 contents',
file_system.ReadSingle('bob/bob0'))
self.assertTrue(fake_fs.CheckAndReset(read_count=1, stat_count=1))
# Resource has been cached, so test resource is not re-fetched.
self.assertEqual('bob/bob0 contents',
file_system.ReadSingle('bob/bob0'))
self.assertTrue(fake_fs.CheckAndReset())
# Test if the Stat version is the same the resource is not re-fetched.
file_system._stat_object_store.Del('bob/bob0')
self.assertEqual('bob/bob0 contents',
file_system.ReadSingle('bob/bob0'))
self.assertTrue(fake_fs.CheckAndReset(stat_count=1))
# Test if there is a newer version, the resource is re-fetched.
file_system._stat_object_store.Del('bob/bob0')
fake_fs.IncrementStat()
self.assertEqual('bob/bob0 contents',
file_system.ReadSingle('bob/bob0'))
self.assertTrue(fake_fs.CheckAndReset(read_count=1, stat_count=1))
# Test directory and subdirectory stats are cached.
file_system._stat_object_store.Del('bob/bob0')
file_system._read_object_store.Del('bob/bob0')
file_system._stat_object_store.Del('bob/bob1')
fake_fs.IncrementStat()
self.assertEqual('bob/bob1 contents',
file_system.ReadSingle('bob/bob1'))
self.assertEqual('bob/bob0 contents',
file_system.ReadSingle('bob/bob0'))
self.assertTrue(fake_fs.CheckAndReset(read_count=2, stat_count=1))
self.assertEqual('bob/bob1 contents',
file_system.ReadSingle('bob/bob1'))
self.assertTrue(fake_fs.CheckAndReset())
# Test a more recent parent directory doesn't force a refetch of children.
file_system._read_object_store.Del('bob/bob0')
file_system._read_object_store.Del('bob/bob1')
self.assertEqual('bob/bob1 contents',
file_system.ReadSingle('bob/bob1'))
self.assertEqual('bob/bob2 contents',
file_system.ReadSingle('bob/bob2'))
self.assertEqual('bob/bob3 contents',
file_system.ReadSingle('bob/bob3'))
self.assertTrue(fake_fs.CheckAndReset(read_count=3))
fake_fs.IncrementStat(path='bob/')
self.assertEqual('bob/bob1 contents',
file_system.ReadSingle('bob/bob1'))
self.assertEqual('bob/bob2 contents',
file_system.ReadSingle('bob/bob2'))
self.assertEqual('bob/bob3 contents',
file_system.ReadSingle('bob/bob3'))
self.assertTrue(fake_fs.CheckAndReset())
file_system._stat_object_store.Del('bob/bob0')
self.assertEqual('bob/bob0 contents',
file_system.ReadSingle('bob/bob0'))
self.assertTrue(fake_fs.CheckAndReset(read_count=1, stat_count=1))
self.assertEqual('bob/bob0 contents',
file_system.ReadSingle('bob/bob0'))
self.assertTrue(fake_fs.CheckAndReset())
def _CreateRefResolver(self, filename):
data_source = FakeAPIAndListDataSource(self._LoadJSON(filename))
return ReferenceResolver.Factory(
data_source, data_source, ObjectStoreCreator.ForTest()).Create()
def testInstanceMethods(self):
test_data = {
'app.yaml': _GenerateAppYaml('1-0'),
'app_yaml_helper.py': 'Copyright notice etc'
}
updates = []
# Pass a specific file system at head to the HostFileSystemProvider so that
# we know it's always going to be backed by a MockFileSystem. The Provider
# may decide to wrap it in caching etc.
file_system_at_head = MockFileSystem(
TestFileSystem(test_data, relative_to=SERVER2))
def apply_update(update):
update = MoveTo(SERVER2, update)
file_system_at_head.Update(update)
updates.append(update)
def host_file_system_constructor(branch, commit=None):
self.assertEqual('master', branch)
self.assertTrue(commit is not None)
return MockFileSystem.Create(
TestFileSystem(test_data, relative_to=SERVER2), updates[:commit])
object_store_creator = ObjectStoreCreator.ForTest()
host_file_system_provider = HostFileSystemProvider(
object_store_creator,
default_master_instance=file_system_at_head,
constructor_for_test=host_file_system_constructor)
helper = AppYamlHelper(object_store_creator, host_file_system_provider)
def assert_is_up_to_date(version):
self.assertTrue(helper.IsUpToDate(version),
'%s is not up to date' % version)
self.assertRaises(ValueError,
helper.GetFirstRevisionGreaterThan, version)
self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0'))
assert_is_up_to_date('1-0-0')
assert_is_up_to_date('1-5-0')
# Revision 1.
apply_update({
'app.yaml': _GenerateAppYaml('1-5-0')
})
self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0'))
self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0'))
assert_is_up_to_date('1-5-0')
assert_is_up_to_date('2-5-0')
# Revision 2.
apply_update({
'app_yaml_helper.py': 'fixed a bug'
})
self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0'))
self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0'))
assert_is_up_to_date('1-5-0')
assert_is_up_to_date('2-5-0')
# Revision 3.
apply_update({
'app.yaml': _GenerateAppYaml('1-6-0')
})
self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0'))
self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0'))
self.assertEqual(3, helper.GetFirstRevisionGreaterThan('1-5-0'))
assert_is_up_to_date('2-5-0')
# Revision 4.
apply_update({
'app.yaml': _GenerateAppYaml('1-8-0')
})
# Revision 5.
apply_update({
'app.yaml': _GenerateAppYaml('2-0-0')
})
# Revision 6.
apply_update({
'app.yaml': _GenerateAppYaml('2-2-0')
})
# Revision 7.
apply_update({
'app.yaml': _GenerateAppYaml('2-4-0')
})
# Revision 8.
apply_update({
'app.yaml': _GenerateAppYaml('2-6-0')
})
self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0'))
self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0'))
self.assertEqual(3, helper.GetFirstRevisionGreaterThan('1-5-0'))
self.assertEqual(5, helper.GetFirstRevisionGreaterThan('1-8-0'))
self.assertEqual(6, helper.GetFirstRevisionGreaterThan('2-0-0'))
self.assertEqual(6, helper.GetFirstRevisionGreaterThan('2-1-0'))
self.assertEqual(7, helper.GetFirstRevisionGreaterThan('2-2-0'))
self.assertEqual(7, helper.GetFirstRevisionGreaterThan('2-3-0'))
self.assertEqual(8, helper.GetFirstRevisionGreaterThan('2-4-0'))
self.assertEqual(8, helper.GetFirstRevisionGreaterThan('2-5-0'))
assert_is_up_to_date('2-6-0')
assert_is_up_to_date('2-7-0')
def testInlineDocs(self):
schema = {
'namespace': 'storage',
'properties': {
'key2': {
'description': 'second key',
'$ref': 'Key'
},
'key1': {
'description': 'first key',
'$ref': 'Key'
}
},
'types': [
{
'inline_doc': True,
'type': 'string',
'id': 'Key', # Should be inlined into both properties and be removed
# from types.
'description': 'This is a key.', # This description should disappear.
'marker': True # This should appear three times in the output.
},
{
'items': {
'$ref': 'Key'
},
'type': 'array',
'id': 'KeyList',
'description': 'A list of keys'
}
]
}
expected_schema = {
'namespace': 'storage',
'properties': {
'key2': {
'marker': True,
'type': 'string',
'description': 'second key'
},
'key1': {
'marker': True,
'type': 'string',
'description': 'first key'
}
},
'types': [
{
'items': {
'marker': True,
'type': 'string'
},
'type': 'array',
'id': 'KeyList',
'description': 'A list of keys'
}
]
}
object_store_creator = ObjectStoreCreator(start_empty=False)
host_file_system_provider = HostFileSystemProvider(object_store_creator)
schema_processor = SchemaProcessor(_FakeReferenceResolver(),
_FakeAPIModels(),
_FakeFeaturesBundle(),
CompiledFileSystem.Factory(
object_store_creator),
host_file_system_provider.GetMaster(),
False)
inlined_schema = deepcopy(schema)
schema_processor._InlineDocs(inlined_schema)
self.assertEqual(expected_schema, inlined_schema)
def setUp(self):
self._base_path = os.path.join(sys.path[0], 'test_data',
'sidenav_data_source')
self._compiled_fs_factory = CompiledFileSystem.Factory(
LocalFileSystem(self._base_path), ObjectStoreCreator.Factory())
def testInstanceMethods(self):
test_data = {
'server2': {
'app.yaml': _GenerateAppYaml('1-0'),
'app_yaml_helper.py': 'Copyright notice etc'
}
}
updates = []
file_system_at_head = MockFileSystem(TestFileSystem(test_data))
def apply_update(update):
file_system_at_head.Update(update)
updates.append(update)
assert_true = self.assertTrue
class TestDelegate(AppYamlHelper.Delegate):
def GetHostFileSystemForRevision(self, revision):
assert_true(revision is not None)
assert_true(revision >= 0)
return MockFileSystem.Create(TestFileSystem(test_data),
updates[:revision])
helper = AppYamlHelper(
'server2/app.yaml', file_system_at_head, TestDelegate(),
ObjectStoreCreator.ForTest(disable_wrappers=False))
def assert_is_up_to_date(version):
self.assertTrue(helper.IsUpToDate(version),
'%s is not up to date' % version)
self.assertRaises(ValueError, helper.GetFirstRevisionGreaterThan,
version)
self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0'))
assert_is_up_to_date('1-0-0')
assert_is_up_to_date('1-5-0')
# Revision 1.
apply_update({'server2': {'app.yaml': _GenerateAppYaml('1-5-0')}})
self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0'))
self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0'))
assert_is_up_to_date('1-5-0')
assert_is_up_to_date('2-5-0')
# Revision 2.
apply_update({'server2': {'app_yaml_helper.py': 'fixed a bug'}})
self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0'))
self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0'))
assert_is_up_to_date('1-5-0')
assert_is_up_to_date('2-5-0')
# Revision 3.
apply_update({'server2': {'app.yaml': _GenerateAppYaml('1-6-0')}})
self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0'))
self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0'))
self.assertEqual(3, helper.GetFirstRevisionGreaterThan('1-5-0'))
assert_is_up_to_date('2-5-0')
# Revision 4.
apply_update({'server2': {'app.yaml': _GenerateAppYaml('1-8-0')}})
# Revision 5.
apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-0-0')}})
# Revision 6.
apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-2-0')}})
# Revision 7.
apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-4-0')}})
# Revision 8.
apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-6-0')}})
self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0'))
self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0'))
self.assertEqual(3, helper.GetFirstRevisionGreaterThan('1-5-0'))
self.assertEqual(5, helper.GetFirstRevisionGreaterThan('1-8-0'))
self.assertEqual(6, helper.GetFirstRevisionGreaterThan('2-0-0'))
self.assertEqual(6, helper.GetFirstRevisionGreaterThan('2-1-0'))
self.assertEqual(7, helper.GetFirstRevisionGreaterThan('2-2-0'))
self.assertEqual(7, helper.GetFirstRevisionGreaterThan('2-3-0'))
self.assertEqual(8, helper.GetFirstRevisionGreaterThan('2-4-0'))
self.assertEqual(8, helper.GetFirstRevisionGreaterThan('2-5-0'))
assert_is_up_to_date('2-6-0')
assert_is_up_to_date('2-7-0')
def Get(self):
object_store_creator = ObjectStoreCreator(start_empty=False)
commit_tracker = CommitTracker(object_store_creator)
return Response.Ok(commit_tracker.Get(self._request.path).Get())
def setUp(self):
self._creator = ObjectStoreCreator(start_empty=False,
store_type=TestObjectStore,
disable_wrappers=True)
class GithubFileSystem(FileSystem):
"""FileSystem implementation which fetches resources from github.
"""
def __init__(self, fetcher, blobstore):
self._fetcher = fetcher
self._stat_object_store = ObjectStoreCreator(GithubFileSystem).Create()
self._blobstore = blobstore
self._version = None
self._GetZip(self.Stat(ZIP_KEY).version)
def _GetZip(self, version):
blob = self._blobstore.Get(_MakeBlobstoreKey(version),
blobstore.BLOBSTORE_GITHUB)
if blob is not None:
try:
self._zip_file = Future(value=ZipFile(StringIO(blob)))
except BadZipfile as e:
self._blobstore.Delete(_MakeBlobstoreKey(version),
blobstore.BLOBSTORE_GITHUB)
logging.error('Bad github zip file: %s' % e)
self._zip_file = Future(value=None)
else:
self._zip_file = Future(
delegate=_AsyncFetchFutureZip(self._fetcher,
self._blobstore,
version,
key_to_delete=self._version))
self._version = version
def _ReadFile(self, path):
try:
zip_file = self._zip_file.Get()
except Exception as e:
logging.error('Github ReadFile error: %s' % e)
return ''
if zip_file is None:
logging.error('Bad github zip file.')
return ''
prefix = zip_file.namelist()[0][:-1]
return zip_file.read(prefix + path)
def _ListDir(self, path):
try:
zip_file = self._zip_file.Get()
except Exception as e:
logging.error('Github ListDir error: %s' % e)
return []
if zip_file is None:
logging.error('Bad github zip file.')
return []
filenames = zip_file.namelist()
# Take out parent directory name (GoogleChrome-chrome-app-samples-c78a30f)
filenames = [f[len(filenames[0]) - 1:] for f in filenames]
# Remove the path of the directory we're listing from the filenames.
filenames = [f[len(path):] for f in filenames
if f != path and f.startswith(path)]
# Remove all files not directly in this directory.
return [f for f in filenames if f[:-1].count('/') == 0]
def Read(self, paths, binary=False):
version = self.Stat(ZIP_KEY).version
if version != self._version:
self._GetZip(version)
result = {}
for path in paths:
if path.endswith('/'):
result[path] = self._ListDir(path)
else:
result[path] = self._ReadFile(path)
return Future(value=result)
def _DefaultStat(self, path):
version = 0
# TODO(kalman): we should replace all of this by wrapping the
# GithubFileSystem in a CachingFileSystem. A lot of work has been put into
# CFS to be robust, and GFS is missing out.
# For example: the following line is wrong, but it could be moot.
self._stat_object_store.Set(path, version)
return StatInfo(version)
def Stat(self, path):
version = self._stat_object_store.Get(path).Get()
if version is not None:
return StatInfo(version)
try:
result = self._fetcher.Fetch('commits/HEAD',
username=USERNAME,
password=PASSWORD)
except urlfetch.DownloadError as e:
logging.error('GithubFileSystem Stat: %s' % e)
return self._DefaultStat(path)
# Check if Github authentication failed.
if result.status_code == 401:
logging.error('Github authentication failed for %s, falling back to '
'unauthenticated.' % USERNAME)
try:
result = self._fetcher.Fetch('commits/HEAD')
except urlfetch.DownloadError as e:
logging.error('GithubFileSystem Stat: %s' % e)
return self._DefaultStat(path)
version = (json.loads(result.content).get('commit', {})
.get('tree', {})
.get('sha', None))
# Check if the JSON was valid, and set to 0 if not.
if version is not None:
self._stat_object_store.Set(path, version)
else:
logging.warning('Problem fetching commit hash from github.')
return self._DefaultStat(path)
return StatInfo(version)