def _GetSafeServerInstance(self):
'''Returns a ServerInstance with a host file system at a safe revision,
meaning the last revision that the current running version of the server
existed.
'''
delegate = self._delegate
server_instance_at_head = self._CreateServerInstance(None)
app_yaml_handler = AppYamlHelper(
svn_constants.APP_YAML_PATH,
server_instance_at_head.host_file_system,
server_instance_at_head.object_store_creator,
server_instance_at_head.host_file_system_creator)
if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()):
# TODO(kalman): return a new ServerInstance at an explicit revision in
# case the HEAD version changes underneath us.
return server_instance_at_head
# The version in app.yaml is greater than the currently running app's.
# The safe version is the one before it changed.
safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan(
delegate.GetAppVersion()) - 1
logging.info('cron: app version %s is out of date, safe is %s' %
(delegate.GetAppVersion(), safe_revision))
return self._CreateServerInstance(safe_revision)
def _GetSafeServerInstance(self):
'''Returns a ServerInstance with a host file system at a safe revision,
meaning the last revision that the current running version of the server
existed.
'''
delegate = self._delegate
# IMPORTANT: Get a ServerInstance pinned to the most recent revision, not
# HEAD. These cron jobs take a while and run very frequently such that
# there is usually one running at any given time, and eventually a file
# that we're dealing with will change underneath it, putting the server in
# an undefined state.
server_instance_near_head = self._CreateServerInstance(
self._GetMostRecentRevision())
app_yaml_handler = AppYamlHelper(
server_instance_near_head.object_store_creator,
server_instance_near_head.host_file_system_provider)
if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()):
return server_instance_near_head
# The version in app.yaml is greater than the currently running app's.
# The safe version is the one before it changed.
safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan(
delegate.GetAppVersion()) - 1
_cronlog.info('app version %s is out of date, safe is %s',
delegate.GetAppVersion(), safe_revision)
return self._CreateServerInstance(safe_revision)
def app_yaml_update(version):
return {
'docs': {
'server2': {
'app.yaml': AppYamlHelper.GenerateAppYaml(version)
}
}
}
def GetAppVersion():
if 'CURRENT_VERSION_ID' in os.environ:
# The version ID looks like 2-0-25.36712548, we only want the 2-0-25.
return os.environ['CURRENT_VERSION_ID'].split('.', 1)[0]
# Not running on appengine, get it from the app.yaml file ourselves.
app_yaml_path = os.path.join(os.path.split(__file__)[0], 'app.yaml')
with open(app_yaml_path, 'r') as app_yaml:
return AppYamlHelper.ExtractVersion(app_yaml.read())
def _CheckYamlConsistency(input_api, output_api):
app_yaml_path = os.path.join(input_api.PresubmitLocalPath(), 'app.yaml')
cron_yaml_path = os.path.join(input_api.PresubmitLocalPath(), 'cron.yaml')
if not (app_yaml_path in input_api.AbsoluteLocalPaths() or
cron_yaml_path in input_api.AbsoluteLocalPaths()):
return []
AppYamlHelper = _ImportAppYamlHelper(input_api)
app_yaml_version = AppYamlHelper.ExtractVersion(
input_api.ReadFile(app_yaml_path))
cron_yaml_version = AppYamlHelper.ExtractVersion(
input_api.ReadFile(cron_yaml_path), key='target')
if app_yaml_version == cron_yaml_version:
return []
return [output_api.PresubmitError(
'Versions of app.yaml (%s) and cron.yaml (%s) must match' % (
app_yaml_version, cron_yaml_version))]
def GetAppVersionNonMemoized():
if 'CURRENT_VERSION_ID' in os.environ:
# The version ID looks like 2-0-25.36712548 or 2-0-25.23/223; we only
# want the 2-0-25.
return re.compile('[./]').split(os.environ['CURRENT_VERSION_ID'])[0]
# Not running on appengine, get it from the app.yaml file ourselves.
app_yaml_path = os.path.join(os.path.split(__file__)[0], 'app.yaml')
with open(app_yaml_path, 'r') as app_yaml:
return AppYamlHelper.ExtractVersion(app_yaml.read())
def _GetSafeServerInstance(self):
'''Returns a ServerInstance with a host file system at a safe revision,
meaning the last revision that the current running version of the server
existed.
'''
channel = self._channel
delegate = self._delegate
server_instance_at_head = self._CreateServerInstance(channel, None)
get_branch_for_channel = self._GetBranchForChannel
class AppYamlHelperDelegate(AppYamlHelper.Delegate):
def GetHostFileSystemForRevision(self, revision):
return delegate.CreateHostFileSystemForBranchAndRevision(
get_branch_for_channel(channel),
revision)
app_yaml_handler = AppYamlHelper(
svn_constants.APP_YAML_PATH,
server_instance_at_head.host_file_system,
AppYamlHelperDelegate(),
server_instance_at_head.object_store_creator)
if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()):
# TODO(kalman): return a new ServerInstance at an explicit revision in
# case the HEAD version changes underneath us.
return server_instance_at_head
# The version in app.yaml is greater than the currently running app's.
# The safe version is the one before it changed.
safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan(
delegate.GetAppVersion()) - 1
logging.info('cron/%s: app version %s is out of date, safe is %s' % (
channel, delegate.GetAppVersion(), safe_revision))
return self._CreateServerInstance(channel, safe_revision)
def testInstanceMethods(self):
test_data = {
'server2': {
'app.yaml': _GenerateAppYaml('1-0'),
'app_yaml_helper.py': 'Copyright notice etc'
}
}
updates = []
# Pass a specific file system at head to the HostFileSystemProvider so that
# we know it's always going to be backed by a MockFileSystem. The Provider
# may decide to wrap it in caching etc.
file_system_at_head = MockFileSystem(TestFileSystem(test_data))
def apply_update(update):
file_system_at_head.Update(update)
updates.append(update)
def host_file_system_constructor(branch, revision=None):
self.assertEqual('trunk', branch)
self.assertTrue(revision is not None)
return MockFileSystem.Create(TestFileSystem(test_data),
updates[:revision])
object_store_creator = ObjectStoreCreator.ForTest()
host_file_system_provider = HostFileSystemProvider(
object_store_creator,
default_trunk_instance=file_system_at_head,
constructor_for_test=host_file_system_constructor)
helper = AppYamlHelper('server2/app.yaml', object_store_creator,
host_file_system_provider)
def assert_is_up_to_date(version):
self.assertTrue(helper.IsUpToDate(version),
'%s is not up to date' % version)
self.assertRaises(ValueError, helper.GetFirstRevisionGreaterThan,
version)
self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0'))
assert_is_up_to_date('1-0-0')
assert_is_up_to_date('1-5-0')
# Revision 1.
apply_update({'server2': {'app.yaml': _GenerateAppYaml('1-5-0')}})
self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0'))
self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0'))
assert_is_up_to_date('1-5-0')
assert_is_up_to_date('2-5-0')
# Revision 2.
apply_update({'server2': {'app_yaml_helper.py': 'fixed a bug'}})
self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0'))
self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0'))
assert_is_up_to_date('1-5-0')
assert_is_up_to_date('2-5-0')
# Revision 3.
apply_update({'server2': {'app.yaml': _GenerateAppYaml('1-6-0')}})
self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0'))
self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0'))
self.assertEqual(3, helper.GetFirstRevisionGreaterThan('1-5-0'))
assert_is_up_to_date('2-5-0')
# Revision 4.
apply_update({'server2': {'app.yaml': _GenerateAppYaml('1-8-0')}})
# Revision 5.
apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-0-0')}})
# Revision 6.
apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-2-0')}})
# Revision 7.
apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-4-0')}})
# Revision 8.
apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-6-0')}})
self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0'))
self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0'))
self.assertEqual(3, helper.GetFirstRevisionGreaterThan('1-5-0'))
self.assertEqual(5, helper.GetFirstRevisionGreaterThan('1-8-0'))
self.assertEqual(6, helper.GetFirstRevisionGreaterThan('2-0-0'))
self.assertEqual(6, helper.GetFirstRevisionGreaterThan('2-1-0'))
self.assertEqual(7, helper.GetFirstRevisionGreaterThan('2-2-0'))
self.assertEqual(7, helper.GetFirstRevisionGreaterThan('2-3-0'))
self.assertEqual(8, helper.GetFirstRevisionGreaterThan('2-4-0'))
self.assertEqual(8, helper.GetFirstRevisionGreaterThan('2-5-0'))
assert_is_up_to_date('2-6-0')
assert_is_up_to_date('2-7-0')
def testSafeRevision(self):
test_data = {
'docs': {
'examples': {
'examples.txt': 'examples.txt contents'
},
'server2': {
'app.yaml': AppYamlHelper.GenerateAppYaml('2-0-8')
},
'static': {
'static.txt': 'static.txt contents'
},
'templates': {
'public': {
'apps': {
'storage.html': 'storage.html contents'
},
'extensions': {
'storage.html': 'storage.html contents'
},
}
}
}
}
updates = []
def app_yaml_update(version):
return {
'docs': {
'server2': {
'app.yaml': AppYamlHelper.GenerateAppYaml(version)
}
}
}
def storage_html_update(update):
return {
'docs': {
'templates': {
'public': {
'apps': {
'storage.html': update
}
}
}
}
}
def static_txt_update(update):
return {'docs': {'static': {'static.txt': update}}}
app_yaml_path = 'docs/server2/app.yaml'
storage_html_path = 'docs/templates/public/apps/storage.html'
static_txt_path = 'docs/static/static.txt'
def create_file_system(revision=None):
'''Creates a MockFileSystem at |revision| by applying that many |updates|
to it.
'''
mock_file_system = MockFileSystem(TestFileSystem(test_data))
for update in updates[:revision]:
mock_file_system.Update(update)
return mock_file_system
delegate = _TestDelegate(create_file_system)
delegate.SetAppVersion('2-0-8')
file_systems = delegate.file_systems
# No updates applied yet.
CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get()
self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'),
file_systems[-1].ReadSingle(app_yaml_path))
self.assertEqual('storage.html contents',
file_systems[-1].ReadSingle(storage_html_path))
# Apply updates to storage.html.
updates.append(storage_html_update('interim contents'))
updates.append(storage_html_update('new contents'))
CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get()
self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'),
file_systems[-1].ReadSingle(app_yaml_path))
self.assertEqual('new contents',
file_systems[-1].ReadSingle(storage_html_path))
# Apply several updates to storage.html and app.yaml. The file system
# should be pinned at the version before app.yaml changed.
updates.append(storage_html_update('stuck here contents'))
double_update = storage_html_update('newer contents')
double_update.update(app_yaml_update('2-0-10'))
updates.append(double_update)
updates.append(storage_html_update('never gonna reach here'))
CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get()
self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'),
file_systems[-1].ReadSingle(app_yaml_path))
self.assertEqual('stuck here contents',
file_systems[-1].ReadSingle(storage_html_path))
# Further pushes to storage.html will keep it pinned.
updates.append(storage_html_update('y u not update!'))
CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get()
self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'),
file_systems[-1].ReadSingle(app_yaml_path))
self.assertEqual('stuck here contents',
file_systems[-1].ReadSingle(storage_html_path))
# Likewise app.yaml.
updates.append(app_yaml_update('2-1-0'))
CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get()
self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'),
file_systems[-1].ReadSingle(app_yaml_path))
self.assertEqual('stuck here contents',
file_systems[-1].ReadSingle(storage_html_path))
# And updates to other content won't happen either.
updates.append(static_txt_update('important content!'))
CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get()
self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'),
file_systems[-1].ReadSingle(app_yaml_path))
self.assertEqual('stuck here contents',
file_systems[-1].ReadSingle(storage_html_path))
self.assertEqual('static.txt contents',
file_systems[-1].ReadSingle(static_txt_path))
# Lastly - when the app version changes, everything should no longer be
# pinned.
delegate.SetAppVersion('2-1-0')
CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get()
self.assertEqual(AppYamlHelper.GenerateAppYaml('2-1-0'),
file_systems[-1].ReadSingle(app_yaml_path))
self.assertEqual('y u not update!',
file_systems[-1].ReadSingle(storage_html_path))
self.assertEqual('important content!',
file_systems[-1].ReadSingle(static_txt_path))
def testSafeRevision(self):
test_data = {
'api': {
'_api_features.json': '{}',
'_manifest_features.json': '{}',
'_permission_features.json': '{}',
},
'docs': {
'examples': {
'examples.txt': 'examples.txt contents'
},
'server2': {
'app.yaml': AppYamlHelper.GenerateAppYaml('2-0-8')
},
'static': {
'static.txt': 'static.txt contents'
},
'templates': {
'articles': {
'activeTab.html': 'activeTab.html contents'
},
'intros': {
'browserAction.html': 'activeTab.html contents'
},
'private': {
'table_of_contents.html': 'table_of_contents.html contents',
},
'public': {
'apps': {
'storage.html': '<h1>storage.html</h1> contents'
},
'extensions': {
'storage.html': '<h1>storage.html</h1> contents'
},
},
'json': {
'chrome_sidenav.json': '{}',
'content_providers.json': ReadFile(CONTENT_PROVIDERS),
'manifest.json': '{}',
'permissions.json': '{}',
'strings.json': '{}',
'whats_new.json': '{}',
},
}
}
}
updates = []
def app_yaml_update(version):
return MoveTo(SERVER2, {
'app.yaml': AppYamlHelper.GenerateAppYaml(version)
})
def storage_html_update(update):
return MoveTo(PUBLIC_TEMPLATES, {
'apps': {'storage.html': update}
})
def static_txt_update(update):
return MoveTo(STATIC_DOCS, {
'static.txt': update
})
storage_html_path = PUBLIC_TEMPLATES + 'apps/storage.html'
static_txt_path = STATIC_DOCS + 'static.txt'
def create_file_system(revision=None):
'''Creates a MockFileSystem at |revision| by applying that many |updates|
to it.
'''
mock_file_system = MockFileSystem(
TestFileSystem(test_data, relative_to=CHROME_EXTENSIONS))
updates_for_revision = (
updates if revision is None else updates[:int(revision)])
for update in updates_for_revision:
mock_file_system.Update(update)
return mock_file_system
delegate = _TestDelegate(create_file_system)
delegate.SetAppVersion('2-0-8')
file_systems = delegate.file_systems
# No updates applied yet.
CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get()
self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'),
file_systems[-1].ReadSingle(APP_YAML).Get())
self.assertEqual('<h1>storage.html</h1> contents',
file_systems[-1].ReadSingle(storage_html_path).Get())
# Apply updates to storage.html.
updates.append(storage_html_update('interim contents'))
updates.append(storage_html_update('<h1>new</h1> contents'))
CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get()
self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'),
file_systems[-1].ReadSingle(APP_YAML).Get())
self.assertEqual('<h1>new</h1> contents',
file_systems[-1].ReadSingle(storage_html_path).Get())
# Apply several updates to storage.html and app.yaml. The file system
# should be pinned at the version before app.yaml changed.
updates.append(storage_html_update('<h1>stuck here</h1> contents'))
double_update = storage_html_update('<h1>newer</h1> contents')
double_update.update(app_yaml_update('2-0-10'))
updates.append(double_update)
updates.append(storage_html_update('never gonna reach here'))
CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get()
self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'),
file_systems[-1].ReadSingle(APP_YAML).Get())
self.assertEqual('<h1>stuck here</h1> contents',
file_systems[-1].ReadSingle(storage_html_path).Get())
# Further pushes to storage.html will keep it pinned.
updates.append(storage_html_update('<h1>y</h1> u not update!'))
CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get()
self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'),
file_systems[-1].ReadSingle(APP_YAML).Get())
self.assertEqual('<h1>stuck here</h1> contents',
file_systems[-1].ReadSingle(storage_html_path).Get())
# Likewise app.yaml.
updates.append(app_yaml_update('2-1-0'))
CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get()
self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'),
file_systems[-1].ReadSingle(APP_YAML).Get())
self.assertEqual('<h1>stuck here</h1> contents',
file_systems[-1].ReadSingle(storage_html_path).Get())
# And updates to other content won't happen either.
updates.append(static_txt_update('important content!'))
CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get()
self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'),
file_systems[-1].ReadSingle(APP_YAML).Get())
self.assertEqual('<h1>stuck here</h1> contents',
file_systems[-1].ReadSingle(storage_html_path).Get())
self.assertEqual('static.txt contents',
file_systems[-1].ReadSingle(static_txt_path).Get())
# Lastly - when the app version changes, everything should no longer be
# pinned.
delegate.SetAppVersion('2-1-0')
CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get()
self.assertEqual(AppYamlHelper.GenerateAppYaml('2-1-0'),
file_systems[-1].ReadSingle(APP_YAML).Get())
self.assertEqual('<h1>y</h1> u not update!',
file_systems[-1].ReadSingle(storage_html_path).Get())
self.assertEqual('important content!',
file_systems[-1].ReadSingle(static_txt_path).Get())
def app_yaml_update(version):
return MoveTo(SERVER2, {
'app.yaml': AppYamlHelper.GenerateAppYaml(version)
})
def testInstanceMethods(self):
test_data = {
'server2': {
'app.yaml': _GenerateAppYaml('1-0'),
'app_yaml_helper.py': 'Copyright notice etc'
}
}
updates = []
file_system_at_head = MockFileSystem(TestFileSystem(test_data))
def apply_update(update):
file_system_at_head.Update(update)
updates.append(update)
def constructor(branch, revision=None):
return MockFileSystem.Create(TestFileSystem(test_data),
updates[:revision])
host_file_system_creator = HostFileSystemCreator(
ObjectStoreCreator.ForTest(), constructor_for_test=constructor)
helper = AppYamlHelper(
'server2/app.yaml', file_system_at_head,
ObjectStoreCreator.ForTest(disable_wrappers=False),
host_file_system_creator)
def assert_is_up_to_date(version):
self.assertTrue(helper.IsUpToDate(version),
'%s is not up to date' % version)
self.assertRaises(ValueError, helper.GetFirstRevisionGreaterThan,
version)
self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0'))
assert_is_up_to_date('1-0-0')
assert_is_up_to_date('1-5-0')
# Revision 1.
apply_update({'server2': {'app.yaml': _GenerateAppYaml('1-5-0')}})
self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0'))
self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0'))
assert_is_up_to_date('1-5-0')
assert_is_up_to_date('2-5-0')
# Revision 2.
apply_update({'server2': {'app_yaml_helper.py': 'fixed a bug'}})
self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0'))
self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0'))
assert_is_up_to_date('1-5-0')
assert_is_up_to_date('2-5-0')
# Revision 3.
apply_update({'server2': {'app.yaml': _GenerateAppYaml('1-6-0')}})
self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0'))
self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0'))
self.assertEqual(3, helper.GetFirstRevisionGreaterThan('1-5-0'))
assert_is_up_to_date('2-5-0')
# Revision 4.
apply_update({'server2': {'app.yaml': _GenerateAppYaml('1-8-0')}})
# Revision 5.
apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-0-0')}})
# Revision 6.
apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-2-0')}})
# Revision 7.
apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-4-0')}})
# Revision 8.
apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-6-0')}})
self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0'))
self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0'))
self.assertEqual(3, helper.GetFirstRevisionGreaterThan('1-5-0'))
self.assertEqual(5, helper.GetFirstRevisionGreaterThan('1-8-0'))
self.assertEqual(6, helper.GetFirstRevisionGreaterThan('2-0-0'))
self.assertEqual(6, helper.GetFirstRevisionGreaterThan('2-1-0'))
self.assertEqual(7, helper.GetFirstRevisionGreaterThan('2-2-0'))
self.assertEqual(7, helper.GetFirstRevisionGreaterThan('2-3-0'))
self.assertEqual(8, helper.GetFirstRevisionGreaterThan('2-4-0'))
self.assertEqual(8, helper.GetFirstRevisionGreaterThan('2-5-0'))
assert_is_up_to_date('2-6-0')
assert_is_up_to_date('2-7-0')
