def _GetSafeServerInstance(self): '''Returns a ServerInstance with a host file system at a safe revision, meaning the last revision that the current running version of the server existed. ''' delegate = self._delegate server_instance_at_head = self._CreateServerInstance(None) app_yaml_handler = AppYamlHelper( svn_constants.APP_YAML_PATH, server_instance_at_head.host_file_system, server_instance_at_head.object_store_creator, server_instance_at_head.host_file_system_creator) if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()): # TODO(kalman): return a new ServerInstance at an explicit revision in # case the HEAD version changes underneath us. return server_instance_at_head # The version in app.yaml is greater than the currently running app's. # The safe version is the one before it changed. safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan( delegate.GetAppVersion()) - 1 logging.info('cron: app version %s is out of date, safe is %s' % (delegate.GetAppVersion(), safe_revision)) return self._CreateServerInstance(safe_revision)
def _GetSafeServerInstance(self): '''Returns a ServerInstance with a host file system at a safe revision, meaning the last revision that the current running version of the server existed. ''' delegate = self._delegate # IMPORTANT: Get a ServerInstance pinned to the most recent revision, not # HEAD. These cron jobs take a while and run very frequently such that # there is usually one running at any given time, and eventually a file # that we're dealing with will change underneath it, putting the server in # an undefined state. server_instance_near_head = self._CreateServerInstance( self._GetMostRecentRevision()) app_yaml_handler = AppYamlHelper( server_instance_near_head.object_store_creator, server_instance_near_head.host_file_system_provider) if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()): return server_instance_near_head # The version in app.yaml is greater than the currently running app's. # The safe version is the one before it changed. safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan( delegate.GetAppVersion()) - 1 _cronlog.info('app version %s is out of date, safe is %s', delegate.GetAppVersion(), safe_revision) return self._CreateServerInstance(safe_revision)
def app_yaml_update(version): return { 'docs': { 'server2': { 'app.yaml': AppYamlHelper.GenerateAppYaml(version) } } }
def GetAppVersion(): if 'CURRENT_VERSION_ID' in os.environ: # The version ID looks like 2-0-25.36712548, we only want the 2-0-25. return os.environ['CURRENT_VERSION_ID'].split('.', 1)[0] # Not running on appengine, get it from the app.yaml file ourselves. app_yaml_path = os.path.join(os.path.split(__file__)[0], 'app.yaml') with open(app_yaml_path, 'r') as app_yaml: return AppYamlHelper.ExtractVersion(app_yaml.read())
def _CheckYamlConsistency(input_api, output_api): app_yaml_path = os.path.join(input_api.PresubmitLocalPath(), 'app.yaml') cron_yaml_path = os.path.join(input_api.PresubmitLocalPath(), 'cron.yaml') if not (app_yaml_path in input_api.AbsoluteLocalPaths() or cron_yaml_path in input_api.AbsoluteLocalPaths()): return [] AppYamlHelper = _ImportAppYamlHelper(input_api) app_yaml_version = AppYamlHelper.ExtractVersion( input_api.ReadFile(app_yaml_path)) cron_yaml_version = AppYamlHelper.ExtractVersion( input_api.ReadFile(cron_yaml_path), key='target') if app_yaml_version == cron_yaml_version: return [] return [output_api.PresubmitError( 'Versions of app.yaml (%s) and cron.yaml (%s) must match' % ( app_yaml_version, cron_yaml_version))]
def GetAppVersionNonMemoized(): if 'CURRENT_VERSION_ID' in os.environ: # The version ID looks like 2-0-25.36712548 or 2-0-25.23/223; we only # want the 2-0-25. return re.compile('[./]').split(os.environ['CURRENT_VERSION_ID'])[0] # Not running on appengine, get it from the app.yaml file ourselves. app_yaml_path = os.path.join(os.path.split(__file__)[0], 'app.yaml') with open(app_yaml_path, 'r') as app_yaml: return AppYamlHelper.ExtractVersion(app_yaml.read())
def _GetSafeServerInstance(self): '''Returns a ServerInstance with a host file system at a safe revision, meaning the last revision that the current running version of the server existed. ''' channel = self._channel delegate = self._delegate server_instance_at_head = self._CreateServerInstance(channel, None) get_branch_for_channel = self._GetBranchForChannel class AppYamlHelperDelegate(AppYamlHelper.Delegate): def GetHostFileSystemForRevision(self, revision): return delegate.CreateHostFileSystemForBranchAndRevision( get_branch_for_channel(channel), revision) app_yaml_handler = AppYamlHelper( svn_constants.APP_YAML_PATH, server_instance_at_head.host_file_system, AppYamlHelperDelegate(), server_instance_at_head.object_store_creator) if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()): # TODO(kalman): return a new ServerInstance at an explicit revision in # case the HEAD version changes underneath us. return server_instance_at_head # The version in app.yaml is greater than the currently running app's. # The safe version is the one before it changed. safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan( delegate.GetAppVersion()) - 1 logging.info('cron/%s: app version %s is out of date, safe is %s' % ( channel, delegate.GetAppVersion(), safe_revision)) return self._CreateServerInstance(channel, safe_revision)
def testInstanceMethods(self): test_data = { 'server2': { 'app.yaml': _GenerateAppYaml('1-0'), 'app_yaml_helper.py': 'Copyright notice etc' } } updates = [] # Pass a specific file system at head to the HostFileSystemProvider so that # we know it's always going to be backed by a MockFileSystem. The Provider # may decide to wrap it in caching etc. file_system_at_head = MockFileSystem(TestFileSystem(test_data)) def apply_update(update): file_system_at_head.Update(update) updates.append(update) def host_file_system_constructor(branch, revision=None): self.assertEqual('trunk', branch) self.assertTrue(revision is not None) return MockFileSystem.Create(TestFileSystem(test_data), updates[:revision]) object_store_creator = ObjectStoreCreator.ForTest() host_file_system_provider = HostFileSystemProvider( object_store_creator, default_trunk_instance=file_system_at_head, constructor_for_test=host_file_system_constructor) helper = AppYamlHelper('server2/app.yaml', object_store_creator, host_file_system_provider) def assert_is_up_to_date(version): self.assertTrue(helper.IsUpToDate(version), '%s is not up to date' % version) self.assertRaises(ValueError, helper.GetFirstRevisionGreaterThan, version) self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0')) assert_is_up_to_date('1-0-0') assert_is_up_to_date('1-5-0') # Revision 1. apply_update({'server2': {'app.yaml': _GenerateAppYaml('1-5-0')}}) self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0')) self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0')) assert_is_up_to_date('1-5-0') assert_is_up_to_date('2-5-0') # Revision 2. apply_update({'server2': {'app_yaml_helper.py': 'fixed a bug'}}) self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0')) self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0')) assert_is_up_to_date('1-5-0') assert_is_up_to_date('2-5-0') # Revision 3. apply_update({'server2': {'app.yaml': _GenerateAppYaml('1-6-0')}}) self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0')) self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0')) self.assertEqual(3, helper.GetFirstRevisionGreaterThan('1-5-0')) assert_is_up_to_date('2-5-0') # Revision 4. apply_update({'server2': {'app.yaml': _GenerateAppYaml('1-8-0')}}) # Revision 5. apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-0-0')}}) # Revision 6. apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-2-0')}}) # Revision 7. apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-4-0')}}) # Revision 8. apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-6-0')}}) self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0')) self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0')) self.assertEqual(3, helper.GetFirstRevisionGreaterThan('1-5-0')) self.assertEqual(5, helper.GetFirstRevisionGreaterThan('1-8-0')) self.assertEqual(6, helper.GetFirstRevisionGreaterThan('2-0-0')) self.assertEqual(6, helper.GetFirstRevisionGreaterThan('2-1-0')) self.assertEqual(7, helper.GetFirstRevisionGreaterThan('2-2-0')) self.assertEqual(7, helper.GetFirstRevisionGreaterThan('2-3-0')) self.assertEqual(8, helper.GetFirstRevisionGreaterThan('2-4-0')) self.assertEqual(8, helper.GetFirstRevisionGreaterThan('2-5-0')) assert_is_up_to_date('2-6-0') assert_is_up_to_date('2-7-0')
def testSafeRevision(self): test_data = { 'docs': { 'examples': { 'examples.txt': 'examples.txt contents' }, 'server2': { 'app.yaml': AppYamlHelper.GenerateAppYaml('2-0-8') }, 'static': { 'static.txt': 'static.txt contents' }, 'templates': { 'public': { 'apps': { 'storage.html': 'storage.html contents' }, 'extensions': { 'storage.html': 'storage.html contents' }, } } } } updates = [] def app_yaml_update(version): return { 'docs': { 'server2': { 'app.yaml': AppYamlHelper.GenerateAppYaml(version) } } } def storage_html_update(update): return { 'docs': { 'templates': { 'public': { 'apps': { 'storage.html': update } } } } } def static_txt_update(update): return {'docs': {'static': {'static.txt': update}}} app_yaml_path = 'docs/server2/app.yaml' storage_html_path = 'docs/templates/public/apps/storage.html' static_txt_path = 'docs/static/static.txt' def create_file_system(revision=None): '''Creates a MockFileSystem at |revision| by applying that many |updates| to it. ''' mock_file_system = MockFileSystem(TestFileSystem(test_data)) for update in updates[:revision]: mock_file_system.Update(update) return mock_file_system delegate = _TestDelegate(create_file_system) delegate.SetAppVersion('2-0-8') file_systems = delegate.file_systems # No updates applied yet. CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get() self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'), file_systems[-1].ReadSingle(app_yaml_path)) self.assertEqual('storage.html contents', file_systems[-1].ReadSingle(storage_html_path)) # Apply updates to storage.html. updates.append(storage_html_update('interim contents')) updates.append(storage_html_update('new contents')) CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get() self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'), file_systems[-1].ReadSingle(app_yaml_path)) self.assertEqual('new contents', file_systems[-1].ReadSingle(storage_html_path)) # Apply several updates to storage.html and app.yaml. The file system # should be pinned at the version before app.yaml changed. updates.append(storage_html_update('stuck here contents')) double_update = storage_html_update('newer contents') double_update.update(app_yaml_update('2-0-10')) updates.append(double_update) updates.append(storage_html_update('never gonna reach here')) CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get() self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'), file_systems[-1].ReadSingle(app_yaml_path)) self.assertEqual('stuck here contents', file_systems[-1].ReadSingle(storage_html_path)) # Further pushes to storage.html will keep it pinned. updates.append(storage_html_update('y u not update!')) CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get() self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'), file_systems[-1].ReadSingle(app_yaml_path)) self.assertEqual('stuck here contents', file_systems[-1].ReadSingle(storage_html_path)) # Likewise app.yaml. updates.append(app_yaml_update('2-1-0')) CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get() self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'), file_systems[-1].ReadSingle(app_yaml_path)) self.assertEqual('stuck here contents', file_systems[-1].ReadSingle(storage_html_path)) # And updates to other content won't happen either. updates.append(static_txt_update('important content!')) CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get() self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'), file_systems[-1].ReadSingle(app_yaml_path)) self.assertEqual('stuck here contents', file_systems[-1].ReadSingle(storage_html_path)) self.assertEqual('static.txt contents', file_systems[-1].ReadSingle(static_txt_path)) # Lastly - when the app version changes, everything should no longer be # pinned. delegate.SetAppVersion('2-1-0') CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get() self.assertEqual(AppYamlHelper.GenerateAppYaml('2-1-0'), file_systems[-1].ReadSingle(app_yaml_path)) self.assertEqual('y u not update!', file_systems[-1].ReadSingle(storage_html_path)) self.assertEqual('important content!', file_systems[-1].ReadSingle(static_txt_path))
def testSafeRevision(self): test_data = { 'api': { '_api_features.json': '{}', '_manifest_features.json': '{}', '_permission_features.json': '{}', }, 'docs': { 'examples': { 'examples.txt': 'examples.txt contents' }, 'server2': { 'app.yaml': AppYamlHelper.GenerateAppYaml('2-0-8') }, 'static': { 'static.txt': 'static.txt contents' }, 'templates': { 'articles': { 'activeTab.html': 'activeTab.html contents' }, 'intros': { 'browserAction.html': 'activeTab.html contents' }, 'private': { 'table_of_contents.html': 'table_of_contents.html contents', }, 'public': { 'apps': { 'storage.html': '<h1>storage.html</h1> contents' }, 'extensions': { 'storage.html': '<h1>storage.html</h1> contents' }, }, 'json': { 'chrome_sidenav.json': '{}', 'content_providers.json': ReadFile(CONTENT_PROVIDERS), 'manifest.json': '{}', 'permissions.json': '{}', 'strings.json': '{}', 'whats_new.json': '{}', }, } } } updates = [] def app_yaml_update(version): return MoveTo(SERVER2, { 'app.yaml': AppYamlHelper.GenerateAppYaml(version) }) def storage_html_update(update): return MoveTo(PUBLIC_TEMPLATES, { 'apps': {'storage.html': update} }) def static_txt_update(update): return MoveTo(STATIC_DOCS, { 'static.txt': update }) storage_html_path = PUBLIC_TEMPLATES + 'apps/storage.html' static_txt_path = STATIC_DOCS + 'static.txt' def create_file_system(revision=None): '''Creates a MockFileSystem at |revision| by applying that many |updates| to it. ''' mock_file_system = MockFileSystem( TestFileSystem(test_data, relative_to=CHROME_EXTENSIONS)) updates_for_revision = ( updates if revision is None else updates[:int(revision)]) for update in updates_for_revision: mock_file_system.Update(update) return mock_file_system delegate = _TestDelegate(create_file_system) delegate.SetAppVersion('2-0-8') file_systems = delegate.file_systems # No updates applied yet. CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get() self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'), file_systems[-1].ReadSingle(APP_YAML).Get()) self.assertEqual('<h1>storage.html</h1> contents', file_systems[-1].ReadSingle(storage_html_path).Get()) # Apply updates to storage.html. updates.append(storage_html_update('interim contents')) updates.append(storage_html_update('<h1>new</h1> contents')) CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get() self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'), file_systems[-1].ReadSingle(APP_YAML).Get()) self.assertEqual('<h1>new</h1> contents', file_systems[-1].ReadSingle(storage_html_path).Get()) # Apply several updates to storage.html and app.yaml. The file system # should be pinned at the version before app.yaml changed. updates.append(storage_html_update('<h1>stuck here</h1> contents')) double_update = storage_html_update('<h1>newer</h1> contents') double_update.update(app_yaml_update('2-0-10')) updates.append(double_update) updates.append(storage_html_update('never gonna reach here')) CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get() self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'), file_systems[-1].ReadSingle(APP_YAML).Get()) self.assertEqual('<h1>stuck here</h1> contents', file_systems[-1].ReadSingle(storage_html_path).Get()) # Further pushes to storage.html will keep it pinned. updates.append(storage_html_update('<h1>y</h1> u not update!')) CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get() self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'), file_systems[-1].ReadSingle(APP_YAML).Get()) self.assertEqual('<h1>stuck here</h1> contents', file_systems[-1].ReadSingle(storage_html_path).Get()) # Likewise app.yaml. updates.append(app_yaml_update('2-1-0')) CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get() self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'), file_systems[-1].ReadSingle(APP_YAML).Get()) self.assertEqual('<h1>stuck here</h1> contents', file_systems[-1].ReadSingle(storage_html_path).Get()) # And updates to other content won't happen either. updates.append(static_txt_update('important content!')) CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get() self.assertEqual(AppYamlHelper.GenerateAppYaml('2-0-8'), file_systems[-1].ReadSingle(APP_YAML).Get()) self.assertEqual('<h1>stuck here</h1> contents', file_systems[-1].ReadSingle(storage_html_path).Get()) self.assertEqual('static.txt contents', file_systems[-1].ReadSingle(static_txt_path).Get()) # Lastly - when the app version changes, everything should no longer be # pinned. delegate.SetAppVersion('2-1-0') CronServlet(Request.ForTest('trunk'), delegate_for_test=delegate).Get() self.assertEqual(AppYamlHelper.GenerateAppYaml('2-1-0'), file_systems[-1].ReadSingle(APP_YAML).Get()) self.assertEqual('<h1>y</h1> u not update!', file_systems[-1].ReadSingle(storage_html_path).Get()) self.assertEqual('important content!', file_systems[-1].ReadSingle(static_txt_path).Get())
def app_yaml_update(version): return MoveTo(SERVER2, { 'app.yaml': AppYamlHelper.GenerateAppYaml(version) })
def testInstanceMethods(self): test_data = { 'server2': { 'app.yaml': _GenerateAppYaml('1-0'), 'app_yaml_helper.py': 'Copyright notice etc' } } updates = [] file_system_at_head = MockFileSystem(TestFileSystem(test_data)) def apply_update(update): file_system_at_head.Update(update) updates.append(update) def constructor(branch, revision=None): return MockFileSystem.Create(TestFileSystem(test_data), updates[:revision]) host_file_system_creator = HostFileSystemCreator( ObjectStoreCreator.ForTest(), constructor_for_test=constructor) helper = AppYamlHelper( 'server2/app.yaml', file_system_at_head, ObjectStoreCreator.ForTest(disable_wrappers=False), host_file_system_creator) def assert_is_up_to_date(version): self.assertTrue(helper.IsUpToDate(version), '%s is not up to date' % version) self.assertRaises(ValueError, helper.GetFirstRevisionGreaterThan, version) self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0')) assert_is_up_to_date('1-0-0') assert_is_up_to_date('1-5-0') # Revision 1. apply_update({'server2': {'app.yaml': _GenerateAppYaml('1-5-0')}}) self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0')) self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0')) assert_is_up_to_date('1-5-0') assert_is_up_to_date('2-5-0') # Revision 2. apply_update({'server2': {'app_yaml_helper.py': 'fixed a bug'}}) self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0')) self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0')) assert_is_up_to_date('1-5-0') assert_is_up_to_date('2-5-0') # Revision 3. apply_update({'server2': {'app.yaml': _GenerateAppYaml('1-6-0')}}) self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0')) self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0')) self.assertEqual(3, helper.GetFirstRevisionGreaterThan('1-5-0')) assert_is_up_to_date('2-5-0') # Revision 4. apply_update({'server2': {'app.yaml': _GenerateAppYaml('1-8-0')}}) # Revision 5. apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-0-0')}}) # Revision 6. apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-2-0')}}) # Revision 7. apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-4-0')}}) # Revision 8. apply_update({'server2': {'app.yaml': _GenerateAppYaml('2-6-0')}}) self.assertEqual(0, helper.GetFirstRevisionGreaterThan('0-5-0')) self.assertEqual(1, helper.GetFirstRevisionGreaterThan('1-0-0')) self.assertEqual(3, helper.GetFirstRevisionGreaterThan('1-5-0')) self.assertEqual(5, helper.GetFirstRevisionGreaterThan('1-8-0')) self.assertEqual(6, helper.GetFirstRevisionGreaterThan('2-0-0')) self.assertEqual(6, helper.GetFirstRevisionGreaterThan('2-1-0')) self.assertEqual(7, helper.GetFirstRevisionGreaterThan('2-2-0')) self.assertEqual(7, helper.GetFirstRevisionGreaterThan('2-3-0')) self.assertEqual(8, helper.GetFirstRevisionGreaterThan('2-4-0')) self.assertEqual(8, helper.GetFirstRevisionGreaterThan('2-5-0')) assert_is_up_to_date('2-6-0') assert_is_up_to_date('2-7-0')