def testLoaderFileWithRetryParameters(self): input_data = ('queue:\n' '- name: server-queue\n' ' rate: 50/s\n' ' max_concurrent_requests: 15\n' ' retry_parameters:\n' ' task_retry_limit: 100\n' ' task_age_limit: 1d\n' ' min_backoff_seconds: 1\n' ' max_backoff_seconds: 1800\n' ' max_doublings: 20\n' '- name: data-queue\n' ' retry_parameters:\n' ' task_retry_limit: 0\n') config = queueinfo.LoadSingleQueue(input_data) self.assertLen(config.queue, 2) self.assertIsInstance(config.queue[0], queueinfo.QueueEntry) self.assertEqual(config.queue[0].name, 'server-queue') self.assertEqual(config.queue[0].rate, '50/s') retry = config.queue[0].retry_parameters self.assertIsInstance(retry, queueinfo.RetryParameters) self.assertEqual(100, retry.task_retry_limit) self.assertEqual('1d', retry.task_age_limit) self.assertEqual(1.0, retry.min_backoff_seconds) self.assertEqual(1800.0, retry.max_backoff_seconds) self.assertEqual(20, retry.max_doublings) self.assertIsInstance(config.queue[1], queueinfo.QueueEntry) self.assertEqual(config.queue[1].name, 'data-queue') retry = config.queue[1].retry_parameters self.assertIsInstance(retry, queueinfo.RetryParameters) self.assertEqual(0, retry.task_retry_limit)
def load_queues_from_file(self): """ Translates an application's queue configuration file to queue objects. Returns: A dictionary mapping queue names to Queue objects. Raises: ValueError: If queue_file is unable to get loaded. """ queue_file = self.get_queue_file_location(self._app_id) using_default = False try: info = file_io.read(queue_file) logging.info('Found queue file for {}'.format(self._app_id)) except IOError: logging.info( 'No queue file found for {}, using default queue'.format( self._app_id)) info = self.DEFAULT_QUEUE_YAML using_default = True #TODO handle bad xml/yaml files. if queue_file.endswith('yaml') or using_default: queue_info = queueinfo.LoadSingleQueue(info).ToDict() elif queue_file.endswith('xml'): queue_info = self.parse_queue_xml(info) else: raise ValueError("Unable to load queue information with %s" % queue_file) if not queue_info: raise ValueError("Queue information with %s not set" % queue_file) # We add in the default queue if its not already in there. has_default = False if 'queue' not in queue_info or len(queue_info['queue']) == 0: queue_info = {'queue': [{'rate': '5/s', 'name': 'default'}]} for queue in queue_info['queue']: if queue['name'] == 'default': has_default = True if not has_default: queue_info['queue'].append({'rate': '5/s', 'name': 'default'}) logging.info('Queue for {}:\n{}'.format(self._app_id, queue_info)) # Discard the invalid queues. queues = {} for queue in queue_info['queue']: if 'mode' in queue and queue['mode'] == 'pull': try: queues[queue['name']] = PullQueue(queue, self._app_id, self.db_access) except InvalidQueueConfiguration: logging.exception('Invalid queue configuration') else: try: queues[queue['name']] = PushQueue(queue, self._app_id) except InvalidQueueConfiguration: logging.exception('Invalid queue configuration') return queues
def setUp(self): unittest.TestCase.setUp(self) self.mox = mox.Mox() self.appid = "testapp" self.version_id = "1.23456789" os.environ["APPLICATION_ID"] = self.appid os.environ["CURRENT_VERSION_ID"] = self.version_id os.environ["HTTP_HOST"] = "localhost" self.memcache = memcache_stub.MemcacheServiceStub() self.taskqueue = taskqueue_stub.TaskQueueServiceStub() self.taskqueue.queue_yaml_parser = ( lambda x: queueinfo.LoadSingleQueue( "queue:\n" "- name: default\n" " rate: 10/s\n" "- name: crazy-queue\n" " rate: 2000/d\n" " bucket_size: 10\n")) self.datastore = datastore_file_stub.DatastoreFileStub( self.appid, "/dev/null", "/dev/null") self.blob_storage_directory = tempfile.mkdtemp() blob_storage = file_blob_storage.FileBlobStorage( self.blob_storage_directory, self.appid) self.blobstore_stub = blobstore_stub.BlobstoreServiceStub(blob_storage) self.file_service = self.createFileServiceStub(blob_storage) apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap() apiproxy_stub_map.apiproxy.RegisterStub("taskqueue", self.taskqueue) apiproxy_stub_map.apiproxy.RegisterStub("memcache", self.memcache) apiproxy_stub_map.apiproxy.RegisterStub("datastore_v3", self.datastore) apiproxy_stub_map.apiproxy.RegisterStub("blobstore", self.blobstore_stub) apiproxy_stub_map.apiproxy.RegisterStub("file", self.file_service)
def testLoaderFileWithVersionedModuleTarget(self): input_data = ('queue:\n' '- name: server-queue\n' ' rate: 50/s\n' ' target: version1.api\n') config = queueinfo.LoadSingleQueue(input_data) self.assertLen(config.queue, 1) self.assertIsInstance(config.queue[0], queueinfo.QueueEntry) self.assertEqual(config.queue[0].name, 'server-queue') self.assertEqual(config.queue[0].rate, '50/s') self.assertEqual(config.queue[0].target, 'version1.api')
def testLoaderFileWithNumericVerisionedModuleTarget(self): input_data = ('queue:\n' '- name: push-queue\n' ' rate: 5/s\n' ' target: 1.module\n') config = queueinfo.LoadSingleQueue(input_data) self.assertLen(config.queue, 1) self.assertIsInstance(config.queue[0], queueinfo.QueueEntry) self.assertEqual(config.queue[0].name, 'push-queue') self.assertEqual(config.queue[0].rate, '5/s') self.assertEqual(config.queue[0].target, '1.module')
def testLoaderFileWithMaxConcurrentRequests(self): input_data = ('queue:\n' '- name: server-queue\n' ' rate: 50/s\n' ' max_concurrent_requests: 15\n') config = queueinfo.LoadSingleQueue(input_data) self.assertLen(config.queue, 1) self.assertIsInstance(config.queue[0], queueinfo.QueueEntry) self.assertEqual(config.queue[0].name, 'server-queue') self.assertEqual(config.queue[0].rate, '50/s') self.assertEqual(config.queue[0].max_concurrent_requests, 15)
def testLoaderFileWithTargetParsingError(self): """Test an erroneous target.""" input_data = ('queue:\n' '- name: push-queue\n' ' rate: 5/s\n' ' target: bad:bad\n') with self.assertRaisesRegex( yaml_errors.EventError, re.escape( "Value 'bad:bad' for target does not match expression '" "^(?:^(?:(?:((?!-)[a-z\d\-]{1,100})\.)?)((?!-)[a-z\d\-]{1,63})$)$'" )): queueinfo.LoadSingleQueue(input_data)
def load_queues_from_file(self, app_id): """ Parses the queue.yaml or queue.xml file of an application and loads it into the class. Args: app_id: The application ID. Returns: A dictionary of the queue settings. Raises: ValueError: If queue_file is unable to get loaded. """ queue_file = self.get_queue_file_location(app_id) info = "" using_default = False try: info = file_io.read(queue_file) logging.info("Found queue file for app {0}".format(app_id)) except IOError: logging.info("No queue file found for app {0}, using default queue" \ .format(app_id)) info = self.DEFAULT_QUEUE_YAML using_default = True queue_info = "" #TODO handle bad xml/yaml files. if queue_file.endswith('yaml') or using_default: queue_info = queueinfo.LoadSingleQueue(info).ToDict() elif queue_file.endswith('xml'): queue_info = self.parse_queue_xml(info) else: raise ValueError("Unable to load queue information with %s" % queue_file) if not queue_info: raise ValueError("Queue information with %s not set" % queue_file) # We add in the default queue if its not already in there. has_default = False if 'queue' not in queue_info or len(queue_info['queue']) == 0: queue_info = {'queue': [{'rate': '5/s', 'name': 'default'}]} for queue in queue_info['queue']: if queue['name'] == 'default': has_default = True if not has_default: queue_info['queue'].append({'rate': '5/s', 'name': 'default'}) self._queue_info_file = queue_info logging.info("AppID {0} -- Loaded queue {1}".format( app_id, queue_info)) return queue_info
def get_queue_info(): """ Walk up the tree until queue.yaml is found """ directory = os.path.abspath(".") while directory: file_path = os.path.join(directory, 'queue.yaml') if os.path.isfile(file_path): with open(os.path.join(directory, 'queue.yaml'), 'r') as fh: return queueinfo.LoadSingleQueue(fh) else: directory = os.path.dirname(directory) if os.path.realpath(directory) == directory: break
def setUp(self): super(TestSetupMixin, self).setUp() from google.appengine.api import apiproxy_stub_map from google.appengine.api import memcache from google.appengine.api import queueinfo from google.appengine.datastore import datastore_stub_util from google.appengine.ext import testbed from google.appengine.ext.testbed import TASKQUEUE_SERVICE_NAME before_level = logging.getLogger().getEffectiveLevel() os.environ['APPLICATION_ID'] = self.TEST_APP_ID os.environ['CURRENT_VERSION_ID'] = self.TEST_VERSION_ID os.environ['HTTP_HOST'] = '%s.appspot.com' % self.TEST_APP_ID os.environ['DEFAULT_VERSION_HOSTNAME'] = os.environ['HTTP_HOST'] os.environ['CURRENT_MODULE_ID'] = 'foo-module' try: logging.getLogger().setLevel(100) self.testbed = testbed.Testbed() self.testbed.activate() self.testbed.setup_env(app_id=self.TEST_APP_ID, overwrite=True) self.testbed.init_memcache_stub() self.testbed.init_mail_stub() hr_policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy( probability=1) self.testbed.init_datastore_v3_stub(consistency_policy=hr_policy) self.testbed.init_taskqueue_stub() root_path = os.path.realpath(os.path.dirname(__file__)) # Actually need to flush, even though we've reallocated. Maybe because the # memcache stub's cache is at the module level, not the API stub? memcache.flush_all() finally: logging.getLogger().setLevel(before_level) define_queues = ['other'] taskqueue_stub = apiproxy_stub_map.apiproxy.GetStub('taskqueue') taskqueue_stub.queue_yaml_parser = ( lambda x: queueinfo.LoadSingleQueue( 'queue:\n- name: default\n rate: 1/s\n' + '\n'.join( '- name: %s\n rate: 1/s' % name for name in define_queues)))
def testLoaderSaneFile(self): input_data = ('application: testapp\n' 'queue:\n' '- name: mail-queue\n' ' rate: 2000/d\n' ' bucket_size: 11\n' '- name: speedy-queue\n' ' rate: 5/s\n') config = queueinfo.LoadSingleQueue(input_data) self.assertEqual('testapp', config.application) self.assertLen(config.queue, 2) self.assertIsInstance(config.queue[0], queueinfo.QueueEntry) self.assertIsInstance(config.queue[1], queueinfo.QueueEntry) self.assertEqual(config.queue[0].name, 'mail-queue') self.assertEqual(config.queue[0].rate, '2000/d') self.assertEqual(config.queue[0].bucket_size, 11)
def testLoaderFileWithInstanceVerisionedModuleTarget(self): """Instance can't be specified in the queue target. It's not known whether this is intentional or a bug. """ input_data = ('queue:\n' '- name: push-queue\n' ' rate: 5/s\n' ' target: 1.1.module\n') with self.assertRaisesRegex( yaml_errors.EventError, re.escape( "Value '1.1.module' for target does not match expression '" "^(?:^(?:(?:((?!-)[a-z\d\-]{1,100})\.)?)((?!-)[a-z\d\-]{1,63})$)$'" )): queueinfo.LoadSingleQueue(input_data)
def testLoaderFileWithMode(self): input_data = ('queue:\n' '- name: pull-queue\n' ' rate: 2000/d\n' ' mode: pull\n' '- name: default\n' ' rate: 10/m\n' ' mode: push\n') config = queueinfo.LoadSingleQueue(input_data) self.assertLen(config.queue, 2) self.assertIsInstance(config.queue[0], queueinfo.QueueEntry) self.assertEqual(config.queue[0].name, 'pull-queue') self.assertEqual(config.queue[0].rate, '2000/d') self.assertEqual(config.queue[0].mode, 'pull') self.assertIsInstance(config.queue[1], queueinfo.QueueEntry) self.assertEqual(config.queue[1].name, 'default') self.assertEqual(config.queue[1].rate, '10/m') self.assertEqual(config.queue[1].mode, 'push')
def setUp(self): unittest.TestCase.setUp(self) self.mox = mox.Mox() self.appid = "testapp" self.major_version_id = "1" self.version_id = self.major_version_id + ".23456789" self.module_id = "foo_module" self.host = "%s.%s.%s" % (self.major_version_id, self.module_id, "testapp.appspot.com") self.testbed = testbed.Testbed() self.testbed.activate() os.environ["APPLICATION_ID"] = self.appid os.environ["CURRENT_VERSION_ID"] = self.version_id os.environ["CURRENT_MODULE_ID"] = self.module_id os.environ["DEFAULT_VERSION_HOSTNAME"] = "%s.appspot.com" % self.appid os.environ["HTTP_HOST"] = self.host self.testbed.init_app_identity_stub() self.testbed.init_blobstore_stub() # HRD with no eventual consistency. policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy( probability=1) self.testbed.init_datastore_v3_stub(consistency_policy=policy) self.testbed.init_logservice_stub() self.testbed.init_files_stub() self.testbed.init_memcache_stub() self.testbed.init_taskqueue_stub() self.testbed.init_urlfetch_stub() # For backwards compatibility, maintain easy references to some stubs self.taskqueue = self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME) self.taskqueue.queue_yaml_parser = ( # pylint: disable=g-long-lambda lambda x: queueinfo.LoadSingleQueue("queue:\n" "- name: default\n" " rate: 10/s\n" "- name: crazy-queue\n" " rate: 2000/d\n" " bucket_size: 10\n"))
def setup_for_testing(require_indexes=True, define_queues=[]): """Sets up the stubs for testing. Args: require_indexes: True if indexes should be required for all indexes. define_queues: Additional queues that should be available. """ from google.appengine.api import apiproxy_stub_map from google.appengine.api import memcache from google.appengine.api import queueinfo from google.appengine.datastore import datastore_stub_util from google.appengine.tools import old_dev_appserver from google.appengine.tools import dev_appserver_index before_level = logging.getLogger().getEffectiveLevel() try: logging.getLogger().setLevel(100) root_path = os.path.realpath(os.path.dirname(__file__)) old_dev_appserver.SetupStubs( TEST_APP_ID, root_path=root_path, login_url='', datastore_path=tempfile.mktemp(suffix='datastore_stub'), history_path=tempfile.mktemp(suffix='datastore_history'), blobstore_path=tempfile.mktemp(suffix='blobstore_stub'), require_indexes=require_indexes, clear_datastore=False) dev_appserver_index.SetupIndexes(TEST_APP_ID, root_path) # Actually need to flush, even though we've reallocated. Maybe because the # memcache stub's cache is at the module level, not the API stub? memcache.flush_all() finally: logging.getLogger().setLevel(before_level) datastore_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3') hr_policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=1) datastore_stub.SetConsistencyPolicy(hr_policy) taskqueue_stub = apiproxy_stub_map.apiproxy.GetStub('taskqueue') taskqueue_stub.queue_yaml_parser = ( lambda x: queueinfo.LoadSingleQueue( 'queue:\n- name: default\n rate: 1/s\n' + '\n'.join('- name: %s\n rate: 1/s' % name for name in define_queues)))
def testLoaderSaneFileWithStorageLimit(self): input_data = ('total_storage_limit: 140M\n' 'queue:\n' '- name: mail-queue\n' ' rate: 2000/d\n' ' bucket_size: 11\n' '- name: speedy-queue\n' ' rate: 5/s\n' ' mode: pull') config = queueinfo.LoadSingleQueue(input_data) self.assertEqual('140M', config.total_storage_limit) self.assertLen(config.queue, 2) self.assertIsInstance(config.queue[0], queueinfo.QueueEntry) self.assertIsInstance(config.queue[1], queueinfo.QueueEntry) self.assertEqual(config.queue[0].name, 'mail-queue') self.assertEqual(config.queue[0].rate, '2000/d') self.assertEqual(config.queue[0].bucket_size, 11) self.assertEqual(config.queue[1].name, 'speedy-queue') self.assertEqual(config.queue[1].rate, '5/s') self.assertEqual(config.queue[1].mode, 'pull')
def testLoaderFileWithAcl(self): input_data = ('queue:\n' '- name: server-queue\n' ' rate: 50/s\n' ' max_concurrent_requests: 15\n' ' acl:\n' ' - user_email: [email protected]\n' ' - user_email: [email protected]\n' ' - writer_email: [email protected]\n') config = queueinfo.LoadSingleQueue(input_data) self.assertLen(config.queue, 1) self.assertIsInstance(config.queue[0], queueinfo.QueueEntry) self.assertEqual(config.queue[0].name, 'server-queue') self.assertEqual(config.queue[0].rate, '50/s') acl = config.queue[0].acl self.assertLen(acl, 3) self.assertIsInstance(acl[0], queueinfo.Acl) self.assertEqual('*****@*****.**', acl[0].user_email) self.assertIsInstance(acl[1], queueinfo.Acl) self.assertEqual('*****@*****.**', acl[1].user_email) self.assertIsInstance(acl[2], queueinfo.Acl) self.assertEqual('*****@*****.**', acl[2].writer_email)
def _ParseQueueYaml(self): """Loads the queue.yaml file and parses it. Returns: None if queue.yaml doesn't exist, otherwise a queueinfo.QueueEntry object populated from the queue.yaml. """ if hasattr(self, 'queue_yaml_parser'): return self.queue_yaml_parser(self._root_path) if self._root_path is None: return None for queueyaml in ('queue.yaml', 'queue.yml'): try: fh = open(os.path.join(self._root_path, queueyaml), 'r') except IOError: continue try: queue_info = queueinfo.LoadSingleQueue(fh) return queue_info finally: fh.close() return None
def _ParseQueueYaml(unused_self, root_path): """Loads the queue.yaml file and parses it. Args: unused_self: Allows this function to be bound to a class member. Not used. root_path: Directory containing queue.yaml. Not used. Returns: None if queue.yaml doesn't exist, otherwise a queueinfo.QueueEntry object populated from the queue.yaml. """ if root_path is None: return None for queueyaml in ('queue.yaml', 'queue.yml'): try: fh = open(os.path.join(root_path, queueyaml), 'r') except IOError: continue try: queue_info = queueinfo.LoadSingleQueue(fh) return queue_info finally: fh.close() return None
def setUp(self): googletest.TestCase.setUp(self) self.appid = "testapp" os.environ["APPLICATION_ID"] = self.appid self.memcache = memcache_stub.MemcacheServiceStub() self.taskqueue = taskqueue_stub.TaskQueueServiceStub() self.taskqueue.queue_yaml_parser = ( lambda x: queueinfo.LoadSingleQueue("queue:\n" "- name: default\n" " rate: 10/s\n" "- name: crazy-queue\n" " rate: 2000/d\n" " bucket_size: 10\n")) self.datastore = datastore_file_stub.DatastoreFileStub( self.appid, "/dev/null", "/dev/null") self.user = user_service_stub.UserServiceStub() apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap() apiproxy_stub_map.apiproxy.RegisterStub("taskqueue", self.taskqueue) apiproxy_stub_map.apiproxy.RegisterStub("memcache", self.memcache) apiproxy_stub_map.apiproxy.RegisterStub("datastore_v3", self.datastore) apiproxy_stub_map.apiproxy.RegisterStub("user", self.user)