def testLoaderFileWithRetryParameters(self):
input_data = ('queue:\n'
'- name: server-queue\n'
' rate: 50/s\n'
' max_concurrent_requests: 15\n'
' retry_parameters:\n'
' task_retry_limit: 100\n'
' task_age_limit: 1d\n'
' min_backoff_seconds: 1\n'
' max_backoff_seconds: 1800\n'
' max_doublings: 20\n'
'- name: data-queue\n'
' retry_parameters:\n'
' task_retry_limit: 0\n')
config = queueinfo.LoadSingleQueue(input_data)
self.assertLen(config.queue, 2)
self.assertIsInstance(config.queue[0], queueinfo.QueueEntry)
self.assertEqual(config.queue[0].name, 'server-queue')
self.assertEqual(config.queue[0].rate, '50/s')
retry = config.queue[0].retry_parameters
self.assertIsInstance(retry, queueinfo.RetryParameters)
self.assertEqual(100, retry.task_retry_limit)
self.assertEqual('1d', retry.task_age_limit)
self.assertEqual(1.0, retry.min_backoff_seconds)
self.assertEqual(1800.0, retry.max_backoff_seconds)
self.assertEqual(20, retry.max_doublings)
self.assertIsInstance(config.queue[1], queueinfo.QueueEntry)
self.assertEqual(config.queue[1].name, 'data-queue')
retry = config.queue[1].retry_parameters
self.assertIsInstance(retry, queueinfo.RetryParameters)
self.assertEqual(0, retry.task_retry_limit)
def load_queues_from_file(self):
""" Translates an application's queue configuration file to queue objects.
Returns:
A dictionary mapping queue names to Queue objects.
Raises:
ValueError: If queue_file is unable to get loaded.
"""
queue_file = self.get_queue_file_location(self._app_id)
using_default = False
try:
info = file_io.read(queue_file)
logging.info('Found queue file for {}'.format(self._app_id))
except IOError:
logging.info(
'No queue file found for {}, using default queue'.format(
self._app_id))
info = self.DEFAULT_QUEUE_YAML
using_default = True
#TODO handle bad xml/yaml files.
if queue_file.endswith('yaml') or using_default:
queue_info = queueinfo.LoadSingleQueue(info).ToDict()
elif queue_file.endswith('xml'):
queue_info = self.parse_queue_xml(info)
else:
raise ValueError("Unable to load queue information with %s" %
queue_file)
if not queue_info:
raise ValueError("Queue information with %s not set" % queue_file)
# We add in the default queue if its not already in there.
has_default = False
if 'queue' not in queue_info or len(queue_info['queue']) == 0:
queue_info = {'queue': [{'rate': '5/s', 'name': 'default'}]}
for queue in queue_info['queue']:
if queue['name'] == 'default':
has_default = True
if not has_default:
queue_info['queue'].append({'rate': '5/s', 'name': 'default'})
logging.info('Queue for {}:\n{}'.format(self._app_id, queue_info))
# Discard the invalid queues.
queues = {}
for queue in queue_info['queue']:
if 'mode' in queue and queue['mode'] == 'pull':
try:
queues[queue['name']] = PullQueue(queue, self._app_id,
self.db_access)
except InvalidQueueConfiguration:
logging.exception('Invalid queue configuration')
else:
try:
queues[queue['name']] = PushQueue(queue, self._app_id)
except InvalidQueueConfiguration:
logging.exception('Invalid queue configuration')
return queues
def setUp(self):
unittest.TestCase.setUp(self)
self.mox = mox.Mox()
self.appid = "testapp"
self.version_id = "1.23456789"
os.environ["APPLICATION_ID"] = self.appid
os.environ["CURRENT_VERSION_ID"] = self.version_id
os.environ["HTTP_HOST"] = "localhost"
self.memcache = memcache_stub.MemcacheServiceStub()
self.taskqueue = taskqueue_stub.TaskQueueServiceStub()
self.taskqueue.queue_yaml_parser = (
lambda x: queueinfo.LoadSingleQueue(
"queue:\n"
"- name: default\n"
" rate: 10/s\n"
"- name: crazy-queue\n"
" rate: 2000/d\n"
" bucket_size: 10\n"))
self.datastore = datastore_file_stub.DatastoreFileStub(
self.appid, "/dev/null", "/dev/null")
self.blob_storage_directory = tempfile.mkdtemp()
blob_storage = file_blob_storage.FileBlobStorage(
self.blob_storage_directory, self.appid)
self.blobstore_stub = blobstore_stub.BlobstoreServiceStub(blob_storage)
self.file_service = self.createFileServiceStub(blob_storage)
apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
apiproxy_stub_map.apiproxy.RegisterStub("taskqueue", self.taskqueue)
apiproxy_stub_map.apiproxy.RegisterStub("memcache", self.memcache)
apiproxy_stub_map.apiproxy.RegisterStub("datastore_v3", self.datastore)
apiproxy_stub_map.apiproxy.RegisterStub("blobstore", self.blobstore_stub)
apiproxy_stub_map.apiproxy.RegisterStub("file", self.file_service)
def testLoaderFileWithVersionedModuleTarget(self):
input_data = ('queue:\n'
'- name: server-queue\n'
' rate: 50/s\n'
' target: version1.api\n')
config = queueinfo.LoadSingleQueue(input_data)
self.assertLen(config.queue, 1)
self.assertIsInstance(config.queue[0], queueinfo.QueueEntry)
self.assertEqual(config.queue[0].name, 'server-queue')
self.assertEqual(config.queue[0].rate, '50/s')
self.assertEqual(config.queue[0].target, 'version1.api')
def testLoaderFileWithNumericVerisionedModuleTarget(self):
input_data = ('queue:\n'
'- name: push-queue\n'
' rate: 5/s\n'
' target: 1.module\n')
config = queueinfo.LoadSingleQueue(input_data)
self.assertLen(config.queue, 1)
self.assertIsInstance(config.queue[0], queueinfo.QueueEntry)
self.assertEqual(config.queue[0].name, 'push-queue')
self.assertEqual(config.queue[0].rate, '5/s')
self.assertEqual(config.queue[0].target, '1.module')
def testLoaderFileWithMaxConcurrentRequests(self):
input_data = ('queue:\n'
'- name: server-queue\n'
' rate: 50/s\n'
' max_concurrent_requests: 15\n')
config = queueinfo.LoadSingleQueue(input_data)
self.assertLen(config.queue, 1)
self.assertIsInstance(config.queue[0], queueinfo.QueueEntry)
self.assertEqual(config.queue[0].name, 'server-queue')
self.assertEqual(config.queue[0].rate, '50/s')
self.assertEqual(config.queue[0].max_concurrent_requests, 15)
def testLoaderFileWithTargetParsingError(self):
"""Test an erroneous target."""
input_data = ('queue:\n'
'- name: push-queue\n'
' rate: 5/s\n'
' target: bad:bad\n')
with self.assertRaisesRegex(
yaml_errors.EventError,
re.escape(
"Value 'bad:bad' for target does not match expression '"
"^(?:^(?:(?:((?!-)[a-z\d\-]{1,100})\.)?)((?!-)[a-z\d\-]{1,63})$)$'"
)):
queueinfo.LoadSingleQueue(input_data)
def load_queues_from_file(self, app_id):
""" Parses the queue.yaml or queue.xml file of an application
and loads it into the class.
Args:
app_id: The application ID.
Returns:
A dictionary of the queue settings.
Raises:
ValueError: If queue_file is unable to get loaded.
"""
queue_file = self.get_queue_file_location(app_id)
info = ""
using_default = False
try:
info = file_io.read(queue_file)
logging.info("Found queue file for app {0}".format(app_id))
except IOError:
logging.info("No queue file found for app {0}, using default queue" \
.format(app_id))
info = self.DEFAULT_QUEUE_YAML
using_default = True
queue_info = ""
#TODO handle bad xml/yaml files.
if queue_file.endswith('yaml') or using_default:
queue_info = queueinfo.LoadSingleQueue(info).ToDict()
elif queue_file.endswith('xml'):
queue_info = self.parse_queue_xml(info)
else:
raise ValueError("Unable to load queue information with %s" %
queue_file)
if not queue_info:
raise ValueError("Queue information with %s not set" % queue_file)
# We add in the default queue if its not already in there.
has_default = False
if 'queue' not in queue_info or len(queue_info['queue']) == 0:
queue_info = {'queue': [{'rate': '5/s', 'name': 'default'}]}
for queue in queue_info['queue']:
if queue['name'] == 'default':
has_default = True
if not has_default:
queue_info['queue'].append({'rate': '5/s', 'name': 'default'})
self._queue_info_file = queue_info
logging.info("AppID {0} -- Loaded queue {1}".format(
app_id, queue_info))
return queue_info
def get_queue_info():
"""
Walk up the tree until queue.yaml is found
"""
directory = os.path.abspath(".")
while directory:
file_path = os.path.join(directory, 'queue.yaml')
if os.path.isfile(file_path):
with open(os.path.join(directory, 'queue.yaml'), 'r') as fh:
return queueinfo.LoadSingleQueue(fh)
else:
directory = os.path.dirname(directory)
if os.path.realpath(directory) == directory:
break
def setUp(self):
super(TestSetupMixin, self).setUp()
from google.appengine.api import apiproxy_stub_map
from google.appengine.api import memcache
from google.appengine.api import queueinfo
from google.appengine.datastore import datastore_stub_util
from google.appengine.ext import testbed
from google.appengine.ext.testbed import TASKQUEUE_SERVICE_NAME
before_level = logging.getLogger().getEffectiveLevel()
os.environ['APPLICATION_ID'] = self.TEST_APP_ID
os.environ['CURRENT_VERSION_ID'] = self.TEST_VERSION_ID
os.environ['HTTP_HOST'] = '%s.appspot.com' % self.TEST_APP_ID
os.environ['DEFAULT_VERSION_HOSTNAME'] = os.environ['HTTP_HOST']
os.environ['CURRENT_MODULE_ID'] = 'foo-module'
try:
logging.getLogger().setLevel(100)
self.testbed = testbed.Testbed()
self.testbed.activate()
self.testbed.setup_env(app_id=self.TEST_APP_ID, overwrite=True)
self.testbed.init_memcache_stub()
self.testbed.init_mail_stub()
hr_policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy(
probability=1)
self.testbed.init_datastore_v3_stub(consistency_policy=hr_policy)
self.testbed.init_taskqueue_stub()
root_path = os.path.realpath(os.path.dirname(__file__))
# Actually need to flush, even though we've reallocated. Maybe because the
# memcache stub's cache is at the module level, not the API stub?
memcache.flush_all()
finally:
logging.getLogger().setLevel(before_level)
define_queues = ['other']
taskqueue_stub = apiproxy_stub_map.apiproxy.GetStub('taskqueue')
taskqueue_stub.queue_yaml_parser = (
lambda x: queueinfo.LoadSingleQueue(
'queue:\n- name: default\n rate: 1/s\n' + '\n'.join(
'- name: %s\n rate: 1/s' % name
for name in define_queues)))
def testLoaderSaneFile(self):
input_data = ('application: testapp\n'
'queue:\n'
'- name: mail-queue\n'
' rate: 2000/d\n'
' bucket_size: 11\n'
'- name: speedy-queue\n'
' rate: 5/s\n')
config = queueinfo.LoadSingleQueue(input_data)
self.assertEqual('testapp', config.application)
self.assertLen(config.queue, 2)
self.assertIsInstance(config.queue[0], queueinfo.QueueEntry)
self.assertIsInstance(config.queue[1], queueinfo.QueueEntry)
self.assertEqual(config.queue[0].name, 'mail-queue')
self.assertEqual(config.queue[0].rate, '2000/d')
self.assertEqual(config.queue[0].bucket_size, 11)
def testLoaderFileWithInstanceVerisionedModuleTarget(self):
"""Instance can't be specified in the queue target.
It's not known whether this is intentional or a bug.
"""
input_data = ('queue:\n'
'- name: push-queue\n'
' rate: 5/s\n'
' target: 1.1.module\n')
with self.assertRaisesRegex(
yaml_errors.EventError,
re.escape(
"Value '1.1.module' for target does not match expression '"
"^(?:^(?:(?:((?!-)[a-z\d\-]{1,100})\.)?)((?!-)[a-z\d\-]{1,63})$)$'"
)):
queueinfo.LoadSingleQueue(input_data)
def testLoaderFileWithMode(self):
input_data = ('queue:\n'
'- name: pull-queue\n'
' rate: 2000/d\n'
' mode: pull\n'
'- name: default\n'
' rate: 10/m\n'
' mode: push\n')
config = queueinfo.LoadSingleQueue(input_data)
self.assertLen(config.queue, 2)
self.assertIsInstance(config.queue[0], queueinfo.QueueEntry)
self.assertEqual(config.queue[0].name, 'pull-queue')
self.assertEqual(config.queue[0].rate, '2000/d')
self.assertEqual(config.queue[0].mode, 'pull')
self.assertIsInstance(config.queue[1], queueinfo.QueueEntry)
self.assertEqual(config.queue[1].name, 'default')
self.assertEqual(config.queue[1].rate, '10/m')
self.assertEqual(config.queue[1].mode, 'push')
def setUp(self):
unittest.TestCase.setUp(self)
self.mox = mox.Mox()
self.appid = "testapp"
self.major_version_id = "1"
self.version_id = self.major_version_id + ".23456789"
self.module_id = "foo_module"
self.host = "%s.%s.%s" % (self.major_version_id, self.module_id,
"testapp.appspot.com")
self.testbed = testbed.Testbed()
self.testbed.activate()
os.environ["APPLICATION_ID"] = self.appid
os.environ["CURRENT_VERSION_ID"] = self.version_id
os.environ["CURRENT_MODULE_ID"] = self.module_id
os.environ["DEFAULT_VERSION_HOSTNAME"] = "%s.appspot.com" % self.appid
os.environ["HTTP_HOST"] = self.host
self.testbed.init_app_identity_stub()
self.testbed.init_blobstore_stub()
# HRD with no eventual consistency.
policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy(
probability=1)
self.testbed.init_datastore_v3_stub(consistency_policy=policy)
self.testbed.init_logservice_stub()
self.testbed.init_files_stub()
self.testbed.init_memcache_stub()
self.testbed.init_taskqueue_stub()
self.testbed.init_urlfetch_stub()
# For backwards compatibility, maintain easy references to some stubs
self.taskqueue = self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME)
self.taskqueue.queue_yaml_parser = (
# pylint: disable=g-long-lambda
lambda x: queueinfo.LoadSingleQueue("queue:\n"
"- name: default\n"
" rate: 10/s\n"
"- name: crazy-queue\n"
" rate: 2000/d\n"
" bucket_size: 10\n"))
def setup_for_testing(require_indexes=True, define_queues=[]):
"""Sets up the stubs for testing.
Args:
require_indexes: True if indexes should be required for all indexes.
define_queues: Additional queues that should be available.
"""
from google.appengine.api import apiproxy_stub_map
from google.appengine.api import memcache
from google.appengine.api import queueinfo
from google.appengine.datastore import datastore_stub_util
from google.appengine.tools import old_dev_appserver
from google.appengine.tools import dev_appserver_index
before_level = logging.getLogger().getEffectiveLevel()
try:
logging.getLogger().setLevel(100)
root_path = os.path.realpath(os.path.dirname(__file__))
old_dev_appserver.SetupStubs(
TEST_APP_ID,
root_path=root_path,
login_url='',
datastore_path=tempfile.mktemp(suffix='datastore_stub'),
history_path=tempfile.mktemp(suffix='datastore_history'),
blobstore_path=tempfile.mktemp(suffix='blobstore_stub'),
require_indexes=require_indexes,
clear_datastore=False)
dev_appserver_index.SetupIndexes(TEST_APP_ID, root_path)
# Actually need to flush, even though we've reallocated. Maybe because the
# memcache stub's cache is at the module level, not the API stub?
memcache.flush_all()
finally:
logging.getLogger().setLevel(before_level)
datastore_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
hr_policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=1)
datastore_stub.SetConsistencyPolicy(hr_policy)
taskqueue_stub = apiproxy_stub_map.apiproxy.GetStub('taskqueue')
taskqueue_stub.queue_yaml_parser = (
lambda x: queueinfo.LoadSingleQueue(
'queue:\n- name: default\n rate: 1/s\n' +
'\n'.join('- name: %s\n rate: 1/s' % name
for name in define_queues)))
def testLoaderSaneFileWithStorageLimit(self):
input_data = ('total_storage_limit: 140M\n'
'queue:\n'
'- name: mail-queue\n'
' rate: 2000/d\n'
' bucket_size: 11\n'
'- name: speedy-queue\n'
' rate: 5/s\n'
' mode: pull')
config = queueinfo.LoadSingleQueue(input_data)
self.assertEqual('140M', config.total_storage_limit)
self.assertLen(config.queue, 2)
self.assertIsInstance(config.queue[0], queueinfo.QueueEntry)
self.assertIsInstance(config.queue[1], queueinfo.QueueEntry)
self.assertEqual(config.queue[0].name, 'mail-queue')
self.assertEqual(config.queue[0].rate, '2000/d')
self.assertEqual(config.queue[0].bucket_size, 11)
self.assertEqual(config.queue[1].name, 'speedy-queue')
self.assertEqual(config.queue[1].rate, '5/s')
self.assertEqual(config.queue[1].mode, 'pull')
def testLoaderFileWithAcl(self):
input_data = ('queue:\n'
'- name: server-queue\n'
' rate: 50/s\n'
' max_concurrent_requests: 15\n'
' acl:\n'
' - user_email: [email protected]\n'
' - user_email: [email protected]\n'
' - writer_email: [email protected]\n')
config = queueinfo.LoadSingleQueue(input_data)
self.assertLen(config.queue, 1)
self.assertIsInstance(config.queue[0], queueinfo.QueueEntry)
self.assertEqual(config.queue[0].name, 'server-queue')
self.assertEqual(config.queue[0].rate, '50/s')
acl = config.queue[0].acl
self.assertLen(acl, 3)
self.assertIsInstance(acl[0], queueinfo.Acl)
self.assertEqual('*****@*****.**', acl[0].user_email)
self.assertIsInstance(acl[1], queueinfo.Acl)
self.assertEqual('*****@*****.**', acl[1].user_email)
self.assertIsInstance(acl[2], queueinfo.Acl)
self.assertEqual('*****@*****.**', acl[2].writer_email)
def _ParseQueueYaml(self):
"""Loads the queue.yaml file and parses it.
Returns:
None if queue.yaml doesn't exist, otherwise a queueinfo.QueueEntry object
populated from the queue.yaml.
"""
if hasattr(self, 'queue_yaml_parser'):
return self.queue_yaml_parser(self._root_path)
if self._root_path is None:
return None
for queueyaml in ('queue.yaml', 'queue.yml'):
try:
fh = open(os.path.join(self._root_path, queueyaml), 'r')
except IOError:
continue
try:
queue_info = queueinfo.LoadSingleQueue(fh)
return queue_info
finally:
fh.close()
return None
def _ParseQueueYaml(unused_self, root_path):
"""Loads the queue.yaml file and parses it.
Args:
unused_self: Allows this function to be bound to a class member. Not used.
root_path: Directory containing queue.yaml. Not used.
Returns:
None if queue.yaml doesn't exist, otherwise a queueinfo.QueueEntry object
populated from the queue.yaml.
"""
if root_path is None:
return None
for queueyaml in ('queue.yaml', 'queue.yml'):
try:
fh = open(os.path.join(root_path, queueyaml), 'r')
except IOError:
continue
try:
queue_info = queueinfo.LoadSingleQueue(fh)
return queue_info
finally:
fh.close()
return None
def setUp(self):
googletest.TestCase.setUp(self)
self.appid = "testapp"
os.environ["APPLICATION_ID"] = self.appid
self.memcache = memcache_stub.MemcacheServiceStub()
self.taskqueue = taskqueue_stub.TaskQueueServiceStub()
self.taskqueue.queue_yaml_parser = (
lambda x: queueinfo.LoadSingleQueue("queue:\n"
"- name: default\n"
" rate: 10/s\n"
"- name: crazy-queue\n"
" rate: 2000/d\n"
" bucket_size: 10\n"))
self.datastore = datastore_file_stub.DatastoreFileStub(
self.appid, "/dev/null", "/dev/null")
self.user = user_service_stub.UserServiceStub()
apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
apiproxy_stub_map.apiproxy.RegisterStub("taskqueue", self.taskqueue)
apiproxy_stub_map.apiproxy.RegisterStub("memcache", self.memcache)
apiproxy_stub_map.apiproxy.RegisterStub("datastore_v3", self.datastore)
apiproxy_stub_map.apiproxy.RegisterStub("user", self.user)
