def setUp(self):
super(ExternalSchedulerApiTestBatchMode, self).setUp()
base = {
'address': u'http://localhost:1',
'id': u'foo',
'dimensions': ['key1:value1', 'key2:value2'],
'all_dimensions': None,
'any_dimensions': None,
'enabled': True,
'allow_es_fallback': True,
}
self.cfg_foo = pools_config.ExternalSchedulerConfig(**base)
base['id'] = u'hoe'
self.cfg_hoe = pools_config.ExternalSchedulerConfig(**base)
self.mock(external_scheduler, '_get_client', self._get_client)
self._enqueue_orig = self.mock(utils, 'enqueue_task', self._enqueue)
self._client = None
# Setup the backend to handle task queues.
self.app = webtest.TestApp(handlers_backend.create_application(True),
extra_environ={
'REMOTE_ADDR':
self.source_ip,
'SERVER_SOFTWARE':
os.environ['SERVER_SOFTWARE'],
})
self.cfg = config.settings()
self.cfg.enable_batch_es_notifications = True
self.mock(config, 'settings', lambda: self.cfg)
def setUp(self):
super(IsolateServiceTest, self).setUp()
self.testbed.init_blobstore_stub()
self.testbed.init_urlfetch_stub()
# It seems like there is a singleton state preserved across the tests,
# making it hard to re-run the complete setUp procedure. Therefore we pre-
# register all the possible identities being used in the tests.
all_authed_ids = [
auth.Identity(auth.IDENTITY_USER, '*****@*****.**'),
auth.Identity(auth.IDENTITY_USER,
'*****@*****.**'),
auth.Identity(auth.IDENTITY_SERVICE, 'adminapp'),
]
admin = all_authed_ids[0]
full_access_group = config.settings().auth.full_access_group
auth.bootstrap_group(full_access_group, all_authed_ids)
auth_testing.mock_get_current_identity(self, admin)
version = utils.get_app_version()
self.mock(utils, 'get_task_queue_host', lambda: version)
self.testbed.setup_env(current_version_id='testbed.version')
self.source_ip = '127.0.0.1'
# It is needed solely for self.execute_tasks(), which processes tasks queues
# on the backend application.
self.app = webtest.TestApp(
handlers_backend.create_application(debug=True),
extra_environ={'REMOTE_ADDR': self.source_ip})
# add a private key; signing depends on config.settings()
make_private_key()
# Remove the check for dev server in should_push_to_gs().
self.mock(utils, 'is_local_dev_server', lambda: False)
def setUp(self):
super(ExternalSchedulerApiTest, self).setUp()
self.es_cfg = pools_config.ExternalSchedulerConfig(
address=u'http://localhost:1',
id=u'foo',
dimensions=['key1:value1', 'key2:value2'],
all_dimensions=None,
any_dimensions=None,
enabled=True,
allow_es_fallback=True)
# Make the values deterministic.
self.mock_now(datetime.datetime(2014, 1, 2, 3, 4, 5, 6))
self.mock(random, 'getrandbits', lambda _: 0x88)
# Use the local fake client to external scheduler..
self.mock(external_scheduler, '_get_client', self._get_client)
self._client = None
# Setup the backend to handle task queues.
self.app = webtest.TestApp(handlers_backend.create_application(True),
extra_environ={
'REMOTE_ADDR':
self.source_ip,
'SERVER_SOFTWARE':
os.environ['SERVER_SOFTWARE'],
})
self._enqueue_orig = self.mock(utils, 'enqueue_task', self._enqueue)
cfg = config.settings()
cfg.enable_batch_es_notifications = False
self.mock(config, 'settings', lambda: cfg)
def create_application():
ereporter2.register_formatter()
# Zap out the ndb in-process cache by default.
# This cache causes excessive memory usage in in handler where a lot of
# entities are fetched in one query. When coupled with high concurrency
# as specified via max_concurrent_requests in app.yaml, this may cause out of
# memory errors.
ndb.Context.default_cache_policy = staticmethod(lambda _key: False)
ndb.Context._cache_policy = staticmethod(lambda _key: False)
# If running on a local dev server, allow bots to connect without prior
# groups configuration. Useful when running smoke test.
if utils.is_local_dev_server():
acl.bootstrap_dev_server_acls()
pools_config.bootstrap_dev_server_acls()
def is_enabled_callback():
return config.settings().enable_ts_monitoring
backend_app = handlers_backend.create_application(False)
gae_ts_mon.initialize(backend_app, is_enabled_fn=is_enabled_callback)
# Local import, because it instantiates the mapreduce app.
# This is for the task queue handlers.
from mapreduce import main
gae_ts_mon.initialize(main.APP, is_enabled_fn=is_enabled_callback)
event_mon_metrics.initialize()
ts_mon_metrics.initialize()
utils.report_memory(backend_app)
return backend_app, main.APP
def create_application():
ereporter2.register_formatter()
backend = handlers_backend.create_application(False)
def is_enabled_callback():
return config.settings().enable_ts_monitoring
gae_ts_mon.initialize(backend, is_enabled_fn=is_enabled_callback)
return backend
def create_application():
ereporter2.register_formatter()
backend = handlers_backend.create_application(False)
def is_enabled_callback():
return config.settings().enable_ts_monitoring
gae_ts_mon.initialize(backend, is_enabled_fn=is_enabled_callback)
return backend
def setUp(self):
super(TaskQueuesApiTest, self).setUp()
# Setup the backend to handle task queues.
self.app = webtest.TestApp(
handlers_backend.create_application(True),
extra_environ={
'REMOTE_ADDR': self.source_ip,
'SERVER_SOFTWARE': os.environ['SERVER_SOFTWARE'],
})
self._enqueue_orig = self.mock(utils, 'enqueue_task', self._enqueue)
def setUp(self):
super(BackendTest, self).setUp()
# By default requests in tests are coming from bot with fake IP.
self.app = webtest.TestApp(handlers_backend.create_application(True),
extra_environ={
'REMOTE_ADDR':
self.source_ip,
'SERVER_SOFTWARE':
os.environ['SERVER_SOFTWARE'],
})
self._enqueue_task_orig = self.mock(utils, 'enqueue_task',
self._enqueue_task)
def setUp(self):
super(TaskToRunApiTest, self).setUp()
self.now = datetime.datetime(2014, 01, 02, 03, 04, 05, 06)
self.mock_now(self.now)
auth_testing.mock_get_current_identity(self)
# Setup the backend to handle task queues for 'task-dimensions'.
self.app = webtest.TestApp(handlers_backend.create_application(True),
extra_environ={
'REMOTE_ADDR':
self.source_ip,
'SERVER_SOFTWARE':
os.environ['SERVER_SOFTWARE'],
})
self._enqueue_orig = self.mock(utils, 'enqueue_task', self._enqueue)
def setUp(self):
super(TaskToRunApiTest, self).setUp()
self.now = datetime.datetime(2019, 01, 02, 03, 04, 05, 06)
self.mock_now(self.now)
auth_testing.mock_get_current_identity(self)
# Setup the backend to handle task queues.
self.app = webtest.TestApp(handlers_backend.create_application(True),
extra_environ={
'REMOTE_ADDR':
self.source_ip,
'SERVER_SOFTWARE':
os.environ['SERVER_SOFTWARE'],
})
self._enqueue_orig = self.mock(utils, 'enqueue_task_async',
self._enqueue)
cfg = config.settings()
cfg.use_lifo = True
self.mock(config, 'settings', lambda: cfg)
def create_application():
ereporter2.register_formatter()
# Zap out the ndb in-process cache by default.
# This cache causes excessive memory usage in in handler where a lot of
# entities are fetched in one query. When coupled with high concurrency
# as specified via max_concurrent_requests in app.yaml, this may cause out of
# memory errors.
ndb.Context.default_cache_policy = staticmethod(lambda _key: False)
ndb.Context._cache_policy = staticmethod(lambda _key: False)
backend = handlers_backend.create_application(False)
def is_enabled_callback():
return config.settings().enable_ts_monitoring
gae_ts_mon.initialize(backend, is_enabled_fn=is_enabled_callback)
return backend
def setUp(self):
"""Creates a new app instance for every test case."""
super(MainTest, self).setUp()
self.testbed.init_user_stub()
# When called during a taskqueue, the call to get_app_version() may fail so
# pre-fetch it.
version = utils.get_app_version()
self.mock(utils, 'get_task_queue_host', lambda: version)
self.source_ip = '192.168.0.1'
self.app_frontend = webtest.TestApp(
handlers_frontend.create_application(debug=True),
extra_environ={'REMOTE_ADDR': self.source_ip})
# This is awkward but both the frontend and backend applications uses the
# same template variables.
template.reset()
self.app_backend = webtest.TestApp(
handlers_backend.create_application(debug=True),
extra_environ={'REMOTE_ADDR': self.source_ip})
# Tasks are enqueued on the backend.
self.app = self.app_backend
self.auth_app = webtest.TestApp(
auth.create_wsgi_application(debug=True),
extra_environ={
'REMOTE_ADDR': self.source_ip,
'SERVER_SOFTWARE': os.environ['SERVER_SOFTWARE'],
})
full_access_group = config.settings().auth.full_access_group
readonly_access_group = config.settings().auth.readonly_access_group
auth.bootstrap_group(
auth.ADMIN_GROUP,
[auth.Identity(auth.IDENTITY_USER, '*****@*****.**')])
auth.bootstrap_group(
readonly_access_group,
[auth.Identity(auth.IDENTITY_USER, '*****@*****.**')])
auth.bootstrap_group(
full_access_group,
[auth.Identity(auth.IDENTITY_USER, '*****@*****.**')])
# TODO(maruel): Create a BOTS_GROUP.
self.set_as_anonymous()
def create_application():
ereporter2.register_formatter()
utils.set_task_queue_module('backend')
template.bootstrap()
# If running on a local dev server, allow bots to connect without prior
# groups configuration. Useful when running smoke test.
if utils.is_local_dev_server():
acl.bootstrap_dev_server_acls()
def is_enabled_callback():
return config.settings().enable_ts_monitoring
# App that serves HTML pages and old API.
frontend_app = handlers_frontend.create_application(False)
gae_ts_mon.initialize(frontend_app, is_enabled_fn=is_enabled_callback)
# App that contains crons and task queues.
backend_app = handlers_backend.create_application(False)
gae_ts_mon.initialize(backend_app, is_enabled_fn=is_enabled_callback)
# Local import, because it instantiates the mapreduce app.
from mapreduce import main
gae_ts_mon.initialize(main.APP, is_enabled_fn=is_enabled_callback)
# TODO(maruel): Remove this once there is no known client anymore.
api = webapp2.WSGIApplication(
endpoints_webapp2.api_server(
[
handlers_endpoints.SwarmingServerService,
handlers_endpoints.SwarmingTaskService,
handlers_endpoints.SwarmingTasksService,
handlers_endpoints.SwarmingQueuesService,
handlers_endpoints.SwarmingBotService,
handlers_endpoints.SwarmingBotsService,
# components.config endpoints for validation and configuring of luci-config
# service URL.
config.ConfigApi,
],
base_path='/_ah/api'))
event_mon_metrics.initialize()
ts_mon_metrics.initialize()
return frontend_app, api, backend_app, main.APP
def setUp(self):
"""Creates a new app instance for every test case."""
super(MainTest, self).setUp()
self.testbed.init_blobstore_stub()
self.testbed.init_urlfetch_stub()
admin = auth.Identity(auth.IDENTITY_USER, '*****@*****.**')
full_access_group = config.settings().auth.full_access_group
auth.bootstrap_group(full_access_group, [admin])
auth_testing.mock_get_current_identity(self, admin)
version = utils.get_app_version()
self.mock(utils, 'get_task_queue_host', lambda: version)
self.testbed.setup_env(current_version_id='testbed.version')
self.source_ip = '127.0.0.1'
self.app = webtest.TestApp(
handlers_backend.create_application(debug=True),
extra_environ={'REMOTE_ADDR': self.source_ip})
# add a private key; signing depends on config.settings()
make_private_key()
# Remove the check for dev server in should_push_to_gs().
self.mock(utils, 'is_local_dev_server', lambda: False)
def setUp(self):
"""Creates a new app instance for every test case."""
super(MainTest, self).setUp()
self.testbed.init_user_stub()
# When called during a taskqueue, the call to get_app_version() may fail so
# pre-fetch it.
version = utils.get_app_version()
self.mock(utils, 'get_task_queue_host', lambda: version)
self.source_ip = '192.168.0.1'
self.app_frontend = webtest.TestApp(
handlers_frontend.create_application(debug=True),
extra_environ={'REMOTE_ADDR': self.source_ip})
# This is awkward but both the frontend and backend applications uses the
# same template variables.
template.reset()
self.app_backend = webtest.TestApp(
handlers_backend.create_application(debug=True),
extra_environ={'REMOTE_ADDR': self.source_ip})
# Tasks are enqueued on the backend.
self.app = self.app_backend
self.auth_app = webtest.TestApp(
auth.create_wsgi_application(debug=True),
extra_environ={
'REMOTE_ADDR': self.source_ip,
'SERVER_SOFTWARE': os.environ['SERVER_SOFTWARE'],
})
auth.bootstrap_group(
auth.ADMIN_GROUP,
[auth.Identity(auth.IDENTITY_USER, '*****@*****.**')])
auth.bootstrap_group(
acl.READONLY_ACCESS_GROUP,
[auth.Identity(auth.IDENTITY_USER, '*****@*****.**')])
auth.bootstrap_group(
acl.FULL_ACCESS_GROUP,
[auth.Identity(auth.IDENTITY_USER, '*****@*****.**')])
# TODO(maruel): Create a BOTS_GROUP.
self.set_as_anonymous()
def create_application():
ereporter2.register_formatter()
return handlers_backend.create_application(False)
def create_application():
ereporter2.register_formatter()
return monitoring.wrap_webapp2_app(
handlers_backend.create_application(False))
def wsgi_app():
return handlers_backend.create_application(True)
def create_application(): ereporter2.register_formatter() return handlers_backend.create_application(False)
