def RunAllDeferredTasks(tb, queue_name=None):
"""Runs all deferred tasks in specified queue.
If no queue is specified, then runs all deferred tasks in queues
known to this module, in QUEUE_NAMES.
Args:
tb: Your test's testbed instance.
queue_name: String. The name the queue whose tasks should be run.
"""
if queue_name:
queue_names = [queue_name]
else:
queue_names = QUEUE_NAMES
taskqueue = tb.get_stub(testbed.TASKQUEUE_SERVICE_NAME)
for name in queue_names:
try:
tasks = taskqueue.GetTasks(name)
for task in tasks:
deferred.run(base64.b64decode(task['body']))
taskqueue.DeleteTask(name, task['name'])
except KeyError:
# If a queue hasn't had a task added to it then we'll get a harmless
# keyerror when trying to get any tasks that have been added to it.
pass
def testAcceptWithDifferentDetails(self):
self.loginUser()
self.givePermission()
suggestion_id = self.createSuggestion()
form = self.getSuggestionForm('review_{}'.format(suggestion_id))
form['webcast_type'] = 'youtube'
form['webcast_channel'] = 'foobar'
form['webcast_file'] = 'meow'
response = form.submit('verdict', value='accept').follow()
self.assertEqual(response.status_int, 200)
request = response.request
self.assertEqual(request.GET.get('success'), 'accept')
# Process task queue
tasks = self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME).get_filtered_tasks()
for task in tasks:
deferred.run(task.payload)
# Make sure we mark the Suggestion as REVIEWED
suggestion = Suggestion.get_by_id(suggestion_id)
self.assertIsNotNone(suggestion)
self.assertEqual(suggestion.review_state, Suggestion.REVIEW_ACCEPTED)
# Make sure the Event has no webcasts
event = Event.get_by_id('2016necmp')
self.assertIsNotNone(event.webcast)
self.assertEqual(len(event.webcast), 1)
webcast = event.webcast[0]
self.assertEqual(webcast['type'], 'youtube')
self.assertEqual(webcast['channel'], 'foobar')
self.assertEqual(webcast['file'], 'meow')
def run_test_task_queue(client):
test_task_stub = stub_manager.testbed.get_stub(
testbed.TASKQUEUE_SERVICE_NAME)
#stub = stub_manager.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME)
for queue in test_task_stub.GetQueues():
tasks = test_task_stub.GetTasks(queue['name'])
for task in tasks:
#logging.info('queue %s %s %s'%(queue['name'],task['method'],task['url']))
if task['method'].upper() == 'POST':
content_type = 'multipart/form-data; boundary="-=-=-=-=-=-"'
data = {}
for k, v in task['headers']:
if k.lower() == 'content-type':
content_type = v
if task['body']:
data = base64.b64decode(task['body'])
if task['url'] == '/_ah/queue/deferred':
#content_type=='application/octet-stream' and queue['name']=='default':
deferred.run(data)
else:
#logging.info(' '.join(['post',task['name'],' - ',task['url']]))
client.post(task['url'],
data=data,
content_type=content_type)
else:
#logging.info(' '.join(['get',task['name'],' - ',task['url']]))
client.get(task['url'])
#logging.info(' '.join(['done',task['name'],' - ',task['url']]))
test_task_stub.FlushQueue(queue['name'])
def run_test_task_queue(client):
test_task_stub = stub_manager.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME)
#stub = stub_manager.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME)
for queue in test_task_stub.GetQueues():
tasks = test_task_stub.GetTasks(queue['name'])
for task in tasks:
#logging.info('queue %s %s %s'%(queue['name'],task['method'],task['url']))
if task['method'].upper()=='POST':
content_type = 'multipart/form-data; boundary="-=-=-=-=-=-"'
data = {}
for k,v in task['headers']:
if k.lower()=='content-type':
content_type = v
if task['body']:
data = base64.b64decode(task['body'])
if task['url']=='/_ah/queue/deferred':
#content_type=='application/octet-stream' and queue['name']=='default':
deferred.run(data)
else:
#logging.info(' '.join(['post',task['name'],' - ',task['url']]))
client.post(task['url'],data=data, content_type=content_type)
else:
#logging.info(' '.join(['get',task['name'],' - ',task['url']]))
client.get(task['url'])
#logging.info(' '.join(['done',task['name'],' - ',task['url']]))
test_task_stub.FlushQueue(queue['name'])
def test_update_search(self):
grant = koalaoauth2.Grants.new(**self.test_grant)
inserted_uid = koalaoauth2.Grants.insert(resource_object=grant)
tasks = self.task_queue.get_filtered_tasks()
self.assertEqual(len(tasks), 1, u'Invalid number of Deferred tasks')
deferred.run(
tasks[0].payload) # Doesn't return anything so nothing to test
retrieved_grant = koalaoauth2.Grants.get(resource_uid=inserted_uid)
retrieved_grant.code = 'updated_code'
koalaoauth2.Grants.update(resource_object=retrieved_grant)
tasks = self.task_queue.get_filtered_tasks()
self.assertEqual(len(tasks), 2, u'Invalid number of Deferred tasks')
deferred.run(
tasks[1].payload) # Doesn't return anything so nothing to test
search_result = koalaoauth2.Grants.search(
query_string='code: {}'.format('updated_code'))
self.assertEqual(search_result.results_count, 1,
u'Query returned incorrect count')
self.assertEqual(len(search_result.results), 1,
u'Query returned incorrect number of results')
def test_accept_with_different_details(self):
self.loginUser()
self.givePermission()
suggestion_id = self.createSuggestion()
form = self.getSuggestionForm('review_{}'.format(suggestion_id))
form['webcast_type'] = 'youtube'
form['webcast_channel'] = 'foobar'
form['webcast_file'] = 'meow'
response = form.submit('verdict', value='accept').follow()
self.assertEqual(response.status_int, 200)
request = response.request
self.assertEqual(request.GET.get('success'), 'accept')
# Process task queue
tasks = self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME).get_filtered_tasks()
for task in tasks:
deferred.run(task.payload)
# Make sure we mark the Suggestion as REVIEWED
suggestion = Suggestion.get_by_id(suggestion_id)
self.assertIsNotNone(suggestion)
self.assertEqual(suggestion.review_state, Suggestion.REVIEW_ACCEPTED)
# Make sure the Event has no webcasts
event = Event.get_by_id('2016necmp')
self.assertIsNotNone(event.webcast)
self.assertEqual(len(event.webcast), 1)
webcast = event.webcast[0]
self.assertEqual(webcast['type'], 'youtube')
self.assertEqual(webcast['channel'], 'foobar')
self.assertEqual(webcast['file'], 'meow')
def delete_all_events(self, monitor=None):
if monitor is None:
monitor = self.default_monitor
models.Event.deleteall(monitor.key.urlsafe())
tasks = self.taskqueue.get_filtered_tasks()
self.assertEqual(1, len(tasks))
deferred.run(tasks[0].payload)
def test_delete_search(self):
refresh_token = koalaoauth2.RefreshTokens.new(
**self.test_refresh_token)
koalaoauth2.RefreshTokens.insert(resource_object=refresh_token)
tasks = self.task_queue.get_filtered_tasks()
self.assertEqual(len(tasks), 1, u'Invalid number of Deferred tasks')
deferred.run(
tasks[0].payload) # Doesn't return anything so nothing to test
koalaoauth2.RefreshTokens.delete(resource_uid=refresh_token.token)
tasks = self.task_queue.get_filtered_tasks()
self.assertEqual(len(tasks), 2, u'Invalid number of Deferred tasks')
deferred.run(
tasks[1].payload) # Doesn't return anything so nothing to test
search_result = koalaoauth2.RefreshTokens.search(
query_string='token: {}'.format(self.test_refresh_token['token']))
self.assertEqual(search_result.results_count, 0,
u'Query returned incorrect count')
self.assertEqual(len(search_result.results), 0,
u'Query returned incorrect number of results')
def test_forgot_password_post_sends_email(self):
self.assertNotLoggedIn()
profile = self.create_profile()
response = self.app.get(self.uri_for('forgot-password'))
form = response.forms['forgot-password']
form['email'] = profile.email
response = form.submit()
# Check the task was put on the mail queue.
tasks = self.taskqueue_stub.get_filtered_tasks(queue_names='mail')
self.assertIn('mail', tasks[0].headers['X-AppEngine-QueueName'])
task, = tasks
deferred.run(task.payload)
messages = self.mail_stub.get_sent_messages()
self.assertLength(1, messages)
message, = messages
profile = Profile.get(profile.key())
# Reload profile to get new activation key.
self.assertEqual('"%s" <%s>' % (profile.name, profile.email),
message.to)
self.assertEqual(constants.FULL_NO_REPLY_EMAIL, message.sender)
self.assertEqual(constants.FULL_SUPPORT_EMAIL, message.reply_to)
self.assertIn(profile.activation_key, message.body.decode())
self.assertIn(profile.activation_key, message.html.decode())
recover_uri = self.uri_for('forgot-password', k=profile.activation_key)
self.assertIn(recover_uri, message.body.decode())
self.assertIn(recover_uri, message.html.decode())
def test_send_fcm_unavailable_error(self):
client = MobileClient(parent=ndb.Key(Account, 'user_id'),
user_id='user_id',
messaging_id='messaging_id',
client_type=ClientType.OS_IOS)
client.put()
# Sanity check
self.assertEqual(fcm_messaging_ids('user_id'), ['messaging_id'])
batch_response = messaging.BatchResponse([
messaging.SendResponse(None, UnavailableError('code', 'message'))
])
with patch.object(FCMRequest, 'send',
return_value=batch_response), patch(
'logging.error') as mock_error:
exit_code = TBANSHelper._send_fcm([client], MockNotification())
self.assertEqual(exit_code, 0)
mock_error.assert_called_once_with(
'FCM unavailable - retrying client...')
# Sanity check
self.assertEqual(fcm_messaging_ids('user_id'), ['messaging_id'])
# Check that we queue'd for a retry
tasks = self.taskqueue_stub.get_filtered_tasks(
queue_names='push-notifications')
self.assertEqual(len(tasks), 1)
# Make sure our taskqueue tasks execute what we expect
with patch.object(TBANSHelper, '_send_fcm') as mock_send_fcm:
deferred.run(tasks[0].payload)
mock_send_fcm.assert_called_once_with([client], ANY, 1)
def test_update_search(self):
user_group = koalausers.UserGroups.new(**self.test_user_group)
user_group_uid = koalausers.UserGroups.insert(
resource_object=user_group)
tasks = self.task_queue.get_filtered_tasks()
self.assertEqual(len(tasks), 3, u'Invalid number of Deferred tasks')
deferred.run(
tasks[0].payload) # Doesn't return anything so nothing to test
deferred.run(
tasks[1].payload) # Doesn't return anything so nothing to test
deferred.run(
tasks[2].payload) # Doesn't return anything so nothing to test
retrieved_user_group = koalausers.UserGroups.get(
resource_uid=user_group_uid)
retrieved_user_group.group_name = 'updated_group_name'
koalausers.UserGroups.update(resource_object=retrieved_user_group)
tasks = self.task_queue.get_filtered_tasks()
self.assertEqual(len(tasks), 5, u'Invalid number of Deferred tasks')
deferred.run(
tasks[3].payload) # Doesn't return anything so nothing to test
deferred.run(
tasks[4].payload) # Doesn't return anything so nothing to test
search_result = koalausers.UserGroups.search(
query_string='group_name: {}'.format('updated_group_name'))
self.assertEqual(search_result.results_count, 1,
u'Query returned incorrect count')
self.assertEqual(len(search_result.results), 1,
u'Query returned incorrect number of results')
def test_delete_search(self):
user = koalausers.Users.new(username='******',
email_address='*****@*****.**',
first_name='Test',
raw_password='******',
last_name='User ',
language_preference='en')
user_uid = koalausers.Users.insert(resource_object=user)
tasks = self.task_queue.get_filtered_tasks()
self.assertEqual(len(tasks), 1, u'Deferred task missing')
deferred.run(
tasks[0].payload) # Doesn't return anything so nothing to test
koalausers.Users.delete(resource_uid=user_uid)
tasks = self.task_queue.get_filtered_tasks()
self.assertEqual(len(tasks), 2, u'Deferred task missing')
deferred.run(
tasks[1].payload) # Doesn't return anything so nothing to test
search_result = koalausers.Users.search(query_string='username: test')
self.assertEqual(search_result.results_count, 0,
u'Query returned incorrect count')
self.assertEqual(len(search_result.results), 0,
u'Query returned incorrect number of results')
def test_send_fcm_retry_backoff_time(self):
client = MobileClient(parent=ndb.Key(Account, 'user_id'),
user_id='user_id',
messaging_id='messaging_id',
client_type=ClientType.OS_IOS)
client.put()
import time
batch_response = messaging.BatchResponse([
messaging.SendResponse(None, QuotaExceededError('code', 'message'))
])
for i in range(0, 6):
with patch.object(
FCMRequest, 'send',
return_value=batch_response), patch('logging.error'):
call_time = time.time()
TBANSHelper._send_fcm([client], MockNotification(), i)
# Check that we queue'd for a retry with the proper countdown time
tasks = self.taskqueue_stub.get_filtered_tasks(
queue_names='push-notifications')
if i > 0:
self.assertGreater(tasks[0].eta_posix - call_time, 0)
# Make sure our taskqueue tasks execute what we expect
with patch.object(TBANSHelper, '_send_fcm') as mock_send_fcm:
deferred.run(tasks[0].payload)
mock_send_fcm.assert_called_once_with([client], ANY, i + 1)
self.taskqueue_stub.FlushQueue('push-notifications')
def testWalkthrough(self, _):
models.ProvisioningVolume(owner='stub',
created_by=users.get_current_user(),
hdd_serial='stub',
passphrase=str(uuid.uuid4()),
created=datetime.datetime.now(),
platform_uuid='stub',
serial='stub',
volume_uuid=str(uuid.uuid4()).upper(),
tag='v1').put()
resp = gae_main.app.get_response(
'/api/internal/maintenance/update_volumes_schema',
{'REQUEST_METHOD': 'GET'})
self.assertEqual(200, resp.status_int)
taskqueue = self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME)
tasks = taskqueue.get_filtered_tasks()
self.assertEqual(9, len(tasks))
for task in tasks:
deferred.run(task.payload)
self.assertEqual('v1', models.ProvisioningVolume.all().fetch(1)[0].tag)
def testGenerateComputersSummaryCache(self):
today = datetime.datetime.now()
models.Computer(active=True,
hostname='xyz-macbook',
serial='SERIAL',
uuid='UUID',
owner='zerocool',
client_version='2.3.3',
os_version='10.10',
site='MTV',
track='unstable',
config_track='unstable',
connection_dates=[today],
connections_on_corp=0,
connections_off_corp=100,
uptime=90000.0,
root_disk_free=0,
user_disk_free=10,
preflight_datetime=today).put()
reports_cache._GenerateComputersSummaryCache()
taskqueue_stub = self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME)
tasks = taskqueue_stub.get_filtered_tasks()
self.assertEqual(1, len(tasks))
deferred.run(tasks[0].payload)
self.assertEqual(1, len(taskqueue_stub.get_filtered_tasks()))
self.assertEqual(
100,
models.ReportsCache.GetStatsSummary()[0]['conns_off_corp'])
def get(self):
"""Executes deferred tasks by invoking the deferred api handler."""
try:
deferred.run(self.request.raw_post_data)
except deferred.PermanentTaskFailure as task_error:
logging.exception(
'Deferred Run failed.Exception: %s', task_error)
def mark_as_error_and_reload(self, fbl):
# Now let's "rename" the source to be id 222
error = (400, {
"error": {
"message":
"Page ID 111 was migrated to page ID 222. Please update your API calls to the new ID",
"code": 21,
"type": "OAuthException"
}
})
fb_api.FBAPI.results.update({
URL_111: error,
'/v2.2/111/feed': error,
})
fbl.clear_local_cache()
# Loading it again should trigger a deferred task to rename to 222
try:
fbl.get(fb_api.LookupThingFeed, '111')
self.fail("Fetching renamed ID unexpectedly worked")
except fb_api.NoFetchedDataException:
pass
# Let's verify a task got created, and run it now
tasks = self.taskqueue_stub.get_filtered_tasks()
self.assertEqual(len(tasks), 1)
task = tasks[0]
deferred.run(task.payload)
# We have to handle "in case of queue_name, set the default manually" scenario ourselves
self.taskqueue_stub.DeleteTask(task.queue_name or 'default', task.name)
def process_and_flush_pending_tasks(self):
from google.appengine.ext import deferred
tasks = self.taskqueue_stub.get_filtered_tasks()
self.taskqueue_stub.FlushQueue('default')
while tasks:
for task in tasks:
if task.url == '/_ah/queue/deferred':
deferred.run(task.payload)
else:
# All other tasks are expected to be mapreduce ones.
headers = {
key: str(val)
for key, val in task.headers.iteritems()
}
headers['Content-Length'] = str(len(task.payload or ''))
response = self.testapp.post(url=str(task.url),
params=(task.payload or ''),
headers=headers)
if response.status_code != 200:
raise RuntimeError('MapReduce task to URL %s failed' %
task.url)
tasks = self.taskqueue_stub.get_filtered_tasks()
self.taskqueue_stub.FlushQueue('default')
def test_signup_sends_welcome_email(self):
# Sign up successfully
response = self.app.post(self.uri_for('signup'), self.SIGNUP_DATA)
self.assertRedirects(response, self.uri_for('dashboard', tour=''))
# Check that a profile was created
profile = Profile.get_by_email(self.SIGNUP_DATA['email'])
self.assertIsNotNone(profile)
# Check that a mail-sending task is in the queue
tasks = self.taskqueue_stub.GetTasks('mail')
self.assertLength(1, tasks)
# Run the task (it should be a deferred call) and check that an e-mail
# is sent
task, = tasks
deferred.run(base64.b64decode(task['body']))
messages = self.mail_stub.get_sent_messages()
self.assertLength(1, messages)
message, = messages
self.assertEqual('"%s" <%s>' % (profile.name, profile.email),
message.to)
self.assertEqual('Welcome to Daily Meeting!', message.subject)
self.assertEqual('"Daily Meeting" <*****@*****.**>',
message.sender)
self.assertEqual(
'"Daily Meeting Support" <*****@*****.**>',
message.reply_to)
activation_key = Profile.all().get().activation_key
activation_url = self.uri_for('profile.activate', k=activation_key)
self.assertIn(activation_url, message.body.decode())
self.assertIn(activation_url, message.html.decode())
def run_queue_tasks(self, queue='default'):
api_tasks = self.task_stub.GetTasks(queue)
while len(api_tasks) >0:
self.task_stub.FlushQueue(queue)
for api_task in api_tasks:
deferred.run(base64.b64decode(api_task['body']))
api_tasks = self.task_stub.GetTasks(queue)
def process_and_flush_pending_tasks(self, queue_name=None):
"""Runs and flushes pending tasks. If queue_name is None, does so for
all queues; otherwise, this only runs and flushes tasks for the
specified queue.
For more information on self.taskqueue_stub see
https://code.google.com/p/googleappengine/source/browse/trunk/python/google/appengine/api/taskqueue/taskqueue_stub.py
"""
queue_names = [queue_name] if queue_name else self._get_all_queue_names()
tasks = self.taskqueue_stub.get_filtered_tasks(queue_names=queue_names)
for queue in queue_names:
self.taskqueue_stub.FlushQueue(queue)
while tasks:
for task in tasks:
if task.url == "/_ah/queue/deferred":
from google.appengine.ext import deferred
deferred.run(task.payload)
else:
# All other tasks are expected to be mapreduce ones.
headers = {key: str(val) for key, val in task.headers.iteritems()}
headers["Content-Length"] = str(len(task.payload or ""))
response = self.testapp.post(url=str(task.url), params=(task.payload or ""), headers=headers)
if response.status_code != 200:
raise RuntimeError("MapReduce task to URL %s failed" % task.url)
tasks = self.taskqueue_stub.get_filtered_tasks(queue_names=queue_names)
for queue in queue_names:
self.taskqueue_stub.FlushQueue(queue)
def runDeferredTasks(self, queue='default'):
tasks = self.taskqueue_stub.GetTasks(queue)
while tasks:
self.taskqueue_stub.FlushQueue(queue)
for task in tasks:
deferred.run(base64.b64decode(task['body']))
tasks = self.taskqueue_stub.GetTasks(queue)
def test_delete_search(self):
company = koalacompanies.Companies.new(**self.test_company)
company_uid = koalacompanies.Companies.insert(resource_object=company)
tasks = self.task_queue.get_filtered_tasks()
self.assertEqual(len(tasks), 1, u'Invalid number of Deferred tasks')
deferred.run(
tasks[0].payload) # Doesn't return anything so nothing to test
koalacompanies.Companies.delete(resource_uid=company_uid)
tasks = self.task_queue.get_filtered_tasks()
self.assertEqual(len(tasks), 2, u'Invalid number of Deferred tasks')
deferred.run(
tasks[1].payload) # Doesn't return anything so nothing to test
search_result = koalacompanies.Companies.search(
query_string='company_name: {}'.format(
self.test_company['company_name']))
self.assertEqual(search_result.results_count, 0,
u'Query returned incorrect count')
self.assertEqual(len(search_result.results), 0,
u'Query returned incorrect number of results')
def RunAllDeferredTasks(tb, queue_name=None):
"""Runs all deferred tasks in specified queue.
If no queue is specified, then runs all deferred tasks in queues
known to this module, in QUEUE_NAMES.
Args:
tb: Your test's testbed instance.
queue_name: String. The name the queue whose tasks should be run.
"""
if queue_name:
queue_names = [queue_name]
else:
queue_names = QUEUE_NAMES
taskqueue = tb.get_stub(testbed.TASKQUEUE_SERVICE_NAME)
for name in queue_names:
try:
tasks = taskqueue.GetTasks(name)
for task in tasks:
deferred.run(base64.b64decode(task['body']))
taskqueue.DeleteTask(name, task['name'])
except KeyError:
# If a queue hasn't had a task added to it then we'll get a harmless
# keyerror when trying to get any tasks that have been added to it.
pass
def run_queue_tasks(self, queue='default'):
api_tasks = self.task_stub.GetTasks(queue)
while len(api_tasks) > 0:
self.task_stub.FlushQueue(queue)
for api_task in api_tasks:
deferred.run(base64.b64decode(api_task['body']))
api_tasks = self.task_stub.GetTasks(queue)
def ExecuteDeferredTasks(self, task_queue_name):
task_queue = self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME)
tasks = task_queue.GetTasks(task_queue_name)
task_queue.FlushQueue(task_queue_name)
for task in tasks:
deferred.run(base64.b64decode(task['body']))
self.ExecuteDeferredTasks(task_queue_name)
def runDeferredTasks(self, queue='default'):
tasks = self.taskqueue_stub.GetTasks(queue)
while tasks:
self.taskqueue_stub.FlushQueue(queue)
for task in tasks:
deferred.run(base64.b64decode(task['body']))
tasks = self.taskqueue_stub.GetTasks(queue)
def test_broadcast_fcm(self):
for client_type in ClientType.FCM_CLIENTS:
client = MobileClient(parent=ndb.Key(Account, 'user_id'),
user_id='user_id',
messaging_id='token',
client_type=client_type,
device_uuid='uuid',
display_name='Phone')
client_key = client.put()
from notifications.base_notification import BaseNotification
with patch.object(BaseNotification, 'send') as mock_send:
TBANSHelper.broadcast([client_type], 'Broadcast',
'Test broadcast')
# Make sure we didn't send to Android
mock_send.assert_not_called()
# Make sure we'll send to FCM clients
tasks = self.taskqueue_stub.get_filtered_tasks(
queue_names='push-notifications')
self.assertEqual(len(tasks), 1)
# Make sure our taskqueue tasks execute what we expect
with patch.object(TBANSHelper, '_send_fcm') as mock_send_fcm:
deferred.run(tasks[0].payload)
mock_send_fcm.assert_called_once_with([client], ANY)
# Make sure the notification is a BroadcastNotification
notification = mock_send_fcm.call_args[0][1]
self.assertTrue(isinstance(notification,
BroadcastNotification))
self.taskqueue_stub.FlushQueue('push-notifications')
client_key.delete()
def test_forgot_password_post_sends_email(self):
self.assertNotLoggedIn()
profile = self.create_profile()
response = self.app.get(self.uri_for('forgot-password'))
form = response.forms['forgot-password']
form['email'] = profile.email
response = form.submit()
# Check the task was put on the mail queue.
tasks = self.taskqueue_stub.get_filtered_tasks(queue_names='mail')
self.assertIn('mail', tasks[0].headers['X-AppEngine-QueueName'])
task, = tasks
deferred.run(task.payload)
messages = self.mail_stub.get_sent_messages()
self.assertLength(1, messages)
message, = messages
profile = Profile.get(profile.key())
# Reload profile to get new activation key.
self.assertEqual('"%s" <%s>' % (profile.name, profile.email),
message.to)
self.assertEqual(constants.FULL_NO_REPLY_EMAIL, message.sender)
self.assertEqual(constants.FULL_SUPPORT_EMAIL, message.reply_to)
self.assertIn(profile.activation_key, message.body.decode())
self.assertIn(profile.activation_key, message.html.decode())
recover_uri = self.uri_for('forgot-password', k=profile.activation_key)
self.assertIn(recover_uri, message.body.decode())
self.assertIn(recover_uri, message.html.decode())
def test_user_groups_property_delete(self):
user_group = koalausers.UserGroups.new(**self.test_user_group)
user_group_uid = koalausers.UserGroups.insert(resource_object=user_group)
tasks = self.task_queue.get_filtered_tasks()
self.assertEqual(len(tasks), 3, u'Invalid number of Deferred tasks')
deferred.run(tasks[0].payload) # Doesn't return anything so nothing to test
deferred.run(tasks[1].payload) # Doesn't return anything so nothing to test
deferred.run(tasks[2].payload) # Doesn't return anything so nothing to test
self.task_queue.FlushQueue('search-index-update')
self.task_queue.FlushQueue('deferredwork')
user_groups = koalausers.UserGroups.get_user_groups(user_uid=self.test_user_group['user_uid'])
self.assertEqual(len(user_groups), 1, u'Incorrect number of user groups returned')
koalausers.UserGroups.delete_by_group_uid(user_uid=self.test_user_group['user_uid'], group_uid=self.test_user_group['group_uid'])
tasks = self.task_queue.get_filtered_tasks()
self.assertEqual(len(tasks), 2, u'Invalid number of Deferred tasks')
deferred.run(tasks[0].payload) # Doesn't return anything so nothing to test
deferred.run(tasks[1].payload) # Doesn't return anything so nothing to test
user = koalausers.Users.get(resource_uid=self.test_user_group['user_uid'])
self.assertEqual(user.groups, {}, u'User groups mismatch')
self.assertFalse(user_group_uid in user.permissions.acl, u'User groups mismatch')
def mark_as_error_and_reload(self, fbl):
# Now let's "rename" the source to be id 222
error = (400, {"error": {"message": "Page ID 111 was migrated to page ID 222. Please update your API calls to the new ID", "code": 21, "type": "OAuthException"}})
fb_api.FBAPI.results.update({
URL_111: error,
'/v2.8/111/feed': error,
})
fbl.clear_local_cache()
# Loading it again should trigger a deferred task to rename to 222
try:
fbl.get(fb_api.LookupThingFeed, '111')
self.fail("Fetching renamed ID unexpectedly worked")
except fb_api.NoFetchedDataException:
pass
# Let's verify a task got created, and run it now
tasks = self.taskqueue_stub.get_filtered_tasks()
self.assertEqual(len(tasks), 1)
task = tasks[0]
deferred.run(task.payload)
# We have to handle "in case of queue_name, set the default manually" scenario ourselves
self.taskqueue_stub.DeleteTask(task.queue_name or 'default', task.name)
def test_signup_sends_welcome_email(self):
# Sign up successfully
response = self.app.post(self.uri_for('signup'), self.SIGNUP_DATA)
self.assertRedirects(response, self.uri_for('dashboard', tour=''))
# Check that a profile was created
profile = Profile.get_by_email(self.SIGNUP_DATA['email'])
self.assertIsNotNone(profile)
# Check that a mail-sending task is in the queue
tasks = self.taskqueue_stub.GetTasks('mail')
self.assertLength(1, tasks)
# Run the task (it should be a deferred call) and check that an e-mail
# is sent
task, = tasks
deferred.run(base64.b64decode(task['body']))
messages = self.mail_stub.get_sent_messages()
self.assertLength(1, messages)
message, = messages
self.assertEqual('"%s" <%s>' % (profile.name, profile.email), message.to)
self.assertEqual('Welcome to Daily Meeting!', message.subject)
self.assertEqual('"Daily Meeting" <*****@*****.**>',
message.sender)
self.assertEqual('"Daily Meeting Support" <*****@*****.**>',
message.reply_to)
activation_key = Profile.all().get().activation_key
activation_url = self.uri_for('profile.activate', k=activation_key)
self.assertIn(activation_url, message.body.decode())
self.assertIn(activation_url, message.html.decode())
def ExecuteDeferredTasks(self, task_queue_name):
task_queue = self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME)
tasks = task_queue.GetTasks(task_queue_name)
task_queue.FlushQueue(task_queue_name)
for task in tasks:
deferred.run(base64.b64decode(task['body']))
self.ExecuteDeferredTasks(task_queue_name)
def run_deferred_tasks(self, queue_name='default'):
import base64
from google.appengine.ext import deferred
tasks = self.get_queued_tasks(queue_name=queue_name)
for task in tasks:
deferred.run(base64.b64decode(task['body']))
# flush tasks from queue after they are run
self.flush_queue(queue_name=queue_name)
def RunAllDeferredTasks(self, queue_name='default'):
"""Runs all deferred tasks in specified queue."""
taskqueue = self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME)
tasks = taskqueue.GetTasks(queue_name)
for task in tasks:
deferred.run(base64.b64decode(task['body']))
taskqueue.DeleteTask(queue_name, task['name'])
def RunAllDeferredTasks(self, queue_name='default'):
"""Runs all deferred tasks in specified queue."""
taskqueue = self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME)
tasks = taskqueue.GetTasks(queue_name)
for task in tasks:
deferred.run(base64.b64decode(task['body']))
taskqueue.DeleteTask(queue_name, task['name'])
def _execute_next_task(self):
# Get the task out of the queue
tasks = self.taskqueue_stub.get_filtered_tasks()
# Run the task
task = tasks[0]
self.taskqueue_stub.FlushQueue('emailFetch')
self.taskqueue_stub.FlushQueue('adminStats')
deferred.run(task.payload)
def test_post_adds_points_to_global(self):
data = json.dumps(self.DATA)
self.app.post('/api/v1/live', data)
tasks = self.taskqueue_stub.GetTasks('live')
self.assertEqual(1, len(tasks))
# Run the task
for task in tasks:
deferred.run(base64.b64decode(task['body']))
def post(self):
headers = ['%s:%s' % (k, v) for k, v in self.request.headers.items()
if k.lower().startswith('x-appengine-')]
logging.info(', '.join(headers))
try:
run(self.request.data)
except PermanentTaskFailure, e:
logging.exception('Permanent failure attempting to execute task')
def post(self):
headers = ["%s:%s" % (k, v) for k, v in self.request.headers.items()
if k.lower().startswith("x-appengine-")]
logging.info(", ".join(headers))
try:
run(self.request.data)
except PermanentTaskFailure, e:
logging.exception("Permanent failure attempting to execute task: %s." %
e)
def test_delete_tags(self):
"""Test destroying a Tag in small batches to test multiple defer calls."""
tag_model._delete_tags(_ModelWithTags, key=self.tag1.key, batch_size=2)
tasks = self.taskqueue_stub.get_filtered_tasks()
deferred.run(tasks[0].payload)
deferred.run(tasks[0].payload)
self.assertNotIn(self.tag1_data, self.entity1.tags)
self.assertNotIn(self.tag1_data, self.entity2.tags)
self.assertNotIn(self.tag1_data, self.entity3.tags)
def test_post_adds_points_to_global(self):
data = json.dumps(self.DATA)
self.app.post('/api/v1/live', data)
tasks = self.taskqueue_stub.GetTasks('live')
self.assertEqual(1, len(tasks))
# Run the task
for task in tasks:
deferred.run(base64.b64decode(task['body']))
def run_deferred_tasks(self, queue='default'):
"""Run all tasks that have been deferred.
Arguments:
queue: The task queue in which the deferred tasks are queued.
"""
taskqueue_stub = self.testbed.get_stub(TASKQUEUE_SERVICE_NAME)
for task in taskqueue_stub.GetTasks(queue):
deferred.run(base64.b64decode(task['body']))
taskqueue_stub.FlushQueue(queue)
def get_response(self, event, body):
if isinstance(body, dict):
body = json.dumps(body)
signature = handlers.make_signature(body)
resp = app.post('/webhook', body,
{'X-Github-Event': event,
'X-Hub-Signature': signature})
for task in self.taskqueue.get_filtered_tasks():
deferred.run(task.payload)
return resp
def run_deferred_tasks(self, queue='default'):
"""Run all tasks that have been deferred.
Arguments:
queue: The task queue in which the deferred tasks are queued.
"""
taskqueue_stub = self.testbed.get_stub(TASKQUEUE_SERVICE_NAME)
for task in taskqueue_stub.GetTasks(queue):
deferred.run(base64.b64decode(task['body']))
taskqueue_stub.FlushQueue(queue)
def post(self):
headers = [
"%s:%s" % (k, v) for k, v in self.request.headers.items()
if k.lower().startswith("x-appengine-")
]
logging.info(", ".join(headers))
try:
run(self.request.data)
except PermanentTaskFailure, e:
logging.exception(
"Permanent failure attempting to execute task: %s." % e)
def run_tasks_in_queue(self, queue_name=None):
""" Run tasks in the testbed taskqueue stub
"""
for queue in self.taskqueue_stub.GetQueues():
if queue_name is None or queue == queue['name']:
tasks = self.taskqueue_stub.GetTasks(queue['name'])
for t in tasks:
logging.debug('Running task %s from queue %s' % (t['name'], queue['name']))
deferred.run(base64.b64decode(t['body']))
self.taskqueue_stub.DeleteTask(queue['name'], t['name'])
def post(self):
headers = [
'%s:%s' % (k, v) for k, v in self.request.headers.items()
if k.lower().startswith('x-appengine-')
]
logging.info(', '.join(headers))
try:
run(self.request.data)
except PermanentTaskFailure, e:
logging.exception('Permanent failure attempting to execute task')
def execute_all_deferred_tasks(self, queue_name='default'):
"""Executes all pending deferred tasks."""
count = 0
for task in self.taskq.GetTasks(queue_name):
data = base64.b64decode(task['body'])
# TODO(psimakov): do we need to pull out the namespace?
# namespace = dict(task['headers']).get(
# 'X-AppEngine-Current-Namespace', '')
deferred.run(data)
count += 1
self.taskq.FlushQueue(queue_name)
return count
def run_tasks_in_queue(self, queue_name=None):
""" Run tasks in the testbed taskqueue stub
"""
for queue in self.taskqueue_stub.GetQueues():
if queue_name is None or queue == queue['name']:
tasks = self.taskqueue_stub.GetTasks(queue['name'])
for t in tasks:
logging.debug('Running task %s from queue %s' %
(t['name'], queue['name']))
deferred.run(base64.b64decode(t['body']))
self.taskqueue_stub.DeleteTask(queue['name'], t['name'])
def ah_queue_deferred(request):
"""
We use this to get around the app engine patch interfering with the
deferred library. Using the default deferred causes all sorts of sporadic
errors to occur from the sequence of unzipping the django libraries.
"""
from google.appengine.ext import deferred
try:
deferred.run(request.raw_post_data)
except PermanentTaskFailure, e:
logging.error("Permanent task failure; task will never finish.")
raise e
def test_insert_search(self):
user = koalausers.Users.new(username='******', email_address='*****@*****.**', first_name='Test', raw_password='******', last_name='User ', language_preference='en')
koalausers.Users.insert(resource_object=user)
tasks = self.task_queue.get_filtered_tasks()
self.assertEqual(len(tasks), 1, u'Deferred task missing')
deferred.run(tasks[0].payload) # Doesn't return anything so nothing to test
search_result = koalausers.Users.search(query_string='username: test')
self.assertEqual(search_result.results_count, 1, u'Query returned incorrect count')
self.assertEqual(len(search_result.results), 1, u'Query returned incorrect number of results')
def tick(self):
tasks = self.get_filtered_tasks()
if not tasks:
raise NothingToDo()
for task in tasks:
try:
deferred.run(task.payload)
except deferred.PermanentTaskFailure:
pass
finally:
self.taskqueue.DeleteTask(task.queue_name or 'default', task.name)
def test_insert_search(self):
refresh_token = koalaoauth2.RefreshTokens.new(**self.test_refresh_token)
koalaoauth2.RefreshTokens.insert(resource_object=refresh_token)
tasks = self.task_queue.get_filtered_tasks()
self.assertEqual(len(tasks), 1, u'Deferred task missing')
deferred.run(tasks[0].payload) # Doesn't return anything so nothing to test
search_result = koalaoauth2.RefreshTokens.search(
query_string='token: {}'.format(self.test_refresh_token['token']))
self.assertEqual(search_result.results_count, 1, u'Query returned incorrect count')
self.assertEqual(len(search_result.results), 1, u'Query returned incorrect number of results')
def newmsg_test(self):
"""Tests that new msgs are correctly dealt with"""
resp = self.testapp.post('/new',
params=json.dumps(fake_slack_msg()),
content_type='application/json')
# make sure that has been deferred properly
tasks = self.taskqueue_stub.get_filtered_tasks()
assert len(tasks) == 1
# make sure it runs
deferred.run(tasks[0].payload)
assert len(models.Message.query().fetch(10)) == 1
# make sure it gave us an appropriate response
assert resp.json['text'] in msg.POSITIVE_RESPONSES
def test_insert_search(self):
grant = koalaoauth2.Grants.new(**self.test_grant)
inserted_uid = koalaoauth2.Grants.insert(resource_object=grant)
tasks = self.task_queue.get_filtered_tasks()
self.assertEqual(len(tasks), 1, u'Deferred task missing')
deferred.run(tasks[0].payload) # Doesn't return anything so nothing to test
search_result = koalaoauth2.Grants.search(
query_string='code: {}'.format(self.test_grant['code']))
self.assertEqual(search_result.results_count, 1, u'Query returned incorrect count')
self.assertEqual(len(search_result.results), 1, u'Query returned incorrect number of results')
def test_insert_search(self):
company = koalacompanies.Companies.new(**self.test_company)
koalacompanies.Companies.insert(resource_object=company)
tasks = self.task_queue.get_filtered_tasks()
self.assertEqual(len(tasks), 1, u'Deferred task missing')
deferred.run(tasks[0].payload) # Doesn't return anything so nothing to test
search_result = koalacompanies.Companies.search(
query_string='company_name: {}'.format(self.test_company['company_name']))
self.assertEqual(search_result.results_count, 1, u'Query returned incorrect count')
self.assertEqual(len(search_result.results), 1, u'Query returned incorrect number of results')
def run_deferred_tasks(test):
tasks = test.taskqueue.get_filtered_tasks()
test.taskqueue.FlushQueue("default")
while tasks:
for task in tasks:
if task.url == '/_ah/queue/deferred':
deferred.run(task.payload)
tasks = test.taskqueue.get_filtered_tasks()
for task in tasks:
# As soon as we hit a non-deferred task, bail out of here.
if task.url != '/_ah/queue/deferred':
return
test.taskqueue.FlushQueue("default")
def test_ca_update_on_import(self):
"""Test updating of last_comment CA when comments imported"""
asmnt_id = self.asmnt_comments.keys()[0]
asmnt_slug = all_models.Assessment.query.get(asmnt_id).slug
response = self.import_data(collections.OrderedDict([
("object_type", "Assessment"),
("Code*", asmnt_slug),
("Comments", "new comment1;;new comment2;;new comment3"),
]))
tasks = self.taskqueue_stub.get_filtered_tasks()
deferred.run(tasks[0].payload)
self._check_csv_response(response, {})
asmnt = all_models.Assessment.query.filter_by(slug=asmnt_slug).first()
self.assertEqual(asmnt.last_comment, "new comment3")
def test_forgot_password_post_with_email_trailing_whitespace(self):
profile = self.create_profile()
params = {'email': profile.email + ' '}
response = self.app.post(self.uri_for('forgot-password'), params)
self.assertOk(response)
tasks = self.taskqueue_stub.get_filtered_tasks(queue_names='mail')
self.assertIn('mail', tasks[0].headers['X-AppEngine-QueueName'])
task, = tasks
deferred.run(task.payload)
self.assertLength(1, self.mail_stub.get_sent_messages())
message, = self.mail_stub.get_sent_messages()
profile = Profile.get(profile.key())
self.assertEqual('"%s" <%s>' % (profile.name, profile.email), message.to)
