def testNoToken(self):
api = rest_api._RestApi('scope')
self.assertEqual(api.scopes, ['scope'])
fut_get_token = ndb.Future()
fut_get_token.set_result(None)
api.get_token_async = mock.create_autospec(api.get_token_async,
return_value=fut_get_token)
fut_urlfetch = ndb.Future()
fut_urlfetch.set_result(
test_utils.MockUrlFetchResult(200, {'foo': 'bar'}, 'yoohoo'))
ctx_urlfetch = mock.Mock(return_value=fut_urlfetch)
ndb.get_context().urlfetch = ctx_urlfetch
res = api.do_request('http://example.com')
self.assertEqual(res, (200, {'foo': 'bar'}, 'yoohoo'))
ctx_urlfetch.assert_called_once_with(
'http://example.com',
headers={'User-Agent': 'AppEngine-Python-GCS'},
follow_redirects=False,
payload=None,
method='GET',
deadline=None,
callback=None)
def testBasicCallWithUserAgent(self):
user_agent = 'Test User Agent String'
retry_params = api_utils.RetryParams(_user_agent=user_agent)
api = rest_api._RestApi('scope', retry_params=retry_params)
self.assertEqual(api.scopes, ['scope'])
fut_get_token = ndb.Future()
fut_get_token.set_result('blah')
api.get_token_async = mock.create_autospec(api.get_token_async,
return_value=fut_get_token)
fut_urlfetch = ndb.Future()
fut_urlfetch.set_result(
test_utils.MockUrlFetchResult(200, {'foo': 'bar'}, 'yoohoo'))
ctx_urlfetch = mock.Mock(return_value=fut_urlfetch)
ndb.get_context().urlfetch = ctx_urlfetch
res = api.do_request('http://example.com')
self.assertEqual(res, (200, {'foo': 'bar'}, 'yoohoo'))
ctx_urlfetch.assert_called_once_with(
'http://example.com',
headers={'authorization': 'OAuth blah',
'User-Agent': user_agent},
follow_redirects=False,
payload=None,
method='GET',
deadline=None,
callback=None)
def testBasicCall(self):
api = rest_api._RestApi('scope')
self.assertEqual(api.scopes, ['scope'])
fut_get_token = ndb.Future()
fut_get_token.set_result('blah')
api.get_token_async = mock.create_autospec(api.get_token_async,
return_value=fut_get_token)
fut_urlfetch = ndb.Future()
fut_urlfetch.set_result(
test_utils.MockUrlFetchResult(200, {'foo': 'bar'}, 'yoohoo'))
api.urlfetch_async = mock.create_autospec(api.urlfetch_async,
return_value=fut_urlfetch)
res = api.do_request('http://example.com')
self.assertEqual(res, (200, {'foo': 'bar'}, 'yoohoo'))
api.urlfetch_async.assert_called_once_with(
'http://example.com',
headers={'authorization': 'OAuth blah'},
follow_redirects=False,
payload=None,
method='GET',
deadline=None,
callback=None)
def setUp(self):
super(ApiTestCase, self).setUp()
self.provider = mock.Mock()
provider_future = ndb.Future()
provider_future.set_result(self.provider)
self.mock(config.api, '_get_config_provider_async', lambda: provider_future)
self.provider.get_async.return_value = ndb.Future()
self.provider.get_async.return_value.set_result(
('deadbeef', test_config_pb2.Config(param='value')))
def testNegativeTimeout(self):
api = rest_api._RestApi('scope')
fut1 = ndb.Future()
fut1.set_result(('token1', 0))
fut2 = ndb.Future()
fut2.set_result(('token2', 0))
api.make_token_async = mock.create_autospec(
api.make_token_async, side_effect=[fut1, fut2])
token1 = api.get_token()
token2 = api.get_token()
self.assertNotEqual(token1, token2)
def map_fn(self, entities):
mimail_client2 = MiMailClient2(self.sendgrid['USERNAME'], self.sendgrid['PASSWORD'])
for entity in entities:
self.count += 1
countdown_sec = self.count // settings.QUEUE_CHUNKS_SIZE
if not self.is_dry_run:
r = tasks.addTask(['worker', 'worker2'],
mimail_client2.run,
schedule=self.schedule_job,
content=self.content,
recipient_queues=entity,
sharding_count_name=self.sharding_count_name,
_countdown=countdown_sec).get_result()
if r:
self.success_worker += 1
n = ndb.Future('yo dawg')
n.set_result('yo')
yield n
else:
self.fail_worker += 1
entity.status = 'fail_worker'
yield entity.put_async()
else:
r2 = true_false_pick(self.dry_run_fail_rate)
if r2:
r = tasks.addTask(['worker', 'worker2'],
mimail_client2.run,
schedule=self.schedule_job,
content=self.content,
recipient_queues=entity,
sharding_count_name=self.sharding_count_name,
_countdown=countdown_sec).get_result()
if r:
self.success_worker += 1
n = ndb.Future('yo dawg')
n.set_result('yo')
yield n
else:
self.fail_worker += 1
entity.status = 'fail_worker'
yield entity.put_async()
else:
self.fail_worker += 1
entity.status = 'fail_worker'
yield entity.put_async()
def testRetryAfterDoRequestUrlFetchTimeout(self):
api = rest_api._RestApi('scope')
resp_fut1 = ndb.Future()
resp_fut1.set_exception(urlfetch.DownloadError())
resp_fut2 = ndb.Future()
resp_fut2.set_result(
test_utils.MockUrlFetchResult(httplib.ACCEPTED, None, None))
ndb.Context.urlfetch = mock.create_autospec(
ndb.Context.urlfetch, side_effect=[resp_fut1, resp_fut2])
self.assertEqual(httplib.ACCEPTED, api.do_request('foo')[0])
self.assertEqual(2, ndb.Context.urlfetch.call_count)
def setUp(self):
super(ApiTestCase, self).setUp()
self.provider = mock.Mock()
self.mock(config.api, '_get_config_provider', lambda: self.provider)
self.provider.get_async.return_value = ndb.Future()
self.provider.get_async.return_value.set_result(
('deadbeef', 'param: "value"'))
def inner(*args, **kwargs):
ret = ndb.Future()
ret.set_result(res.SerializeToString())
if kwargs.get('response_headers') is not None:
kwargs['response_headers'][
'Some-Bytes-Bin'] = 'MTIzNA==' # '1234'
return ret
def test_get_projects(self):
self.provider.get_projects_async.return_value = ndb.Future()
self.provider.get_projects_async.return_value.set_result([
{
'id': 'chromium',
'repo_type': 'GITILES',
'repo_url': 'https://chromium.googlesource.com/chromium/src',
'name': 'Chromium browser'
},
{
'id': 'infra',
'repo_type': 'GITILES',
'repo_url': 'https://chromium.googlesource.com/infra/infra',
},
])
projects = config.get_projects()
self.assertEqual(projects, [
config.Project(
id='chromium',
repo_type='GITILES',
repo_url='https://chromium.googlesource.com/chromium/src',
name='Chromium browser'),
config.Project(
id='infra',
repo_type='GITILES',
repo_url='https://chromium.googlesource.com/infra/infra',
name=''),
])
def test_pop_future(self):
items = [ndb.Future() for _ in xrange(5)]
items[1].set_result(None)
items[3].set_result('foo')
inputs = items[:]
mapping.pop_future_done(inputs)
self.assertEqual([items[0], items[2], items[4]], inputs)
def testPickling(self):
retry_params = api_utils.RetryParams(max_retries=1000)
api = rest_api._RestApi('scope',
service_account_id=1,
retry_params=retry_params)
self.assertNotEqual(None, api.get_token())
pickled_api = pickle.loads(pickle.dumps(api))
self.assertEqual(
0,
len(set(api.__dict__.keys()) ^ set(pickled_api.__dict__.keys())))
for k, v in api.__dict__.iteritems():
if not hasattr(v, '__call__'):
self.assertEqual(v, pickled_api.__dict__[k])
pickled_api.token = None
fut_urlfetch = ndb.Future()
fut_urlfetch.set_result(
test_utils.MockUrlFetchResult(200, {'foo': 'bar'}, 'yoohoo'))
pickled_api.urlfetch_async = mock.create_autospec(
pickled_api.urlfetch_async, return_value=fut_urlfetch)
res = pickled_api.do_request('http://example.com')
self.assertEqual(res, (200, {'foo': 'bar'}, 'yoohoo'))
def test_http_403(self):
res = ndb.Future()
res.set_result(self.Response(403, 'Not authorized'))
self.mock(delegation, '_urlfetch_async', lambda **_k: res)
with self.assertRaises(delegation.DelegationAuthorizationError):
delegation.delegate(auth_service_url='https://example.com')
def test_http_500(self):
res = ndb.Future()
res.set_result(self.Response(500, 'Server internal error'))
self.mock(delegation, '_urlfetch_async', lambda **_k: res)
with self.assertRaises(delegation.DelegationTokenCreationError):
delegation.delegate(auth_service_url='https://example.com')
def urlfetch(url,
payload=None,
method='GET',
headers={},
allow_truncated=False,
follow_redirects=True,
validate_certificate=None,
deadline=None,
callback=None):
fut = ndb.Future()
rpc = _urlfetch.create_rpc(deadline=deadline, callback=callback)
_urlfetch.make_fetch_call(rpc,
url,
payload=payload,
method=method,
headers=headers,
allow_truncated=allow_truncated,
follow_redirects=follow_redirects,
validate_certificate=validate_certificate)
def _on_completion():
try:
result = rpc.get_result()
except Exception, err:
_, _, tb = sys.exc_info()
fut.set_exception(err, tb)
else:
def testA(self):
future = ndb.Future()
lock = Lock()
messages = collections.deque()
@utils.tasklet(ff=True)
def work(i):
messages.append('init %s' % i)
with (yield lock.acquire()):
messages.append('acquired %s' % i)
yield future
messages.append('released %s' % i)
raise ndb.Return(11)
w1 = work(1)
assert messages.popleft() == 'init 1'
assert messages.popleft() == 'acquired 1'
w2 = work(2)
assert messages.popleft() == 'init 2'
assert not messages
future.set_result(11)
w2.get_result()
w1.get_result()
assert len(messages) == 3
assert messages.popleft() == 'released 1'
assert messages.popleft() == 'acquired 2'
assert messages.popleft() == 'released 2'
def test_get_ref_configs(self):
self.provider.get_ref_configs_async.return_value = ndb.Future()
self.provider.get_ref_configs_async.return_value.set_result({
'projects/chromium/refs/heads/master': ('dead', 'param: "master"'),
'projects/chromium/refs/non-branch': ('beef', 'param: "ref"'),
'projects/v8/refs/heads/master': ('aaaa', 'param: "value2"'),
'projects/skia/refs/heads/master': ('badcoffee', 'invalid config'),
})
actual = config.get_ref_configs('bar.cfg', test_config_pb2.Config)
self.assertIsInstance(
actual['skia']['refs/heads/master'][2], config.ConfigFormatError)
expected = {
'chromium': {
'refs/heads/master': (
'dead', test_config_pb2.Config(param='master'), None),
'refs/non-branch': (
'beef', test_config_pb2.Config(param='ref'), None),
},
'v8': {
'refs/heads/master': (
'aaaa', test_config_pb2.Config(param='value2'), None),
},
'skia': {
'refs/heads/master': (
'badcoffee',
None,
actual['skia']['refs/heads/master'][2],
),
}
}
self.assertEqual(expected, actual)
def testNoExpiredToken(self):
with mock.patch('time.time') as t:
t.side_effect = [2, 4, 5, 6]
api = rest_api._RestApi('scope')
fut1 = ndb.Future()
fut1.set_result(('token1', 3 + api.expiration_headroom))
fut2 = ndb.Future()
fut2.set_result(('token2', 7 + api.expiration_headroom))
api.make_token_async = mock.create_autospec(
api.make_token_async, side_effect=[fut1, fut2])
token = api.get_token()
self.assertEqual('token1', token)
token = api.get_token()
self.assertEqual('token2', token)
token = api.get_token()
self.assertEqual('token2', token)
def acquire(self):
fut = ndb.Future()
if self.locked():
self._waiting.append(fut)
else:
self.counter -= 1
fut.set_result(self)
return fut
def testSetAndGet_NoMemcache(self, async_mock):
async_mock.return_value = ndb.Future()
async_mock.return_value.set_result(None)
new_account = SampleSerializableClass('Some account data.')
stored_object.Set('chris', new_account)
chris_account = stored_object.Get('chris')
self.assertEqual(new_account, chris_account)
def testUrlFetchCalledWithUserProvidedDeadline(self):
retry_params = api_utils.RetryParams(urlfetch_timeout=90)
api = rest_api._RestApi('scope', retry_params=retry_params)
resp_fut1 = ndb.Future()
resp_fut1.set_exception(urlfetch.DownloadError())
resp_fut2 = ndb.Future()
resp_fut2.set_result(
test_utils.MockUrlFetchResult(httplib.ACCEPTED, None, None))
ndb.Context.urlfetch = mock.create_autospec(
ndb.Context.urlfetch, side_effect=[resp_fut1, resp_fut2])
self.assertEqual(httplib.ACCEPTED, api.do_request('foo')[0])
self.assertEqual(90,
ndb.Context.urlfetch.call_args_list[0][1]['deadline'])
self.assertEqual(90,
ndb.Context.urlfetch.call_args_list[1][1]['deadline'])
def put_async(*args, **kwargs):
callers = [i[3] for i in inspect.stack()]
self.assertIn('schedule_request', callers)
out = ndb.Future()
if (index[0] % SKIP) == 2:
out.set_exception(search.Error())
else:
out.set_result(old_put_async(*args, **kwargs).get_result())
return out
def map_fn(items):
actual.extend(items)
# Note that it is returning more Future than what is called. It's fine.
for _ in xrange(len(items) * 5):
n = ndb.Future('yo dawg')
# TODO(maruel): It'd be nice to not set them completed right away to
# have better code coverage but I'm not sure how to do this.
n.set_result('yo')
yield n
def testNoRetryAfterDoRequestUrlFetchTimeout(self):
retry_params = api_utils.RetryParams(max_retries=0)
api = rest_api._RestApi('scope', retry_params=retry_params)
fut = ndb.Future()
fut.set_exception(urlfetch.DownloadError())
ndb.Context.urlfetch = mock.create_autospec(ndb.Context.urlfetch,
return_value=fut)
self.assertRaises(urlfetch.DownloadError, api.do_request, 'foo')
def test_http_500(self):
res = ndb.Future()
res.set_result(self.Response(500, 'Server internal error'))
self.mock(service_account, '_urlfetch_async', lambda **_k: res)
with self.assertRaises(exceptions.TokenCreationError):
delegation.delegate(audience=['*'],
services=['*'],
token_server_url='https://example.com')
def new_key_callback_async():
key = None
if new_key_callback:
key = new_key_callback()
if isinstance(key, ndb.Future):
return key
future = ndb.Future()
future.set_result(key)
return future
def setUp(self):
super(RemoteTestCase, self).setUp()
self.mock(net, 'json_request_async', mock.Mock())
net.json_request_async.side_effect = self.json_request_async
self.provider = remote.Provider('luci-config.appspot.com')
provider_future = ndb.Future()
provider_future.set_result(self.provider)
self.mock(remote, 'get_provider_async', lambda: provider_future)
def test_http_403(self):
res = ndb.Future()
res.set_result(self.Response(403, 'Not authorized'))
self.mock(service_account, '_urlfetch_async', lambda **_k: res)
with self.assertRaises(exceptions.TokenAuthorizationError):
delegation.delegate(audience=['*'],
services=['*'],
token_server_url='https://example.com')
def testNoRetryAfterDoRequestResponseTimeout(self):
retry_params = api_utils.RetryParams(max_retries=0)
api = rest_api._RestApi('scope', retry_params=retry_params)
fut = ndb.Future()
fut.set_result(
test_utils.MockUrlFetchResult(httplib.REQUEST_TIMEOUT, None, None))
ndb.Context.urlfetch = mock.create_autospec(ndb.Context.urlfetch,
return_value=fut)
self.assertEqual(httplib.REQUEST_TIMEOUT, api.do_request('foo')[0])
def test_has_project_access_no_access(self):
self.set_up_identity()
self.mock(api, 'get_project_config_async', mock.Mock())
api.get_project_config_async.return_value = ndb.Future()
api.get_project_config_async.return_value.set_result(
project_config_pb2.ProjectCfg(), # access not configured => internal
)
self.assertFalse(config.has_project_access('chromium'))
