def test_close(self):
proc = psutil.Process()
ip_addresses = {
info[4][0]
for info in
socket.getaddrinfo('example.com', 80, socket.AF_INET,
socket.SOCK_DGRAM, socket.IPPROTO_IP)
}
self.assertGreater(len(ip_addresses), 0)
protocol = Protocol(
config=Configuration(service_endpoint='http://example.com',
credentials=Credentials('A', 'B'),
auth_type=NOAUTH,
version=Version(Build(15, 1)),
retry_policy=FailFast()))
session = protocol.get_session()
session.get('http://example.com')
self.assertEqual(
len({
p.raddr[0]
for p in proc.connections() if p.raddr[0] in ip_addresses
}), 1)
protocol.release_session(session)
protocol.close()
self.assertEqual(
len({
p.raddr[0]
for p in proc.connections() if p.raddr[0] in ip_addresses
}), 0)
def test_protocol_instance_caching(self, m):
# Verify that we get the same Protocol instance for the same combination of (endpoint, credentials)
m.get('https://example.com/EWS/types.xsd', status_code=200)
base_p = Protocol(config=Configuration(
service_endpoint='https://example.com/Foo.asmx', credentials=Credentials('A', 'B'),
auth_type=NTLM, version=Version(Build(15, 1)), retry_policy=FailFast()
))
for i in range(10):
p = Protocol(config=Configuration(
service_endpoint='https://example.com/Foo.asmx', credentials=Credentials('A', 'B'),
auth_type=NTLM, version=Version(Build(15, 1)), retry_policy=FailFast()
))
self.assertEqual(base_p, p)
self.assertEqual(id(base_p), id(p))
self.assertEqual(hash(base_p), hash(p))
self.assertEqual(id(base_p._session_pool), id(p._session_pool))
def test_autodiscover_cache(self, m):
# Mock the default endpoint that we test in step 1 of autodiscovery
m.post(self.dummy_ad_endpoint,
status_code=200,
content=self.dummy_ad_response)
# Also mock the EWS URL. We try to guess its auth method as part of autodiscovery
m.post(self.dummy_ews_endpoint, status_code=200)
discovery = Autodiscovery(
email=self.account.primary_smtp_address,
credentials=self.account.protocol.credentials,
retry_policy=self.retry_policy,
)
# Not cached
self.assertNotIn(discovery._cache_key, autodiscover_cache)
discovery.discover()
# Now it's cached
self.assertIn(discovery._cache_key, autodiscover_cache)
# Make sure the cache can be looked by value, not by id(). This is important for multi-threading/processing
self.assertIn(
(self.account.primary_smtp_address.split('@')[1],
Credentials(self.account.protocol.credentials.username,
self.account.protocol.credentials.password), True),
autodiscover_cache)
# Poison the cache with a failing autodiscover endpoint. discover() must handle this and rebuild the cache
autodiscover_cache[discovery._cache_key] = AutodiscoverProtocol(
config=Configuration(
service_endpoint=
'https://example.com/Autodiscover/Autodiscover.xml',
credentials=Credentials('leet_user', 'cannaguess'),
auth_type=NTLM,
retry_policy=FailFast(),
))
m.post('https://example.com/Autodiscover/Autodiscover.xml',
status_code=404)
discovery.discover()
self.assertIn(discovery._cache_key, autodiscover_cache)
# Make sure that the cache is actually used on the second call to discover()
_orig = discovery._step_1
def _mock(slf, *args, **kwargs):
raise NotImplementedError()
discovery._step_1 = MethodType(_mock, discovery)
discovery.discover()
# Fake that another thread added the cache entry into the persistent storage but we don't have it in our
# in-memory cache. The cache should work anyway.
autodiscover_cache._protocols.clear()
discovery.discover()
discovery._step_1 = _orig
# Make sure we can delete cache entries even though we don't have it in our in-memory cache
autodiscover_cache._protocols.clear()
del autodiscover_cache[discovery._cache_key]
# This should also work if the cache does not contain the entry anymore
del autodiscover_cache[discovery._cache_key]
def test_session(self, m):
m.get('https://example.com/EWS/types.xsd', status_code=200)
protocol = Protocol(config=Configuration(
service_endpoint='https://example.com/Foo.asmx', credentials=Credentials('A', 'B'),
auth_type=NTLM, version=Version(Build(15, 1)), retry_policy=FailFast()
))
session = protocol.create_session()
new_session = protocol.renew_session(session)
self.assertNotEqual(id(session), id(new_session))
def test_autodiscover_direct_gc(self, m):
# Test garbage collection of the autodiscover cache
c = Credentials('leet_user', 'cannaguess')
autodiscover_cache[('example.com', c)] = AutodiscoverProtocol(
config=Configuration(
service_endpoint=
'https://example.com/Autodiscover/Autodiscover.xml',
credentials=c,
auth_type=NTLM,
retry_policy=FailFast(),
))
self.assertEqual(len(autodiscover_cache), 1)
autodiscover_cache.__del__()
def test_close_autodiscover_connections(self, m):
# A live test that we can close TCP connections
c = Credentials('leet_user', 'cannaguess')
autodiscover_cache[('example.com', c)] = AutodiscoverProtocol(
config=Configuration(
service_endpoint=
'https://example.com/Autodiscover/Autodiscover.xml',
credentials=c,
auth_type=NTLM,
retry_policy=FailFast(),
))
self.assertEqual(len(autodiscover_cache), 1)
close_connections()
def test_autodiscover_cache(self, m):
# Empty the cache
from exchangelib.autodiscover import _autodiscover_cache
_autodiscover_cache.clear()
cache_key = (self.account.domain, self.account.protocol.credentials)
# Not cached
self.assertNotIn(cache_key, _autodiscover_cache)
discover(email=self.account.primary_smtp_address,
credentials=self.account.protocol.credentials)
# Now it's cached
self.assertIn(cache_key, _autodiscover_cache)
# Make sure the cache can be looked by value, not by id(). This is important for multi-threading/processing
self.assertIn(
(self.account.primary_smtp_address.split('@')[1],
Credentials(self.account.protocol.credentials.username,
self.account.protocol.credentials.password), True),
_autodiscover_cache)
# Poison the cache. discover() must survive and rebuild the cache
_autodiscover_cache[cache_key] = AutodiscoverProtocol(
config=Configuration(
service_endpoint='https://example.com/blackhole.asmx',
credentials=Credentials('leet_user', 'cannaguess'),
auth_type=NTLM,
retry_policy=FailFast(),
))
m.post('https://example.com/blackhole.asmx', status_code=404)
discover(email=self.account.primary_smtp_address,
credentials=self.account.protocol.credentials)
self.assertIn(cache_key, _autodiscover_cache)
# Make sure that the cache is actually used on the second call to discover()
_orig = exchangelib.autodiscover._try_autodiscover
def _mock(*args, **kwargs):
raise NotImplementedError()
exchangelib.autodiscover._try_autodiscover = _mock
discover(email=self.account.primary_smtp_address,
credentials=self.account.protocol.credentials)
# Fake that another thread added the cache entry into the persistent storage but we don't have it in our
# in-memory cache. The cache should work anyway.
_autodiscover_cache._protocols.clear()
discover(email=self.account.primary_smtp_address,
credentials=self.account.protocol.credentials)
exchangelib.autodiscover._try_autodiscover = _orig
# Make sure we can delete cache entries even though we don't have it in our in-memory cache
_autodiscover_cache._protocols.clear()
del _autodiscover_cache[cache_key]
# This should also work if the cache does not contain the entry anymore
del _autodiscover_cache[cache_key]
def test_decrease_poolsize(self):
protocol = Protocol(config=Configuration(
service_endpoint='https://example.com/Foo.asmx', credentials=Credentials('A', 'B'),
auth_type=NTLM, version=Version(Build(15, 1)), retry_policy=FailFast()
))
self.assertEqual(protocol._session_pool.qsize(), Protocol.SESSION_POOLSIZE)
protocol.decrease_poolsize()
self.assertEqual(protocol._session_pool.qsize(), 3)
protocol.decrease_poolsize()
self.assertEqual(protocol._session_pool.qsize(), 2)
protocol.decrease_poolsize()
self.assertEqual(protocol._session_pool.qsize(), 1)
with self.assertRaises(SessionPoolMinSizeReached):
protocol.decrease_poolsize()
self.assertEqual(protocol._session_pool.qsize(), 1)
def test_close(self):
# Don't use example.com here - it does not resolve or answer on all ISPs
proc = psutil.Process()
ip_addresses = {
info[4][0]
for info in
socket.getaddrinfo('httpbin.org', 80, socket.AF_INET,
socket.SOCK_DGRAM, socket.IPPROTO_IP)
}
def conn_count():
return len(
[p for p in proc.connections() if p.raddr[0] in ip_addresses])
self.assertGreater(len(ip_addresses), 0)
protocol = Protocol(
config=Configuration(service_endpoint='http://httpbin.org',
credentials=Credentials('A', 'B'),
auth_type=NOAUTH,
version=Version(Build(15, 1)),
retry_policy=FailFast(),
max_connections=3))
# Merely getting a session should not create conections
session = protocol.get_session()
self.assertEqual(conn_count(), 0)
# Open one URL - we have 1 connection
session.get('http://httpbin.org')
self.assertEqual(conn_count(), 1)
# Open the same URL - we should still have 1 connection
session.get('http://httpbin.org')
self.assertEqual(conn_count(), 1)
# Open some more connections
s2 = protocol.get_session()
s2.get('http://httpbin.org')
s3 = protocol.get_session()
s3.get('http://httpbin.org')
self.assertEqual(conn_count(), 3)
# Releasing the sessions does not close the connections
protocol.release_session(session)
protocol.release_session(s2)
protocol.release_session(s3)
self.assertEqual(conn_count(), 3)
# But closing explicitly does
protocol.close()
self.assertEqual(conn_count(), 0)
def test_magic(self, m):
# Just test we don't fail when calling repr() and str(). Insert a dummy cache entry for testing
c = Credentials('leet_user', 'cannaguess')
autodiscover_cache[('example.com', c)] = AutodiscoverProtocol(
config=Configuration(
service_endpoint=
'https://example.com/Autodiscover/Autodiscover.xml',
credentials=c,
auth_type=NTLM,
retry_policy=FailFast(),
))
self.assertEqual(len(autodiscover_cache), 1)
str(autodiscover_cache)
repr(autodiscover_cache)
for protocol in autodiscover_cache._protocols.values():
str(protocol)
repr(protocol)
def test_decrease_poolsize(self):
# Test increasing and decreasing the pool size
max_connections = 3
protocol = Protocol(config=Configuration(
service_endpoint='https://example.com/Foo.asmx',
credentials=Credentials('A', 'B'),
auth_type=NTLM,
version=Version(Build(15, 1)),
retry_policy=FailFast(),
max_connections=max_connections))
self.assertEqual(protocol._session_pool.qsize(), 0)
self.assertEqual(protocol.session_pool_size, 0)
protocol.increase_poolsize()
protocol.increase_poolsize()
protocol.increase_poolsize()
with self.assertRaises(SessionPoolMaxSizeReached):
protocol.increase_poolsize()
self.assertEqual(protocol._session_pool.qsize(), max_connections)
self.assertEqual(protocol.session_pool_size, max_connections)
protocol.decrease_poolsize()
protocol.decrease_poolsize()
with self.assertRaises(SessionPoolMinSizeReached):
protocol.decrease_poolsize()
self.assertEqual(protocol._session_pool.qsize(), 1)
def test_post_ratelimited(self):
url = 'https://example.com'
protocol = self.account.protocol
retry_policy = protocol.config.retry_policy
RETRY_WAIT = exchangelib.util.RETRY_WAIT
MAX_REDIRECTS = exchangelib.util.MAX_REDIRECTS
session = protocol.get_session()
try:
# Make sure we fail fast in error cases
protocol.config.retry_policy = FailFast()
# Test the straight, HTTP 200 path
session.post = mock_post(url, 200, {}, 'foo')
r, session = post_ratelimited(protocol=protocol, session=session, url='http://', headers=None, data='')
self.assertEqual(r.content, b'foo')
# Test exceptions raises by the POST request
for err_cls in CONNECTION_ERRORS:
session.post = mock_session_exception(err_cls)
with self.assertRaises(err_cls):
r, session = post_ratelimited(
protocol=protocol, session=session, url='http://', headers=None, data='')
# Test bad exit codes and headers
session.post = mock_post(url, 401, {})
with self.assertRaises(UnauthorizedError):
r, session = post_ratelimited(protocol=protocol, session=session, url='http://', headers=None, data='')
session.post = mock_post(url, 999, {'connection': 'close'})
with self.assertRaises(TransportError):
r, session = post_ratelimited(protocol=protocol, session=session, url='http://', headers=None, data='')
session.post = mock_post(url, 302,
{'location': '/ews/genericerrorpage.htm?aspxerrorpath=/ews/exchange.asmx'})
with self.assertRaises(TransportError):
r, session = post_ratelimited(protocol=protocol, session=session, url='http://', headers=None, data='')
session.post = mock_post(url, 503, {})
with self.assertRaises(TransportError):
r, session = post_ratelimited(protocol=protocol, session=session, url='http://', headers=None, data='')
# No redirect header
session.post = mock_post(url, 302, {})
with self.assertRaises(TransportError):
r, session = post_ratelimited(protocol=protocol, session=session, url=url, headers=None, data='')
# Redirect header to same location
session.post = mock_post(url, 302, {'location': url})
with self.assertRaises(TransportError):
r, session = post_ratelimited(protocol=protocol, session=session, url=url, headers=None, data='')
# Redirect header to relative location
session.post = mock_post(url, 302, {'location': url + '/foo'})
with self.assertRaises(RedirectError):
r, session = post_ratelimited(protocol=protocol, session=session, url=url, headers=None, data='')
# Redirect header to other location and allow_redirects=False
session.post = mock_post(url, 302, {'location': 'https://contoso.com'})
with self.assertRaises(TransportError):
r, session = post_ratelimited(protocol=protocol, session=session, url=url, headers=None, data='')
# Redirect header to other location and allow_redirects=True
exchangelib.util.MAX_REDIRECTS = 0
session.post = mock_post(url, 302, {'location': 'https://contoso.com'})
with self.assertRaises(TransportError):
r, session = post_ratelimited(protocol=protocol, session=session, url=url, headers=None, data='',
allow_redirects=True)
# CAS error
session.post = mock_post(url, 999, {'X-CasErrorCode': 'AAARGH!'})
with self.assertRaises(CASError):
r, session = post_ratelimited(protocol=protocol, session=session, url=url, headers=None, data='')
# Allow XML data in a non-HTTP 200 response
session.post = mock_post(url, 500, {}, '<?xml version="1.0" ?><foo></foo>')
r, session = post_ratelimited(protocol=protocol, session=session, url=url, headers=None, data='')
self.assertEqual(r.content, b'<?xml version="1.0" ?><foo></foo>')
# Bad status_code and bad text
session.post = mock_post(url, 999, {})
with self.assertRaises(TransportError):
r, session = post_ratelimited(protocol=protocol, session=session, url=url, headers=None, data='')
# Test rate limit exceeded
exchangelib.util.RETRY_WAIT = 1
protocol.config.retry_policy = FaultTolerance(max_wait=0.5) # Fail after first RETRY_WAIT
session.post = mock_post(url, 503, {'connection': 'close'})
# Mock renew_session to return the same session so the session object's 'post' method is still mocked
protocol.renew_session = lambda s: s
with self.assertRaises(RateLimitError) as rle:
r, session = post_ratelimited(protocol=protocol, session=session, url='http://', headers=None, data='')
self.assertEqual(rle.exception.status_code, 503)
self.assertEqual(rle.exception.url, url)
self.assertTrue(1 <= rle.exception.total_wait < 2) # One RETRY_WAIT plus some overhead
# Test something larger than the default wait, so we retry at least once
protocol.retry_policy.max_wait = 3 # Fail after second RETRY_WAIT
session.post = mock_post(url, 503, {'connection': 'close'})
with self.assertRaises(RateLimitError) as rle:
r, session = post_ratelimited(protocol=protocol, session=session, url='http://', headers=None, data='')
self.assertEqual(rle.exception.status_code, 503)
self.assertEqual(rle.exception.url, url)
# We double the wait for each retry, so this is RETRY_WAIT + 2*RETRY_WAIT plus some overhead
self.assertTrue(3 <= rle.exception.total_wait < 4, rle.exception.total_wait)
finally:
protocol.retire_session(session) # We have patched the session, so discard it
# Restore patched attributes and functions
protocol.config.retry_policy = retry_policy
exchangelib.util.RETRY_WAIT = RETRY_WAIT
exchangelib.util.MAX_REDIRECTS = MAX_REDIRECTS
try:
delattr(protocol, 'renew_session')
except AttributeError:
pass
def test_fail_fast_back_off(self):
# Test that FailFast does not support back-off logic
c = FailFast()
self.assertIsNone(c.back_off_until)
with self.assertRaises(AttributeError):
c.back_off_until = 1
def test_post_ratelimited(self):
url = 'https://example.com'
protocol = self.account.protocol
retry_policy = protocol.config.retry_policy
renew_session = protocol.renew_session
session = protocol.get_session()
try:
# Make sure we fail fast in error cases
protocol.config.retry_policy = FailFast()
# Test the straight, HTTP 200 path
session.post = mock_post(url, 200, {}, 'foo')
r, session = post_ratelimited(protocol=protocol,
session=session,
url='http://',
headers=None,
data='')
self.assertEqual(r.content, b'foo')
# Test exceptions raises by the POST request
for err_cls in CONNECTION_ERRORS:
session.post = mock_session_exception(err_cls)
with self.assertRaises(err_cls):
r, session = post_ratelimited(protocol=protocol,
session=session,
url='http://',
headers=None,
data='')
# Test bad exit codes and headers
session.post = mock_post(url, 401, {})
with self.assertRaises(UnauthorizedError):
r, session = post_ratelimited(protocol=protocol,
session=session,
url='http://',
headers=None,
data='')
session.post = mock_post(url, 999, {'connection': 'close'})
with self.assertRaises(TransportError):
r, session = post_ratelimited(protocol=protocol,
session=session,
url='http://',
headers=None,
data='')
session.post = mock_post(
url, 302, {
'location':
'/ews/genericerrorpage.htm?aspxerrorpath=/ews/exchange.asmx'
})
with self.assertRaises(TransportError):
r, session = post_ratelimited(protocol=protocol,
session=session,
url='http://',
headers=None,
data='')
session.post = mock_post(url, 503, {})
with self.assertRaises(TransportError):
r, session = post_ratelimited(protocol=protocol,
session=session,
url='http://',
headers=None,
data='')
# No redirect header
session.post = mock_post(url, 302, {})
with self.assertRaises(TransportError):
r, session = post_ratelimited(protocol=protocol,
session=session,
url=url,
headers=None,
data='')
# Redirect header to same location
session.post = mock_post(url, 302, {'location': url})
with self.assertRaises(TransportError):
r, session = post_ratelimited(protocol=protocol,
session=session,
url=url,
headers=None,
data='')
# Redirect header to relative location
session.post = mock_post(url, 302, {'location': url + '/foo'})
with self.assertRaises(RedirectError):
r, session = post_ratelimited(protocol=protocol,
session=session,
url=url,
headers=None,
data='')
# Redirect header to other location and allow_redirects=False
session.post = mock_post(url, 302,
{'location': 'https://contoso.com'})
with self.assertRaises(TransportError):
r, session = post_ratelimited(protocol=protocol,
session=session,
url=url,
headers=None,
data='')
# Redirect header to other location and allow_redirects=True
import exchangelib.util
exchangelib.util.MAX_REDIRECTS = 0
session.post = mock_post(url, 302,
{'location': 'https://contoso.com'})
with self.assertRaises(TransportError):
r, session = post_ratelimited(protocol=protocol,
session=session,
url=url,
headers=None,
data='',
allow_redirects=True)
# CAS error
session.post = mock_post(url, 999, {'X-CasErrorCode': 'AAARGH!'})
with self.assertRaises(CASError):
r, session = post_ratelimited(protocol=protocol,
session=session,
url=url,
headers=None,
data='')
# Allow XML data in a non-HTTP 200 response
session.post = mock_post(url, 500, {},
'<?xml version="1.0" ?><foo></foo>')
r, session = post_ratelimited(protocol=protocol,
session=session,
url=url,
headers=None,
data='')
self.assertEqual(r.content, b'<?xml version="1.0" ?><foo></foo>')
# Bad status_code and bad text
session.post = mock_post(url, 999, {})
with self.assertRaises(TransportError):
r, session = post_ratelimited(protocol=protocol,
session=session,
url=url,
headers=None,
data='')
# Rate limit exceeded
protocol.config.retry_policy = FaultTolerance(max_wait=1)
session.post = mock_post(url, 503, {'connection': 'close'})
protocol.renew_session = lambda s: s # Return the same session so it's still mocked
with self.assertRaises(RateLimitError) as rle:
r, session = post_ratelimited(protocol=protocol,
session=session,
url='http://',
headers=None,
data='')
self.assertEqual(
str(rle.exception),
'Max timeout reached (gave up after 10 seconds. URL https://example.com returned status code 503)'
)
self.assertEqual(rle.exception.url, url)
self.assertEqual(rle.exception.status_code, 503)
# Test something larger than the default wait, so we retry at least once
protocol.retry_policy.max_wait = 15
session.post = mock_post(url, 503, {'connection': 'close'})
with self.assertRaises(RateLimitError) as rle:
r, session = post_ratelimited(protocol=protocol,
session=session,
url='http://',
headers=None,
data='')
self.assertEqual(
str(rle.exception),
'Max timeout reached (gave up after 20 seconds. URL https://example.com returned status code 503)'
)
self.assertEqual(rle.exception.url, url)
self.assertEqual(rle.exception.status_code, 503)
finally:
protocol.retire_session(
session) # We have patched the session, so discard it
# Restore patched attributes and functions
protocol.config.retry_policy = retry_policy
protocol.renew_session = renew_session