def test_enable_disable_httpretty():
"#417 urllib3.contrib.pyopenssl enable -> disable extract"
expect(urllib3.util.IS_PYOPENSSL).to.be.false
httpretty.enable()
httpretty.disable()
extract_from_urllib3()
expect(urllib3.util.IS_PYOPENSSL).to.be.false
def enable(namespace=None, truesocket_recording_dir=None):
Mocket._namespace = namespace
Mocket._truesocket_recording_dir = truesocket_recording_dir
if truesocket_recording_dir:
# JSON dumps will be saved here
assert os.path.isdir(truesocket_recording_dir)
socket.socket = socket.__dict__["socket"] = MocketSocket
socket._socketobject = socket.__dict__["_socketobject"] = MocketSocket
socket.SocketType = socket.__dict__["SocketType"] = MocketSocket
socket.create_connection = socket.__dict__[
"create_connection"
] = create_connection
socket.gethostname = socket.__dict__["gethostname"] = lambda: "localhost"
socket.gethostbyname = socket.__dict__[
"gethostbyname"
] = lambda host: "127.0.0.1"
socket.getaddrinfo = socket.__dict__[
"getaddrinfo"
] = lambda host, port, family=None, socktype=None, proto=None, flags=None: [
(2, 1, 6, "", (host, port))
]
ssl.wrap_socket = ssl.__dict__["wrap_socket"] = FakeSSLContext.wrap_socket
ssl.SSLSocket = ssl.__dict__["SSLSocket"] = MocketSocket
ssl.SSLContext = ssl.__dict__["SSLContext"] = FakeSSLContext
socket.inet_pton = socket.__dict__["inet_pton"] = lambda family, ip: byte_type(
"\x7f\x00\x00\x01", "utf-8"
)
if pyopenssl_override:
# Take out the pyopenssl version - use the default implementation
extract_from_urllib3()
def enable(namespace=None, truesocket_recording_dir=None):
Mocket._namespace = namespace
Mocket._truesocket_recording_dir = truesocket_recording_dir
if truesocket_recording_dir:
# JSON dumps will be saved here
assert os.path.isdir(truesocket_recording_dir)
socket.socket = socket.__dict__['socket'] = MocketSocket
socket._socketobject = socket.__dict__['_socketobject'] = MocketSocket
socket.SocketType = socket.__dict__['SocketType'] = MocketSocket
socket.create_connection = socket.__dict__[
'create_connection'] = create_connection
socket.gethostname = socket.__dict__[
'gethostname'] = lambda: 'localhost'
socket.gethostbyname = socket.__dict__[
'gethostbyname'] = lambda host: '127.0.0.1'
socket.getaddrinfo = socket.__dict__['getaddrinfo'] = \
lambda host, port, family=None, socktype=None, proto=None, flags=None: [(2, 1, 6, '', (host, port))]
ssl.wrap_socket = ssl.__dict__[
'wrap_socket'] = FakeSSLContext.wrap_socket
ssl.SSLSocket = ssl.__dict__['SSLSocket'] = MocketSocket
ssl.SSLContext = ssl.__dict__['SSLContext'] = FakeSSLContext
socket.inet_pton = socket.__dict__[
'inet_pton'] = lambda family, ip: byte_type(
'\x7f\x00\x00\x01', 'utf-8')
if pyopenssl_override:
# Take out the pyopenssl version - use the default implementation
extract_from_urllib3()
def teardown_module():
try:
from urllib3.contrib.pyopenssl import extract_from_urllib3
extract_from_urllib3()
except ImportError:
pass
def wrapper(*args: Any, **kwargs: Any) -> _RT:
if not pyopenssl:
pytest.skip("pyopenssl not available, skipping test.")
return test(*args, **kwargs)
pyopenssl.inject_into_urllib3()
result = test(*args, **kwargs)
pyopenssl.extract_from_urllib3()
return result
def test_inject_validate_fail_cryptography(self):
"""
Injection should not be supported if cryptography is too old.
"""
try:
with patch("cryptography.x509.extensions.Extensions") as mock:
del mock.get_extension_for_class
self.assertRaises(ImportError, inject_into_urllib3)
finally:
# `inject_into_urllib3` is not supposed to succeed.
# If it does, this test should fail, but we need to
# clean up so that subsequent tests are unaffected.
extract_from_urllib3()
def test_inject_validate_fail_cryptography(self):
"""
Injection should not be supported if cryptography is too old.
"""
try:
with patch("cryptography.x509.extensions.Extensions") as mock:
del mock.get_extension_for_class
self.assertRaises(ImportError, inject_into_urllib3)
finally:
# `inject_into_urllib3` is not supposed to succeed.
# If it does, this test should fail, but we need to
# clean up so that subsequent tests are unaffected.
extract_from_urllib3()
def test_inject_validate_fail_pyopenssl(self):
"""
Injection should not be supported if pyOpenSSL is too old.
"""
try:
return_val = Mock()
del return_val._x509
with patch("OpenSSL.crypto.X509", return_value=return_val):
self.assertRaises(ImportError, inject_into_urllib3)
finally:
# `inject_into_urllib3` is not supposed to succeed.
# If it does, this test should fail, but we need to
# clean up so that subsequent tests are unaffected.
extract_from_urllib3()
def test_inject_validate_fail_pyopenssl(self):
"""
Injection should not be supported if pyOpenSSL is too old.
"""
try:
return_val = Mock()
del return_val._x509
with patch("OpenSSL.crypto.X509", return_value=return_val):
self.assertRaises(ImportError, inject_into_urllib3)
finally:
# `inject_into_urllib3` is not supposed to succeed.
# If it does, this test should fail, but we need to
# clean up so that subsequent tests are unaffected.
extract_from_urllib3()
def request(self, method, endpoint, *args, data=None, content_type='application/json', headers=None,
endpoint_is_url=False, target_is_insecure=False, **kwargs):
"""
Perform a request. Kind of a pass-through to requests.request, but with some goodies. `**kwargs` and `*args`
are passed to requests.request.
:param method:
:param endpoint:
:param args:
:param data:
:param content_type:
:param headers:
:param endpoint_is_url:
:param target_is_insecure: will use the insecure adapter with the insecure ciphers
:param kwargs:
:return:
"""
method = method.lower()
url = self.url(endpoint, endpoint_is_url)
action_f = getattr(requests, method)
if headers:
if 'Content-Type' not in headers:
headers['Content-Type'] = content_type
else:
headers = {
'Content-Type': content_type
}
if data:
if content_type == 'application/json' and not isinstance(data, str):
data = json.dumps(data)
if target_is_insecure:
action_f = self.insecure_target(url, method)
pyopenssl.extract_from_urllib3()
result = action_f(url, *args, headers=self.headers(headers), verify=False, data=data,
auth=self.auth, **kwargs)
else:
if target_is_insecure:
action_f = self.insecure_target(url, method)
pyopenssl.extract_from_urllib3()
result = action_f(url, *args, headers=self.headers(headers), verify=False,
auth=self.auth, **kwargs)
if result.status_code >= 400:
raise requests.HTTPError('The request generated an error: {0}: {1}'.format(result.status_code, result.text),
response=result)
return result.json()
def test_inject_validate_fail(self):
"""
Injection should not be supported if we are missing required dependencies.
"""
successfully_injected = False
try:
with patch("cryptography.x509.extensions.Extensions") as mock:
# The following two lines are what this test intends to test.
# The remainder of this function is setup and clean-up logic.
del mock.get_extension_for_class
self.assertRaises(ImportError, inject_into_urllib3)
successfully_injected = True
finally:
if successfully_injected:
# `inject_into_urllib3` is not supposed to succeed.
# If it does, this test should fail, but we should
# clean up so that subsequent tests are unaffected.
extract_from_urllib3()
def teardown_module():
extract_from_urllib3()
def test_enable_disable_httpretty():
expect(urllib3.util.IS_PYOPENSSL).to.be.false
httpretty.enable()
httpretty.disable()
extract_from_urllib3()
expect(urllib3.util.IS_PYOPENSSL).to.be.false
def teardown_module():
extract_from_urllib3()
def pyopenssl_inject_into_urllib3():
inject_into_urllib3()
try:
yield
finally:
extract_from_urllib3()
import arvados.retry as retry
import arvados.util
try:
# Workaround for urllib3 bug.
# The 'requests' library enables urllib3's SNI support by default, which uses pyopenssl.
# However, urllib3 prior to version 1.10 has a major bug in this feature
# (OpenSSL WantWriteError, https://github.com/shazow/urllib3/issues/412)
# Unfortunately Debian 8 is stabilizing on urllib3 1.9.1 which means the
# following workaround is necessary to be able to use
# the arvados python sdk with the distribution-provided packages.
import urllib3
from pkg_resources import parse_version
if parse_version(urllib3.__version__) < parse_version('1.10'):
from urllib3.contrib import pyopenssl
pyopenssl.extract_from_urllib3()
except ImportError:
pass
_logger = logging.getLogger('arvados.keep')
global_client_object = None
class KeepLocator(object):
EPOCH_DATETIME = datetime.datetime.utcfromtimestamp(0)
HINT_RE = re.compile(r'^[A-Z][A-Za-z0-9@_-]+$')
def __init__(self, locator_str):
self.hints = []
self._perm_sig = None
self._perm_expiry = None
pieces = iter(locator_str.split('+'))
S3_READ_TIMEOUT = 30
S3_TIMEOUT_RETRIES = 3
CONTENT_RANGE_RE = re.compile(r'^bytes (\d+)-(\d+)/(\d+)$')
CHUNK_SIZE = 4096
ZLIB_LEVEL = 2
# pyOpenSSL and S3 don't play well together. pyOpenSSL is completely optional, but gets enabled by requests.
# So... We disable it. That's what boto does.
# https://github.com/boto/botocore/issues/760
# https://github.com/boto/botocore/pull/803
try:
from urllib3.contrib import pyopenssl
pyopenssl.extract_from_urllib3()
except ImportError:
pass
def create_s3_session():
"""
Creates a session with automatic retries on 5xx errors.
"""
sess = requests.Session()
retries = Retry(total=3,
backoff_factor=.5,
status_forcelist=[500, 502, 503, 504])
sess.mount('https://', HTTPAdapter(max_retries=retries))
return sess
def teardown_module():
try:
from urllib3.contrib.pyopenssl import extract_from_urllib3
extract_from_urllib3()
except ImportError:
pass
