def test_enable_disable_httpretty(): "#417 urllib3.contrib.pyopenssl enable -> disable extract" expect(urllib3.util.IS_PYOPENSSL).to.be.false httpretty.enable() httpretty.disable() extract_from_urllib3() expect(urllib3.util.IS_PYOPENSSL).to.be.false
def enable(namespace=None, truesocket_recording_dir=None): Mocket._namespace = namespace Mocket._truesocket_recording_dir = truesocket_recording_dir if truesocket_recording_dir: # JSON dumps will be saved here assert os.path.isdir(truesocket_recording_dir) socket.socket = socket.__dict__["socket"] = MocketSocket socket._socketobject = socket.__dict__["_socketobject"] = MocketSocket socket.SocketType = socket.__dict__["SocketType"] = MocketSocket socket.create_connection = socket.__dict__[ "create_connection" ] = create_connection socket.gethostname = socket.__dict__["gethostname"] = lambda: "localhost" socket.gethostbyname = socket.__dict__[ "gethostbyname" ] = lambda host: "127.0.0.1" socket.getaddrinfo = socket.__dict__[ "getaddrinfo" ] = lambda host, port, family=None, socktype=None, proto=None, flags=None: [ (2, 1, 6, "", (host, port)) ] ssl.wrap_socket = ssl.__dict__["wrap_socket"] = FakeSSLContext.wrap_socket ssl.SSLSocket = ssl.__dict__["SSLSocket"] = MocketSocket ssl.SSLContext = ssl.__dict__["SSLContext"] = FakeSSLContext socket.inet_pton = socket.__dict__["inet_pton"] = lambda family, ip: byte_type( "\x7f\x00\x00\x01", "utf-8" ) if pyopenssl_override: # Take out the pyopenssl version - use the default implementation extract_from_urllib3()
def enable(namespace=None, truesocket_recording_dir=None): Mocket._namespace = namespace Mocket._truesocket_recording_dir = truesocket_recording_dir if truesocket_recording_dir: # JSON dumps will be saved here assert os.path.isdir(truesocket_recording_dir) socket.socket = socket.__dict__['socket'] = MocketSocket socket._socketobject = socket.__dict__['_socketobject'] = MocketSocket socket.SocketType = socket.__dict__['SocketType'] = MocketSocket socket.create_connection = socket.__dict__[ 'create_connection'] = create_connection socket.gethostname = socket.__dict__[ 'gethostname'] = lambda: 'localhost' socket.gethostbyname = socket.__dict__[ 'gethostbyname'] = lambda host: '127.0.0.1' socket.getaddrinfo = socket.__dict__['getaddrinfo'] = \ lambda host, port, family=None, socktype=None, proto=None, flags=None: [(2, 1, 6, '', (host, port))] ssl.wrap_socket = ssl.__dict__[ 'wrap_socket'] = FakeSSLContext.wrap_socket ssl.SSLSocket = ssl.__dict__['SSLSocket'] = MocketSocket ssl.SSLContext = ssl.__dict__['SSLContext'] = FakeSSLContext socket.inet_pton = socket.__dict__[ 'inet_pton'] = lambda family, ip: byte_type( '\x7f\x00\x00\x01', 'utf-8') if pyopenssl_override: # Take out the pyopenssl version - use the default implementation extract_from_urllib3()
def teardown_module(): try: from urllib3.contrib.pyopenssl import extract_from_urllib3 extract_from_urllib3() except ImportError: pass
def wrapper(*args: Any, **kwargs: Any) -> _RT: if not pyopenssl: pytest.skip("pyopenssl not available, skipping test.") return test(*args, **kwargs) pyopenssl.inject_into_urllib3() result = test(*args, **kwargs) pyopenssl.extract_from_urllib3() return result
def test_inject_validate_fail_cryptography(self): """ Injection should not be supported if cryptography is too old. """ try: with patch("cryptography.x509.extensions.Extensions") as mock: del mock.get_extension_for_class self.assertRaises(ImportError, inject_into_urllib3) finally: # `inject_into_urllib3` is not supposed to succeed. # If it does, this test should fail, but we need to # clean up so that subsequent tests are unaffected. extract_from_urllib3()
def test_inject_validate_fail_pyopenssl(self): """ Injection should not be supported if pyOpenSSL is too old. """ try: return_val = Mock() del return_val._x509 with patch("OpenSSL.crypto.X509", return_value=return_val): self.assertRaises(ImportError, inject_into_urllib3) finally: # `inject_into_urllib3` is not supposed to succeed. # If it does, this test should fail, but we need to # clean up so that subsequent tests are unaffected. extract_from_urllib3()
def request(self, method, endpoint, *args, data=None, content_type='application/json', headers=None, endpoint_is_url=False, target_is_insecure=False, **kwargs): """ Perform a request. Kind of a pass-through to requests.request, but with some goodies. `**kwargs` and `*args` are passed to requests.request. :param method: :param endpoint: :param args: :param data: :param content_type: :param headers: :param endpoint_is_url: :param target_is_insecure: will use the insecure adapter with the insecure ciphers :param kwargs: :return: """ method = method.lower() url = self.url(endpoint, endpoint_is_url) action_f = getattr(requests, method) if headers: if 'Content-Type' not in headers: headers['Content-Type'] = content_type else: headers = { 'Content-Type': content_type } if data: if content_type == 'application/json' and not isinstance(data, str): data = json.dumps(data) if target_is_insecure: action_f = self.insecure_target(url, method) pyopenssl.extract_from_urllib3() result = action_f(url, *args, headers=self.headers(headers), verify=False, data=data, auth=self.auth, **kwargs) else: if target_is_insecure: action_f = self.insecure_target(url, method) pyopenssl.extract_from_urllib3() result = action_f(url, *args, headers=self.headers(headers), verify=False, auth=self.auth, **kwargs) if result.status_code >= 400: raise requests.HTTPError('The request generated an error: {0}: {1}'.format(result.status_code, result.text), response=result) return result.json()
def test_inject_validate_fail(self): """ Injection should not be supported if we are missing required dependencies. """ successfully_injected = False try: with patch("cryptography.x509.extensions.Extensions") as mock: # The following two lines are what this test intends to test. # The remainder of this function is setup and clean-up logic. del mock.get_extension_for_class self.assertRaises(ImportError, inject_into_urllib3) successfully_injected = True finally: if successfully_injected: # `inject_into_urllib3` is not supposed to succeed. # If it does, this test should fail, but we should # clean up so that subsequent tests are unaffected. extract_from_urllib3()
def teardown_module(): extract_from_urllib3()
def test_enable_disable_httpretty(): expect(urllib3.util.IS_PYOPENSSL).to.be.false httpretty.enable() httpretty.disable() extract_from_urllib3() expect(urllib3.util.IS_PYOPENSSL).to.be.false
def pyopenssl_inject_into_urllib3(): inject_into_urllib3() try: yield finally: extract_from_urllib3()
import arvados.retry as retry import arvados.util try: # Workaround for urllib3 bug. # The 'requests' library enables urllib3's SNI support by default, which uses pyopenssl. # However, urllib3 prior to version 1.10 has a major bug in this feature # (OpenSSL WantWriteError, https://github.com/shazow/urllib3/issues/412) # Unfortunately Debian 8 is stabilizing on urllib3 1.9.1 which means the # following workaround is necessary to be able to use # the arvados python sdk with the distribution-provided packages. import urllib3 from pkg_resources import parse_version if parse_version(urllib3.__version__) < parse_version('1.10'): from urllib3.contrib import pyopenssl pyopenssl.extract_from_urllib3() except ImportError: pass _logger = logging.getLogger('arvados.keep') global_client_object = None class KeepLocator(object): EPOCH_DATETIME = datetime.datetime.utcfromtimestamp(0) HINT_RE = re.compile(r'^[A-Z][A-Za-z0-9@_-]+$') def __init__(self, locator_str): self.hints = [] self._perm_sig = None self._perm_expiry = None pieces = iter(locator_str.split('+'))
S3_READ_TIMEOUT = 30 S3_TIMEOUT_RETRIES = 3 CONTENT_RANGE_RE = re.compile(r'^bytes (\d+)-(\d+)/(\d+)$') CHUNK_SIZE = 4096 ZLIB_LEVEL = 2 # pyOpenSSL and S3 don't play well together. pyOpenSSL is completely optional, but gets enabled by requests. # So... We disable it. That's what boto does. # https://github.com/boto/botocore/issues/760 # https://github.com/boto/botocore/pull/803 try: from urllib3.contrib import pyopenssl pyopenssl.extract_from_urllib3() except ImportError: pass def create_s3_session(): """ Creates a session with automatic retries on 5xx errors. """ sess = requests.Session() retries = Retry(total=3, backoff_factor=.5, status_forcelist=[500, 502, 503, 504]) sess.mount('https://', HTTPAdapter(max_retries=retries)) return sess