def download(url, filename):
pyopenssl.inject_into_urllib3()
http = ul.PoolManager(cert_reqs = 'CERT_REQUIRED', \
ca_certs = certifi.where())
res = http.request('GET', url)
with open(filename, 'wb') as f:
f.write(res.data)
def setup_module():
try:
from urllib3.contrib.pyopenssl import inject_into_urllib3
inject_into_urllib3()
except ImportError as e:
pytest.skip(f"Could not import PyOpenSSL: {e!r}")
def download(url, filename):
pyopenssl.inject_into_urllib3()
http = ul.PoolManager(cert_reqs = 'CERT_REQUIRED', \
ca_certs = certifi.where())
res = http.request('GET', url)
with open(filename, 'wb') as f:
f.write(res.data)
def disable():
socket.socket = socket.__dict__["socket"] = true_socket
socket._socketobject = socket.__dict__["_socketobject"] = true_socket
socket.SocketType = socket.__dict__["SocketType"] = true_socket
socket.create_connection = socket.__dict__[
"create_connection"] = true_create_connection
socket.gethostname = socket.__dict__["gethostname"] = true_gethostname
socket.gethostbyname = socket.__dict__[
"gethostbyname"] = true_gethostbyname
socket.getaddrinfo = socket.__dict__["getaddrinfo"] = true_getaddrinfo
ssl.wrap_socket = ssl.__dict__["wrap_socket"] = true_ssl_wrap_socket
ssl.SSLContext = ssl.__dict__["SSLContext"] = true_ssl_context
socket.inet_pton = socket.__dict__["inet_pton"] = true_inet_pton
urllib3.util.ssl_.wrap_socket = urllib3.util.ssl_.__dict__[
"wrap_socket"] = true_urllib3_wrap_socket
urllib3.util.ssl_.ssl_wrap_socket = urllib3.util.ssl_.__dict__[
"ssl_wrap_socket"] = true_urllib3_ssl_wrap_socket
urllib3.connection.ssl_wrap_socket = urllib3.connection.__dict__[
"ssl_wrap_socket"] = true_urllib3_ssl_wrap_socket
urllib3.connection.match_hostname = urllib3.connection.__dict__[
"match_hostname"] = true_urllib3_match_hostname
Mocket.reset()
if pyopenssl_override: # pragma: no cover
# Put the pyopenssl version back in place
inject_into_urllib3()
def get_html(url):
#certificate for python2.7
pyopenssl.inject_into_urllib3()
mgr = urllib.PoolManager(cert_reqs = 'CERT_REQUIRED', \
ca_certs = certifi.where())
res = mgr.request('GET', url)
return res.data
def get_html(url):
#certificate for python2.7
pyopenssl.inject_into_urllib3()
mgr = urllib.PoolManager(cert_reqs = 'CERT_REQUIRED', \
ca_certs = certifi.where())
res = mgr.request('GET', url)
return res.data
def wrapper(*args: Any, **kwargs: Any) -> _RT:
if not pyopenssl:
pytest.skip("pyopenssl not available, skipping test.")
return test(*args, **kwargs)
pyopenssl.inject_into_urllib3()
result = test(*args, **kwargs)
pyopenssl.extract_from_urllib3()
return result
def test_inject_validate_fail_cryptography(self) -> None:
"""
Injection should not be supported if cryptography is too old.
"""
try:
with patch("cryptography.x509.extensions.Extensions") as mock:
del mock.get_extension_for_class
with pytest.raises(ImportError):
inject_into_urllib3()
finally:
# `inject_into_urllib3` is not supposed to succeed.
# If it does, this test should fail, but we need to
# clean up so that subsequent tests are unaffected.
extract_from_urllib3()
def test_inject_validate_fail_pyopenssl(self) -> None:
"""
Injection should not be supported if pyOpenSSL is too old.
"""
try:
return_val = Mock()
del return_val._x509
with patch("OpenSSL.crypto.X509", return_value=return_val):
with pytest.raises(ImportError):
inject_into_urllib3()
finally:
# `inject_into_urllib3` is not supposed to succeed.
# If it does, this test should fail, but we need to
# clean up so that subsequent tests are unaffected.
extract_from_urllib3()
def disable():
socket.socket = socket.__dict__['socket'] = true_socket
socket._socketobject = socket.__dict__['_socketobject'] = true_socket
socket.SocketType = socket.__dict__['SocketType'] = true_socket
socket.create_connection = socket.__dict__['create_connection'] = true_create_connection
socket.gethostname = socket.__dict__['gethostname'] = true_gethostname
socket.gethostbyname = socket.__dict__['gethostbyname'] = true_gethostbyname
socket.getaddrinfo = socket.__dict__['getaddrinfo'] = true_getaddrinfo
ssl.wrap_socket = ssl.__dict__['wrap_socket'] = true_ssl_wrap_socket
ssl.SSLSocket = ssl.__dict__['SSLSocket'] = true_ssl_socket
ssl.SSLContext = ssl.__dict__['SSLContext'] = true_ssl_context
socket.inet_pton = socket.__dict__['inet_pton'] = true_inet_pton
Mocket.reset()
if pyopenssl_override:
# Put the pyopenssl version back in place
inject_into_urllib3()
def _get_content(self, url):
headers = {}
api_key = self.config.get('api_key')
if api_key:
headers['Authorization'] = api_key
pyopenssl.inject_into_urllib3()
try:
http_request = requests.get(url, headers=headers)
except RequestException as e:
raise ContentFetchError('HTTP error: %s' % e.code)
except Exception as e:
raise ContentFetchError('HTTP general exception: %s' % e)
return http_request.text
def disable():
socket.socket = socket.__dict__["socket"] = true_socket
socket._socketobject = socket.__dict__["_socketobject"] = true_socket
socket.SocketType = socket.__dict__["SocketType"] = true_socket
socket.create_connection = socket.__dict__[
"create_connection"
] = true_create_connection
socket.gethostname = socket.__dict__["gethostname"] = true_gethostname
socket.gethostbyname = socket.__dict__["gethostbyname"] = true_gethostbyname
socket.getaddrinfo = socket.__dict__["getaddrinfo"] = true_getaddrinfo
ssl.wrap_socket = ssl.__dict__["wrap_socket"] = true_ssl_wrap_socket
ssl.SSLSocket = ssl.__dict__["SSLSocket"] = true_ssl_socket
ssl.SSLContext = ssl.__dict__["SSLContext"] = true_ssl_context
socket.inet_pton = socket.__dict__["inet_pton"] = true_inet_pton
Mocket.reset()
if pyopenssl_override:
# Put the pyopenssl version back in place
inject_into_urllib3()
def _get_content(self, url):
headers = {}
api_key = self.config.get('api_key')
if api_key:
headers['Authorization'] = api_key
pyopenssl.inject_into_urllib3()
try:
http_request = requests.get(url, headers=headers)
except HTTPError as e:
raise ContentFetchError('HTTP error: %s %s' % (e.response.status_code, e.request.url))
except RequestException as e:
raise ContentFetchError('Request error: %s' % e)
except Exception as e:
raise ContentFetchError('HTTP general exception: %s' % e)
return http_request.text
def gather_stage(self, harvest_job, collection_package_id=None):
log = logging.getLogger(__name__ + '.WAF.gather')
log.debug('WafHarvester gather_stage for job: %r', harvest_job)
self.harvest_job = harvest_job
# Get source URL
source_url = harvest_job.source.url
self._set_source_config(harvest_job.source.config)
# Get contents
pyopenssl.inject_into_urllib3()
try:
response = requests.get(source_url, timeout=60)
response.raise_for_status()
except requests.exceptions.RequestException, e:
self._save_gather_error('Unable to get content for URL: %s: %r' % \
(source_url, e),harvest_job)
return None
if cryptography_version < [1, 3, 4]:
warning = 'Old version of cryptography ({0}) may cause slowdown.'.format(cryptography_version)
warnings.warn(warning, RequestsDependencyWarning)
# Check imported dependencies for compatibility.
try:
check_compatibility(urllib3.__version__, chardet.__version__)
except (AssertionError, ValueError):
warnings.warn("urllib3 ({0}) or chardet ({1}) doesn't match a supported "
"version!".format(urllib3.__version__, chardet.__version__),
RequestsDependencyWarning)
# Attempt to enable urllib3's SNI support, if possible
try:
from urllib3.contrib import pyopenssl
pyopenssl.inject_into_urllib3()
# Check cryptography version
from cryptography import __version__ as cryptography_version
_check_cryptography(cryptography_version)
except ImportError:
pass
# urllib3's DependencyWarnings should be silenced.
from urllib3.exceptions import DependencyWarning
warnings.simplefilter('ignore', DependencyWarning)
from .__version__ import __title__, __description__, __url__, __version__
from .__version__ import __build__, __author__, __author_email__, __license__
from .__version__ import __copyright__, __cake__
def setup_module():
inject_into_urllib3()
def __init__(self, *args, **kwargs):
pyopenssl.inject_into_urllib3()
self.session = requests.Session()
def setup_module():
inject_into_urllib3()
def get_html(link):
pyopenssl.inject_into_urllib3()
http = ul.PoolManager(cert_reqs = 'CERT_REQUIRED', \
ca_certs = certifi.where())
r = http.request('GET', link)
return r.data
def pyopenssl_inject_into_urllib3():
inject_into_urllib3()
try:
yield
finally:
extract_from_urllib3()
from functools import partial
from itertools import compress
from ipaddress import ip_address, ip_network
from datetime import datetime
from dateutil.tz import tzlocal
from ansible.module_utils.six.moves import configparser as ConfigParser
try:
from orionsdk import SwisClient
except ImportError:
sys.exit("SolarWinds OrionSDK python client is not installed. See https://github.com/solarwinds/orionsdk-python``")
from urllib3 import disable_warnings, exceptions as urllib3exc
from urllib3.contrib.pyopenssl import inject_into_urllib3
disable_warnings(category=urllib3exc.InsecureRequestWarning)
inject_into_urllib3()
class _SwisClient (SwisClient): # to override proxy settings
def _req(self, method, frag, data=None):
resp = self._session.request(method, self.url + frag,
data=json.dumps(data, default=lambda obj: obj.isoformat() if isinstance(obj, datetime) else None), #default=_json_serial
proxies={"http": None, "https": None}, # quick fix to override proxy environment settings if any
)
if 400 <= resp.status_code < 600: # try to extract reason from response when request returns error
try:
resp.reason = json.loads(resp.text)['Message'];
except:
pass;
resp.raise_for_status()
return resp
# -*- coding: utf-8 -*-
import praw, time, datetime, re, urllib, urllib2, pickle, pyimgur, os, traceback, wikia, string, socket, sys, collections
from urllib3.contrib.pyopenssl import inject_into_urllib3
#from nsfw import getnsfw
from util import success, warn, log, fail, special, bluelog
from bs4 import BeautifulSoup
from HTMLParser import HTMLParser
# This makes urllib3 verify HTTPS requests
inject_into_urllib3()
WIKI_URL = 'wikia.com/wiki'
### Uncomment to debug
#import logging
#logging.basicConfig(level=logging.DEBUG)
### Set root directory to script directory
abspath = os.path.abspath(__file__)
dname = os.path.dirname(abspath)
os.chdir(dname)
def find_link(body):
begin_index = body.find("http://")
for index, char in enumerate(body[begin_index:]):
if char in (" ", ")"):
end_index = index + begin_index
break
else:
# -2 because one is added to it, and that would return an empty string
end_index = -2
def setup_module():
try:
from urllib3.contrib.pyopenssl import inject_into_urllib3
inject_into_urllib3()
except ImportError as e:
pytest.skip('Could not import PyOpenSSL: %r' % e)
def get_html(link):
pyopenssl.inject_into_urllib3()
http = ul.PoolManager(cert_reqs = 'CERT_REQUIRED', \
ca_certs = certifi.where())
r = http.request('GET', link)
return r.data
warnings.warn(warning, RequestsDependencyWarning)
# Check imported dependencies for compatibility.
try:
check_compatibility(urllib3.__version__, chardet.__version__)
except (AssertionError, ValueError):
warnings.warn(
"urllib3 ({}) or chardet ({}) doesn't match a supported "
"version!".format(urllib3.__version__, chardet.__version__),
RequestsDependencyWarning)
# Attempt to enable urllib3's SNI support, if possible
try:
from urllib3.contrib import pyopenssl
pyopenssl.inject_into_urllib3()
# Check cryptography version
from cryptography import __version__ as cryptography_version
_check_cryptography(cryptography_version)
except ImportError:
pass
# urllib3's DependencyWarnings should be silenced.
from urllib3.exceptions import DependencyWarning
warnings.simplefilter('ignore', DependencyWarning)
from .__version__ import __title__, __description__, __url__, __version__
from .__version__ import __build__, __author__, __author_email__, __license__
from .__version__ import __copyright__, __cake__
from Twitter_Utils.EternalProcess import EternalProcess # pragma: no cover
from urllib3.contrib import pyopenssl # pragma: no cover
from Eternal_Utils import SetupLogging # pragma: no cover
pyopenssl.inject_into_urllib3() # pragma: no cover
if __name__ == "__main__": # pragma: no cover
SetupLogging.setup_logging()
process = EternalProcess()
process.start_process()
