def __init__(self, good_domains=None):
"""
:param good_domains: List of domains. The URLs must be in bytes
"""
if not good_domains:
self.good_domains = []
else:
self.good_domains = good_domains
# by default, handle requests like a browser would
self.default_policy = BrowserLikePolicyForHTTPS()
def __init__(
self,
reactor,
proxy_reactor=None,
contextFactory=BrowserLikePolicyForHTTPS(),
connectTimeout=None,
bindAddress=None,
pool=None,
http_proxy=None,
https_proxy=None,
):
_AgentBase.__init__(self, reactor, pool)
if proxy_reactor is None:
self.proxy_reactor = reactor
else:
self.proxy_reactor = proxy_reactor
self._endpoint_kwargs = {}
if connectTimeout is not None:
self._endpoint_kwargs["timeout"] = connectTimeout
if bindAddress is not None:
self._endpoint_kwargs["bindAddress"] = bindAddress
self.http_proxy_endpoint = _http_proxy_endpoint(
http_proxy, self.proxy_reactor, **self._endpoint_kwargs)
self.https_proxy_endpoint = _http_proxy_endpoint(
https_proxy, self.proxy_reactor, **self._endpoint_kwargs)
self._policy_for_https = contextFactory
self._reactor = reactor
def __init__(
self,
reactor,
contextFactory=BrowserLikePolicyForHTTPS(),
connectTimeout=None,
bindAddress=None,
pool=None,
proxy_url_str: Optional[str] = None,
):
_AgentBase.__init__(self, reactor, pool)
self._endpoint_kwargs = {}
if connectTimeout is not None:
self._endpoint_kwargs["timeout"] = connectTimeout
if bindAddress is not None:
self._endpoint_kwargs["bindAddress"] = bindAddress
if proxy_url_str is not None:
parsed_url = decompose_http_proxy_url(proxy_url_str)
self._proxy_auth = parsed_url.credentials
self.proxy_endpoint = HostnameEndpoint(reactor,
parsed_url.hostname,
parsed_url.port,
**self._endpoint_kwargs)
else:
self.proxy_endpoint = None
self._policy_for_https = contextFactory
self._reactor = reactor
def __init__(self,
reactor,
proxy_scheme,
proxy_host,
proxy_port,
proxy_username=None,
proxy_password=None,
contextFactory=BrowserLikePolicyForHTTPS(),
connectTimeout=None,
bindAddress=None,
pool=None):
if not IPolicyForHTTPS.providedBy(contextFactory):
raise NotImplementedError(
'contextFactory must implement IPolicyForHTTPS')
self._policyForHTTPS = contextFactory
self._wrappedAgent = BaseAgent.usingEndpointFactory(reactor,
self,
pool=pool)
self._bindAddress = bindAddress
self._connectTimeout = connectTimeout
self.reactor = reactor
self.proxy_scheme = proxy_scheme
self.proxy_host = proxy_host
self.proxy_port = proxy_port
self.proxy_username = proxy_username
self.proxy_password = proxy_password
def build_http_client_context_factory(self):
config = self.get_config()
return (
InsecureInterceptableContextFactory()
if config.use_insecure_ssl_client_just_for_testing_do_not_use
else BrowserLikePolicyForHTTPS()
)
def _get_http_client():
response = yield treq.get(ca)
pemdata = yield response.text()
cert = Certificate.loadPEM(pemdata)
policy = BrowserLikePolicyForHTTPS(trustRoot=cert)
agent = Agent(reactor, contextFactory=policy)
client = treq.client.HTTPClient(agent)
returnValue(client)
class OneHostnameWorkaroundPolicy(object):
def __init__(self):
self._normalPolicy = BrowserLikePolicyForHTTPS()
def creatorForNetloc(self, hostname, port):
if hostname == b"wrong.host.badssl.com":
hostname = b"badssl.com"
return self._normalPolicy.creatorForNetloc(hostname, port)
def patch():
try:
base.WebVerifyingContextFactory
except AttributeError:
# Don't do anything if that thing has gone away.
pass
else:
# Replace that thing with another thing that makes a better thing!
# See the ticket for details.
base.WebVerifyingContextFactory = lambda host: BrowserLikePolicyForHTTPS()
def __init__(self, reactor, contextFactory=BrowserLikePolicyForHTTPS(),
connectTimeout=None, bindAddress=None, pool=None, proxyEndpoint=None, endpointArgs={}):
if not IPolicyForHTTPS.providedBy(contextFactory):
raise NotImplementedError(
'contextFactory must implement IPolicyForHTTPS')
self.proxyEndpoint = proxyEndpoint
self.endpointArgs = endpointArgs
self._policyForHTTPS = contextFactory
self._wrappedAgent = _Agent.usingEndpointFactory(
reactor, self, pool=pool)
class VerifyHTTPS(object):
def __init__(self):
# by default, handle requests like a browser would
self.default_policy = BrowserLikePolicyForHTTPS()
def creatorForNetloc(self, hostname, port):
# check if the hostname is in the the whitelist, otherwise return the default policy
if not SETTINGS['verify_https']:
return ssl.CertificateOptions(verify=False)
return self.default_policy.creatorForNetloc(hostname, port)
def __init__(self, config):
if not have_twisted:
raise ImportError('Twisted required for TwistedTransport')
TransportBase.__init__(self, config, self.__module__)
if config.get("ignore-ssl-errors"):
policy = NoValidationPolicy()
else:
policy = BrowserLikePolicyForHTTPS()
self.agent = Agent(reactor, policy)
def get_test_https_policy():
"""Get a test IPolicyForHTTPS which trusts the test CA cert
Returns:
IPolicyForHTTPS
"""
ca_file = get_test_ca_cert_file()
with open(ca_file) as stream:
content = stream.read()
cert = Certificate.loadPEM(content)
trust_root = trustRootFromCertificates([cert])
return BrowserLikePolicyForHTTPS(trustRoot=trust_root)
def __init__(
self,
reactor,
proxy_reactor=None,
contextFactory: Optional[IPolicyForHTTPS] = None,
connectTimeout=None,
bindAddress=None,
pool=None,
use_proxy=False,
):
contextFactory = contextFactory or BrowserLikePolicyForHTTPS()
_AgentBase.__init__(self, reactor, pool)
if proxy_reactor is None:
self.proxy_reactor = reactor
else:
self.proxy_reactor = proxy_reactor
self._endpoint_kwargs = {}
if connectTimeout is not None:
self._endpoint_kwargs["timeout"] = connectTimeout
if bindAddress is not None:
self._endpoint_kwargs["bindAddress"] = bindAddress
http_proxy = None
https_proxy = None
no_proxy = None
if use_proxy:
proxies = getproxies_environment()
http_proxy = proxies["http"].encode(
) if "http" in proxies else None
https_proxy = proxies["https"].encode(
) if "https" in proxies else None
no_proxy = proxies["no"] if "no" in proxies else None
# Parse credentials from https proxy connection string if present
self.https_proxy_creds, https_proxy = parse_username_password(
https_proxy)
self.http_proxy_endpoint = http_proxy_endpoint(http_proxy,
self.proxy_reactor,
**self._endpoint_kwargs)
self.https_proxy_endpoint = http_proxy_endpoint(
https_proxy, self.proxy_reactor, **self._endpoint_kwargs)
self.no_proxy = no_proxy
self._policy_for_https = contextFactory
self._reactor = reactor
def __init__(
self,
reactor: IReactorCore,
proxy_reactor: Optional[ISynapseReactor] = None,
contextFactory: Optional[IPolicyForHTTPS] = None,
connectTimeout: Optional[float] = None,
bindAddress: Optional[bytes] = None,
pool: Optional[HTTPConnectionPool] = None,
use_proxy: bool = False,
):
contextFactory = contextFactory or BrowserLikePolicyForHTTPS()
_AgentBase.__init__(self, reactor, pool)
if proxy_reactor is None:
self.proxy_reactor = reactor
else:
self.proxy_reactor = proxy_reactor
self._endpoint_kwargs: Dict[str, Any] = {}
if connectTimeout is not None:
self._endpoint_kwargs["timeout"] = connectTimeout
if bindAddress is not None:
self._endpoint_kwargs["bindAddress"] = bindAddress
http_proxy = None
https_proxy = None
no_proxy = None
if use_proxy:
proxies = getproxies_environment()
http_proxy = proxies["http"].encode(
) if "http" in proxies else None
https_proxy = proxies["https"].encode(
) if "https" in proxies else None
no_proxy = proxies["no"] if "no" in proxies else None
self.http_proxy_endpoint, self.http_proxy_creds = http_proxy_endpoint(
http_proxy, self.proxy_reactor, contextFactory,
**self._endpoint_kwargs)
self.https_proxy_endpoint, self.https_proxy_creds = http_proxy_endpoint(
https_proxy, self.proxy_reactor, contextFactory,
**self._endpoint_kwargs)
self.no_proxy = no_proxy
self._policy_for_https = contextFactory
self._reactor = reactor
def agent_call(url_trustRoot):
url, trustRoot = url_trustRoot
if trustRoot:
customPolicy = BrowserLikePolicyForHTTPS(
PrivateCertificate.loadPEM(trustRoot))
agent = Agent(reactor, customPolicy)
else:
agent = Agent(reactor)
headers = Headers({
'User-Agent': ['Twisted Webbot'],
'Content-Type': ['text/x-greeting']
})
d = agent.request('HEAD', url, headers=headers)
return d
def __init__(self, config):
if sys.version_info[0] == 3 and sys.version_info <= (3, 5, 0):
raise ResourceException(
"Twisted on Python 3 requires Python 3.5 or later.")
if not have_twisted:
raise ImportError("Twisted required for TwistedTransport")
TransportBase.__init__(self, config, self.__module__)
self._timeout = self._config.get("timeout", None)
if config.get("ignore-ssl-errors"):
policy = NoValidationPolicy()
else:
policy = BrowserLikePolicyForHTTPS()
self.agent = Agent(reactor, policy, connectTimeout=self._timeout)
def __init__(
self,
reactor: ReactorBase,
pool: H2ConnectionPool,
context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(
),
connect_timeout: Optional[float] = None,
bind_address: Optional[bytes] = None,
) -> None:
self._reactor = reactor
self._pool = pool
self._context_factory = AcceptableProtocolsContextFactory(
context_factory, acceptable_protocols=[b'h2'])
self.endpoint_factory = _StandardEndpointFactory(
self._reactor, self._context_factory, connect_timeout,
bind_address)
class WhitelistContextFactory(object):
def __init__(self, good_domains=None):
"""
:param good_domains: List of domains. The URLs must be in bytes
"""
if not good_domains:
self.good_domains = []
else:
self.good_domains = good_domains
# by default, handle requests like a browser would
self.default_policy = BrowserLikePolicyForHTTPS()
def creatorForNetloc(self, hostname, port):
# check if the hostname is in the the whitelist,
# otherwise return the default policy
if hostname in self.good_domains:
return CertificateOptions(verify=False)
return self.default_policy.creatorForNetloc(hostname, port)
def __init__(
self,
reactor: ReactorBase,
proxy_uri: URI,
pool: H2ConnectionPool,
context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(
),
connect_timeout: Optional[float] = None,
bind_address: Optional[bytes] = None,
) -> None:
super(ScrapyProxyH2Agent, self).__init__(
reactor=reactor,
pool=pool,
context_factory=context_factory,
connect_timeout=connect_timeout,
bind_address=bind_address,
)
self._proxy_uri = proxy_uri
def __init__(
self,
reactor,
proxy_reactor=None,
contextFactory=BrowserLikePolicyForHTTPS(),
connectTimeout=None,
bindAddress=None,
pool=None,
use_proxy=False,
):
_AgentBase.__init__(self, reactor, pool)
if proxy_reactor is None:
self.proxy_reactor = reactor
else:
self.proxy_reactor = proxy_reactor
self._endpoint_kwargs = {}
if connectTimeout is not None:
self._endpoint_kwargs["timeout"] = connectTimeout
if bindAddress is not None:
self._endpoint_kwargs["bindAddress"] = bindAddress
http_proxy = None
https_proxy = None
no_proxy = None
if use_proxy:
proxies = getproxies_environment()
http_proxy = proxies["http"].encode(
) if "http" in proxies else None
https_proxy = proxies["https"].encode(
) if "https" in proxies else None
no_proxy = proxies["no"] if "no" in proxies else None
self.http_proxy_endpoint = _http_proxy_endpoint(
http_proxy, self.proxy_reactor, **self._endpoint_kwargs)
self.https_proxy_endpoint = _http_proxy_endpoint(
https_proxy, self.proxy_reactor, **self._endpoint_kwargs)
self.no_proxy = no_proxy
self._policy_for_https = contextFactory
self._reactor = reactor
def createVerifyingHTTPClient(reactor,
extra_ca_certs=None,
agent_kwds=None,
**kwds):
"""
extra_ca_certs: Should be a list of PEM formatted certificates that are trust anchors.
"""
agent_kwds = normalizeDict_(agent_kwds)
if extra_ca_certs is not None:
trust_anchors = []
for ca_cert in extra_ca_certs:
with open(ca_cert, "rb") as f:
data = f.read()
cert = crypto.load_certificate(crypto.FILETYPE_PEM, data)
del data
trust_anchors.append(cert)
policy = CustomPolicyForHTTPS(trust_anchors)
else:
policy = BrowserLikePolicyForHTTPS()
agent_kwds['contextFactory'] = policy
return HTTPClient(Agent(reactor, **agent_kwds), **kwds)
class MyEndpointFactory(object):
"""Endpoint factory that will only connect to one specific address."""
log = Logger()
def __init__(self, reactor, address):
"""Initialize."""
self.reactor = reactor
self.address = address
self.context_factory = BrowserLikePolicyForHTTPS()
def endpointForURI(self, uri): # noqa: N802
"""Create an endpoint for URI."""
endpoint = HostnameEndpoint(self.reactor, self.address, uri.port)
if uri.scheme == b'http':
return endpoint
elif uri.scheme == b'https':
connection_creator = self.context_factory.creatorForNetloc(
uri.host, uri.port)
return wrapClientTLS(connection_creator, endpoint)
def getPolicyForHTTPS(trustRoot=None):
if SKIP_SSL_CHECK:
log.info("---------------------------------------")
log.info("SKIPPING SSL CERT VERIFICATION!!!")
log.info("I assume you know WHAT YOU ARE DOING...")
log.info("---------------------------------------")
class WebClientContextFactory(ClientContextFactory):
"""
A web context factory which ignores the hostname and port and does
no certificate verification.
"""
def getContext(self, hostname, port):
return ClientContextFactory.getContext(self)
contextFactory = WebClientContextFactory()
return contextFactory
if isinstance(trustRoot, str):
trustRoot = Certificate.loadPEM(FilePath(trustRoot).getContent())
return BrowserLikePolicyForHTTPS(trustRoot)
def __init__(self, reactor, address):
"""Initialize."""
self.reactor = reactor
self.address = address
self.context_factory = BrowserLikePolicyForHTTPS()
def __init__(self):
self._normalPolicy = BrowserLikePolicyForHTTPS()
def cookieAgentFactory(verify_path, connectTimeout=30):
customPolicy = BrowserLikePolicyForHTTPS(
Certificate.loadPEM(FilePath(verify_path).getContent()))
agent = Agent(reactor, customPolicy, connectTimeout=connectTimeout)
cookiejar = cookielib.CookieJar()
return CookieAgent(agent, cookiejar)
def __init__(self, config):
# NOTE: ServerFactory.__init__ does not exist?
# Read configuration options.
if config['privacyidea']['verify']:
if config['privacyidea']['certificate']:
certificate_path = config['privacyidea']['certificate']
certificate = Certificate.loadPEM(FilePath(certificate_path).getContent())
log.info('privacyIDEA HTTPS certificate will be checked against {certificate!r} from {path!r}',
certificate=certificate, path=certificate_path)
else:
certificate = None
log.info('privacyIDEA HTTPS certificate will be checked against system certificate store')
https_policy = BrowserLikePolicyForHTTPS(certificate)
else:
log.warn('privacyIDEA HTTPS certificate will NOT be checked!')
https_policy = DisabledVerificationPolicyForHTTPS()
self.agent = Agent(reactor, https_policy)
if config['ldap-backend']['use-tls']:
# TODO: This seems to get lost if we use log.info
log.warn('The use-tls config option is deprecated and will be ignored.')
self.proxied_endpoint_string = config['ldap-backend']['endpoint']
self.privacyidea_instance = config['privacyidea']['instance']
# Construct the validate url from the instance location
if self.privacyidea_instance[-1] != '/':
self.privacyidea_instance += '/'
self.validate_url = VALIDATE_URL_TEMPLATE.format(self.privacyidea_instance)
self.service_account_dn = config['service-account']['dn']
self.service_account_password = config['service-account']['password']
# We have to make a small workaround for configobj here: An empty config value
# is interpreted as a list with one element, the empty string.
self.passthrough_binds = config['ldap-proxy']['passthrough-binds']
if len(self.passthrough_binds) == 1 and self.passthrough_binds[0] == '':
self.passthrough_binds = []
log.info('Passthrough DNs: {binds!r}', binds=self.passthrough_binds)
self.forward_anonymous_binds = config['ldap-proxy']['forward-anonymous-binds']
self.allow_search = config['ldap-proxy']['allow-search']
self.bind_service_account = config['ldap-proxy']['bind-service-account']
self.allow_connection_reuse = config['ldap-proxy']['allow-connection-reuse']
self.ignore_search_result_references = config['ldap-proxy']['ignore-search-result-references']
user_mapping_strategy = USER_MAPPING_STRATEGIES[config['user-mapping']['strategy']]
log.info('Using user mapping strategy: {strategy!r}', strategy=user_mapping_strategy)
self.user_mapper = user_mapping_strategy(self, config['user-mapping'])
realm_mapping_strategy = REALM_MAPPING_STRATEGIES[config['realm-mapping']['strategy']]
log.info('Using realm mapping strategy: {strategy!r}', strategy=realm_mapping_strategy)
self.realm_mapper = realm_mapping_strategy(self, config['realm-mapping'])
enable_bind_cache = config['bind-cache']['enabled']
if enable_bind_cache:
self.bind_cache = BindCache(config['bind-cache']['timeout'])
else:
self.bind_cache = None
enable_app_cache = config['app-cache']['enabled']
if enable_app_cache:
self.app_cache = AppCache(config['app-cache']['timeout'], config['app-cache']['case-insensitive'])
else:
self.app_cache = None
self.app_cache_attribute = config['app-cache']['attribute']
self.app_cache_value_prefix = config['app-cache']['value-prefix']
if config['ldap-backend']['test-connection']:
self.test_connection()
def main(reactor):
agent = Agent(reactor, contextFactory=BrowserLikePolicyForHTTPS())
for url in urls:
d = agent.request(method=METHOD, uri=url, headers=Headers(HEADERS))
d.addCallbacks(callback_request, error_handler)
return d
def __init__(self):
self.default_policy = BrowserLikePolicyForHTTPS()
monitoring_data["{}".format(results.get("name"))] = {
"unit": param.get("unit"),
"param_name": param.get("displayName"),
"last_value": param.get("lastValue"),
"last_updated": param.get("lastUpdated"),
"parameter_id": param.get("parameterId")
}
writeJson(fp, monitoring_data)
def readJson(filename='modeling_data.json'):
with open(filename, 'r') as read_file:
downloaded_file = json.load(read_file)
return downloaded_file
def writeJson(filename, data):
try:
downloaded_file = readJson(filename)
downloaded_file.append(data)
except ValueError:
downloaded_file = [data]
with open(filename, 'w') as data_file:
json.dump(downloaded_file, fp=data_file, indent=2, encoding='utf-8')
if __name__ == '__main__':
agent = Agent(reactor, contextFactory=BrowserLikePolicyForHTTPS())
main(agent, generated_urls=URLS)
reactor.run()