def __init__(self, reactor, proxy_scheme, proxy_host, proxy_port, proxy_username=None, proxy_password=None, contextFactory=BrowserLikePolicyForHTTPS(), connectTimeout=None, bindAddress=None, pool=None): if not IPolicyForHTTPS.providedBy(contextFactory): raise NotImplementedError( 'contextFactory must implement IPolicyForHTTPS') self._policyForHTTPS = contextFactory self._wrappedAgent = BaseAgent.usingEndpointFactory(reactor, self, pool=pool) self._bindAddress = bindAddress self._connectTimeout = connectTimeout self.reactor = reactor self.proxy_scheme = proxy_scheme self.proxy_host = proxy_host self.proxy_port = proxy_port self.proxy_username = proxy_username self.proxy_password = proxy_password
def __init__( self, reactor, contextFactory=BrowserLikePolicyForHTTPS(), connectTimeout=None, bindAddress=None, pool=None, proxy_url_str: Optional[str] = None, ): _AgentBase.__init__(self, reactor, pool) self._endpoint_kwargs = {} if connectTimeout is not None: self._endpoint_kwargs["timeout"] = connectTimeout if bindAddress is not None: self._endpoint_kwargs["bindAddress"] = bindAddress if proxy_url_str is not None: parsed_url = decompose_http_proxy_url(proxy_url_str) self._proxy_auth = parsed_url.credentials self.proxy_endpoint = HostnameEndpoint(reactor, parsed_url.hostname, parsed_url.port, **self._endpoint_kwargs) else: self.proxy_endpoint = None self._policy_for_https = contextFactory self._reactor = reactor
def build_http_client_context_factory(self): config = self.get_config() return ( InsecureInterceptableContextFactory() if config.use_insecure_ssl_client_just_for_testing_do_not_use else BrowserLikePolicyForHTTPS() )
def __init__( self, reactor, proxy_reactor=None, contextFactory=BrowserLikePolicyForHTTPS(), connectTimeout=None, bindAddress=None, pool=None, http_proxy=None, https_proxy=None, ): _AgentBase.__init__(self, reactor, pool) if proxy_reactor is None: self.proxy_reactor = reactor else: self.proxy_reactor = proxy_reactor self._endpoint_kwargs = {} if connectTimeout is not None: self._endpoint_kwargs["timeout"] = connectTimeout if bindAddress is not None: self._endpoint_kwargs["bindAddress"] = bindAddress self.http_proxy_endpoint = _http_proxy_endpoint( http_proxy, self.proxy_reactor, **self._endpoint_kwargs) self.https_proxy_endpoint = _http_proxy_endpoint( https_proxy, self.proxy_reactor, **self._endpoint_kwargs) self._policy_for_https = contextFactory self._reactor = reactor
def _get_http_client(): response = yield treq.get(ca) pemdata = yield response.text() cert = Certificate.loadPEM(pemdata) policy = BrowserLikePolicyForHTTPS(trustRoot=cert) agent = Agent(reactor, contextFactory=policy) client = treq.client.HTTPClient(agent) returnValue(client)
def __init__(self, good_domains=None): """ :param good_domains: List of domains. The URLs must be in bytes """ if not good_domains: self.good_domains = [] else: self.good_domains = good_domains # by default, handle requests like a browser would self.default_policy = BrowserLikePolicyForHTTPS()
def patch(): try: base.WebVerifyingContextFactory except AttributeError: # Don't do anything if that thing has gone away. pass else: # Replace that thing with another thing that makes a better thing! # See the ticket for details. base.WebVerifyingContextFactory = lambda host: BrowserLikePolicyForHTTPS()
def __init__(self, reactor, contextFactory=BrowserLikePolicyForHTTPS(), connectTimeout=None, bindAddress=None, pool=None, proxyEndpoint=None, endpointArgs={}): if not IPolicyForHTTPS.providedBy(contextFactory): raise NotImplementedError( 'contextFactory must implement IPolicyForHTTPS') self.proxyEndpoint = proxyEndpoint self.endpointArgs = endpointArgs self._policyForHTTPS = contextFactory self._wrappedAgent = _Agent.usingEndpointFactory( reactor, self, pool=pool)
def __init__(self, config): if not have_twisted: raise ImportError('Twisted required for TwistedTransport') TransportBase.__init__(self, config, self.__module__) if config.get("ignore-ssl-errors"): policy = NoValidationPolicy() else: policy = BrowserLikePolicyForHTTPS() self.agent = Agent(reactor, policy)
def get_test_https_policy(): """Get a test IPolicyForHTTPS which trusts the test CA cert Returns: IPolicyForHTTPS """ ca_file = get_test_ca_cert_file() with open(ca_file) as stream: content = stream.read() cert = Certificate.loadPEM(content) trust_root = trustRootFromCertificates([cert]) return BrowserLikePolicyForHTTPS(trustRoot=trust_root)
def __init__( self, reactor, proxy_reactor=None, contextFactory: Optional[IPolicyForHTTPS] = None, connectTimeout=None, bindAddress=None, pool=None, use_proxy=False, ): contextFactory = contextFactory or BrowserLikePolicyForHTTPS() _AgentBase.__init__(self, reactor, pool) if proxy_reactor is None: self.proxy_reactor = reactor else: self.proxy_reactor = proxy_reactor self._endpoint_kwargs = {} if connectTimeout is not None: self._endpoint_kwargs["timeout"] = connectTimeout if bindAddress is not None: self._endpoint_kwargs["bindAddress"] = bindAddress http_proxy = None https_proxy = None no_proxy = None if use_proxy: proxies = getproxies_environment() http_proxy = proxies["http"].encode( ) if "http" in proxies else None https_proxy = proxies["https"].encode( ) if "https" in proxies else None no_proxy = proxies["no"] if "no" in proxies else None # Parse credentials from https proxy connection string if present self.https_proxy_creds, https_proxy = parse_username_password( https_proxy) self.http_proxy_endpoint = http_proxy_endpoint(http_proxy, self.proxy_reactor, **self._endpoint_kwargs) self.https_proxy_endpoint = http_proxy_endpoint( https_proxy, self.proxy_reactor, **self._endpoint_kwargs) self.no_proxy = no_proxy self._policy_for_https = contextFactory self._reactor = reactor
def __init__( self, reactor: IReactorCore, proxy_reactor: Optional[ISynapseReactor] = None, contextFactory: Optional[IPolicyForHTTPS] = None, connectTimeout: Optional[float] = None, bindAddress: Optional[bytes] = None, pool: Optional[HTTPConnectionPool] = None, use_proxy: bool = False, ): contextFactory = contextFactory or BrowserLikePolicyForHTTPS() _AgentBase.__init__(self, reactor, pool) if proxy_reactor is None: self.proxy_reactor = reactor else: self.proxy_reactor = proxy_reactor self._endpoint_kwargs: Dict[str, Any] = {} if connectTimeout is not None: self._endpoint_kwargs["timeout"] = connectTimeout if bindAddress is not None: self._endpoint_kwargs["bindAddress"] = bindAddress http_proxy = None https_proxy = None no_proxy = None if use_proxy: proxies = getproxies_environment() http_proxy = proxies["http"].encode( ) if "http" in proxies else None https_proxy = proxies["https"].encode( ) if "https" in proxies else None no_proxy = proxies["no"] if "no" in proxies else None self.http_proxy_endpoint, self.http_proxy_creds = http_proxy_endpoint( http_proxy, self.proxy_reactor, contextFactory, **self._endpoint_kwargs) self.https_proxy_endpoint, self.https_proxy_creds = http_proxy_endpoint( https_proxy, self.proxy_reactor, contextFactory, **self._endpoint_kwargs) self.no_proxy = no_proxy self._policy_for_https = contextFactory self._reactor = reactor
def agent_call(url_trustRoot): url, trustRoot = url_trustRoot if trustRoot: customPolicy = BrowserLikePolicyForHTTPS( PrivateCertificate.loadPEM(trustRoot)) agent = Agent(reactor, customPolicy) else: agent = Agent(reactor) headers = Headers({ 'User-Agent': ['Twisted Webbot'], 'Content-Type': ['text/x-greeting'] }) d = agent.request('HEAD', url, headers=headers) return d
def __init__(self, config): if sys.version_info[0] == 3 and sys.version_info <= (3, 5, 0): raise ResourceException( "Twisted on Python 3 requires Python 3.5 or later.") if not have_twisted: raise ImportError("Twisted required for TwistedTransport") TransportBase.__init__(self, config, self.__module__) self._timeout = self._config.get("timeout", None) if config.get("ignore-ssl-errors"): policy = NoValidationPolicy() else: policy = BrowserLikePolicyForHTTPS() self.agent = Agent(reactor, policy, connectTimeout=self._timeout)
def __init__( self, reactor: ReactorBase, pool: H2ConnectionPool, context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS( ), connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None, ) -> None: self._reactor = reactor self._pool = pool self._context_factory = AcceptableProtocolsContextFactory( context_factory, acceptable_protocols=[b'h2']) self.endpoint_factory = _StandardEndpointFactory( self._reactor, self._context_factory, connect_timeout, bind_address)
def __init__( self, reactor: ReactorBase, proxy_uri: URI, pool: H2ConnectionPool, context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS( ), connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None, ) -> None: super(ScrapyProxyH2Agent, self).__init__( reactor=reactor, pool=pool, context_factory=context_factory, connect_timeout=connect_timeout, bind_address=bind_address, ) self._proxy_uri = proxy_uri
def __init__( self, reactor, proxy_reactor=None, contextFactory=BrowserLikePolicyForHTTPS(), connectTimeout=None, bindAddress=None, pool=None, use_proxy=False, ): _AgentBase.__init__(self, reactor, pool) if proxy_reactor is None: self.proxy_reactor = reactor else: self.proxy_reactor = proxy_reactor self._endpoint_kwargs = {} if connectTimeout is not None: self._endpoint_kwargs["timeout"] = connectTimeout if bindAddress is not None: self._endpoint_kwargs["bindAddress"] = bindAddress http_proxy = None https_proxy = None no_proxy = None if use_proxy: proxies = getproxies_environment() http_proxy = proxies["http"].encode( ) if "http" in proxies else None https_proxy = proxies["https"].encode( ) if "https" in proxies else None no_proxy = proxies["no"] if "no" in proxies else None self.http_proxy_endpoint = _http_proxy_endpoint( http_proxy, self.proxy_reactor, **self._endpoint_kwargs) self.https_proxy_endpoint = _http_proxy_endpoint( https_proxy, self.proxy_reactor, **self._endpoint_kwargs) self.no_proxy = no_proxy self._policy_for_https = contextFactory self._reactor = reactor
def createVerifyingHTTPClient(reactor, extra_ca_certs=None, agent_kwds=None, **kwds): """ extra_ca_certs: Should be a list of PEM formatted certificates that are trust anchors. """ agent_kwds = normalizeDict_(agent_kwds) if extra_ca_certs is not None: trust_anchors = [] for ca_cert in extra_ca_certs: with open(ca_cert, "rb") as f: data = f.read() cert = crypto.load_certificate(crypto.FILETYPE_PEM, data) del data trust_anchors.append(cert) policy = CustomPolicyForHTTPS(trust_anchors) else: policy = BrowserLikePolicyForHTTPS() agent_kwds['contextFactory'] = policy return HTTPClient(Agent(reactor, **agent_kwds), **kwds)
def getPolicyForHTTPS(trustRoot=None): if SKIP_SSL_CHECK: log.info("---------------------------------------") log.info("SKIPPING SSL CERT VERIFICATION!!!") log.info("I assume you know WHAT YOU ARE DOING...") log.info("---------------------------------------") class WebClientContextFactory(ClientContextFactory): """ A web context factory which ignores the hostname and port and does no certificate verification. """ def getContext(self, hostname, port): return ClientContextFactory.getContext(self) contextFactory = WebClientContextFactory() return contextFactory if isinstance(trustRoot, str): trustRoot = Certificate.loadPEM(FilePath(trustRoot).getContent()) return BrowserLikePolicyForHTTPS(trustRoot)
def __init__(self): self._normalPolicy = BrowserLikePolicyForHTTPS()
monitoring_data["{}".format(results.get("name"))] = { "unit": param.get("unit"), "param_name": param.get("displayName"), "last_value": param.get("lastValue"), "last_updated": param.get("lastUpdated"), "parameter_id": param.get("parameterId") } writeJson(fp, monitoring_data) def readJson(filename='modeling_data.json'): with open(filename, 'r') as read_file: downloaded_file = json.load(read_file) return downloaded_file def writeJson(filename, data): try: downloaded_file = readJson(filename) downloaded_file.append(data) except ValueError: downloaded_file = [data] with open(filename, 'w') as data_file: json.dump(downloaded_file, fp=data_file, indent=2, encoding='utf-8') if __name__ == '__main__': agent = Agent(reactor, contextFactory=BrowserLikePolicyForHTTPS()) main(agent, generated_urls=URLS) reactor.run()
def cookieAgentFactory(verify_path, connectTimeout=30): customPolicy = BrowserLikePolicyForHTTPS( Certificate.loadPEM(FilePath(verify_path).getContent())) agent = Agent(reactor, customPolicy, connectTimeout=connectTimeout) cookiejar = cookielib.CookieJar() return CookieAgent(agent, cookiejar)
def main(reactor): agent = Agent(reactor, contextFactory=BrowserLikePolicyForHTTPS()) for url in urls: d = agent.request(method=METHOD, uri=url, headers=Headers(HEADERS)) d.addCallbacks(callback_request, error_handler) return d
def __init__(self): self.default_policy = BrowserLikePolicyForHTTPS()
def __init__(self): # by default, handle requests like a browser would self.default_policy = BrowserLikePolicyForHTTPS()
def __init__(self, reactor, address): """Initialize.""" self.reactor = reactor self.address = address self.context_factory = BrowserLikePolicyForHTTPS()
print(failure) return failure def cbBody(body): # print('Response body:') body = body.decode('utf8') return body pool = HTTPConnectionPool(reactor) endpoint = TCP4ClientEndpoint(reactor, settings.PROXY_ADDRESS, settings.PROXY_PORT) tunnelingAgent = TunnelingAgent( reactor, (settings.PROXY_ADDRESS, settings.PROXY_PORT, None), BrowserLikePolicyForHTTPS(), pool=pool) proxyAgent = ProxyAgent(endpoint, pool=pool) normalAgent = Agent(reactor, pool=pool) # pool = None def get(reactor, url, headers={}, body=None): ssl = url.split(':')[0] if ssl == 'https' and settings.USE_PROXY: agent = tunnelingAgent else: if settings.USE_PROXY: agent = proxyAgent
def __init__(self, config): # NOTE: ServerFactory.__init__ does not exist? # Read configuration options. if config['privacyidea']['verify']: if config['privacyidea']['certificate']: certificate_path = config['privacyidea']['certificate'] certificate = Certificate.loadPEM(FilePath(certificate_path).getContent()) log.info('privacyIDEA HTTPS certificate will be checked against {certificate!r} from {path!r}', certificate=certificate, path=certificate_path) else: certificate = None log.info('privacyIDEA HTTPS certificate will be checked against system certificate store') https_policy = BrowserLikePolicyForHTTPS(certificate) else: log.warn('privacyIDEA HTTPS certificate will NOT be checked!') https_policy = DisabledVerificationPolicyForHTTPS() self.agent = Agent(reactor, https_policy) if config['ldap-backend']['use-tls']: # TODO: This seems to get lost if we use log.info log.warn('The use-tls config option is deprecated and will be ignored.') self.proxied_endpoint_string = config['ldap-backend']['endpoint'] self.privacyidea_instance = config['privacyidea']['instance'] # Construct the validate url from the instance location if self.privacyidea_instance[-1] != '/': self.privacyidea_instance += '/' self.validate_url = VALIDATE_URL_TEMPLATE.format(self.privacyidea_instance) self.service_account_dn = config['service-account']['dn'] self.service_account_password = config['service-account']['password'] # We have to make a small workaround for configobj here: An empty config value # is interpreted as a list with one element, the empty string. self.passthrough_binds = config['ldap-proxy']['passthrough-binds'] if len(self.passthrough_binds) == 1 and self.passthrough_binds[0] == '': self.passthrough_binds = [] log.info('Passthrough DNs: {binds!r}', binds=self.passthrough_binds) self.forward_anonymous_binds = config['ldap-proxy']['forward-anonymous-binds'] self.allow_search = config['ldap-proxy']['allow-search'] self.bind_service_account = config['ldap-proxy']['bind-service-account'] self.allow_connection_reuse = config['ldap-proxy']['allow-connection-reuse'] self.ignore_search_result_references = config['ldap-proxy']['ignore-search-result-references'] user_mapping_strategy = USER_MAPPING_STRATEGIES[config['user-mapping']['strategy']] log.info('Using user mapping strategy: {strategy!r}', strategy=user_mapping_strategy) self.user_mapper = user_mapping_strategy(self, config['user-mapping']) realm_mapping_strategy = REALM_MAPPING_STRATEGIES[config['realm-mapping']['strategy']] log.info('Using realm mapping strategy: {strategy!r}', strategy=realm_mapping_strategy) self.realm_mapper = realm_mapping_strategy(self, config['realm-mapping']) enable_bind_cache = config['bind-cache']['enabled'] if enable_bind_cache: self.bind_cache = BindCache(config['bind-cache']['timeout']) else: self.bind_cache = None enable_app_cache = config['app-cache']['enabled'] if enable_app_cache: self.app_cache = AppCache(config['app-cache']['timeout'], config['app-cache']['case-insensitive']) else: self.app_cache = None self.app_cache_attribute = config['app-cache']['attribute'] self.app_cache_value_prefix = config['app-cache']['value-prefix'] if config['ldap-backend']['test-connection']: self.test_connection()