def test_proxy_cgi_ignore(self): try: self.env.set('HTTP_PROXY', 'http://somewhere:3128') proxies = urllib.getproxies_environment() self.assertEqual('http://somewhere:3128', proxies['http']) self.env.set('REQUEST_METHOD', 'GET') proxies = urllib.getproxies_environment() self.assertNotIn('http', proxies) finally: self.env.unset('REQUEST_METHOD') self.env.unset('HTTP_PROXY')
def testHost(url): ''' Test if we can connect to the host of `url` (not to the complete URL). ''' (scheme, netloc, _path, _params, _query, _fragment) = urlparse(str(url)) if len(urllib.getproxies_environment()) > 0: return testUrl(scheme + "://" + netloc) if ":" in netloc: (host, port) = netloc.split(":") else: host = netloc if scheme == "https": port = "443" else: port = "80" logger = ObsLightPrintManager.getLogger() logger.info("Testing connection to '%s:%s'", host, port) conn = createConn(host, port, scheme) try: conn.connect() except BaseException, e: message = "Could not connect to %s: %s" % (str(host), str(e)) logger.warning(message) return False
def __init__(self, service, operation, region_name, endpoint_url=None, session=None, connect_timeout=None, request_timeout=None): # set credentials manually session = session or botocore.session.get_session() # get_session accepts access_key, secret_key self.client = session.create_client( service, region_name=region_name, endpoint_url=endpoint_url ) self.endpoint = self.client._endpoint self.operation = operation self.http_client = AsyncHTTPClient() self.proxy_host = None self.proxy_port = None https_proxy = getproxies_environment().get('https') if https_proxy: proxy_parts = https_proxy.split(':') if len(proxy_parts) == 2 and proxy_parts[-1].isdigit(): self.proxy_host, self.proxy_port = proxy_parts self.proxy_port = int(self.proxy_port) else: proxy = urlparse(https_proxy) self.proxy_host = proxy.hostname self.proxy_port = proxy.port self.request_timeout = request_timeout self.connect_timeout = connect_timeout
def __init__(self, service, operation, region_name, endpoint_url=None, session=None, connect_timeout=None, request_timeout=None): # set credentials manually session = session or botocore.session.get_session() # get_session accepts access_key, secret_key self.client = session.create_client( service, region_name=region_name, endpoint_url=endpoint_url ) try: self.endpoint = self.client.endpoint except AttributeError: self.endpoint = self.client._endpoint self.operation = operation self.http_client = AsyncHTTPClient() self.proxy_host = None self.proxy_port = None https_proxy = getproxies_environment().get('https') if https_proxy: self._enable_curl_httpclient() proxy_parts = https_proxy.split(':') if len(proxy_parts) == 2 and proxy_parts[-1].isdigit(): self.proxy_host, self.proxy_port = proxy_parts self.proxy_port = int(self.proxy_port) else: proxy = urlparse(https_proxy) self.proxy_host = proxy.hostname self.proxy_port = proxy.port self.request_timeout = request_timeout self.connect_timeout = connect_timeout
def importCert(url): ''' Import the SSL certificate of an HTTPS server into osc configuration. ''' (scheme, netloc, _path, _params, _query, _fragment) = urlparse(str(url)) if scheme == "http": return if ":" in netloc: (host, port) = netloc.split(":") port = int(port) else: host = netloc port = 443 ctx = SSL.Context() ctx.set_allow_unknown_ca(True) ctx.set_verify(SSL.verify_none, 1) proxyEnv = urllib.getproxies_environment() # If there is a proxy environment and the host is not in no_proxy useProxy = len(proxyEnv) > 0 and host not in proxyEnv.get('no', []) if useProxy: valProxy = proxyEnv['https'] netlocProxy = urlparse(valProxy)[1] [__PROXYHOST__, __PROXYPORT__] = netlocProxy.split(":") conn = M2Crypto.httpslib.ProxyHTTPSConnection(host=__PROXYHOST__, port=__PROXYPORT__) conn.ssl_ctx = ctx conn.putrequest('HEAD', url) try: conn.connect() except: raise else: conn = SSL.Connection(ctx) conn.postConnectionCheck = None timeout = SSL.timeout(SOCKETTIMEOUT) conn.set_socket_read_timeout(timeout) conn.set_socket_write_timeout(timeout) try: conn.connect((host, port)) except: raise if useProxy: cert = conn.sock.get_peer_cert() else: cert = conn.get_peer_cert() # if the peer did not provide a certificate chain, cert is None. if cert is not None: dirpath = expanduser('~/.config/osc/trusted-certs') if not exists(dirpath): makedirs(dirpath) filePath = dirpath + '/%s_%d.pem' % (host, port) cert.save_pem(filePath) conn.close()
def test_getproxies_environment_keep_no_proxies(self): self.env.set('NO_PROXY', 'localhost') proxies = urllib.getproxies_environment() # getproxies_environment use lowered case truncated (no '_proxy') keys self.assertEqual('localhost', proxies['no']) # List of no_proxies with space. self.env.set('NO_PROXY', 'localhost, anotherdomain.com, newdomain.com') self.assertTrue(urllib.proxy_bypass_environment('anotherdomain.com'))
def test_getproxies_environment_keep_no_proxies(self): self.env.set("NO_PROXY", "localhost") proxies = urllib.getproxies_environment() # getproxies_environment use lowered case truncated (no '_proxy') keys self.assertEqual("localhost", proxies["no"]) # List of no_proxies with space. self.env.set("NO_PROXY", "localhost, anotherdomain.com, newdomain.com") self.assertTrue(urllib.proxy_bypass_environment("anotherdomain.com"))
def getUrl(scheme, netloc, path): if len(urllib.getproxies_environment()) > 0: if path == "": return scheme + "://" + netloc + "/" else: return scheme + "://" + netloc + path else: return path
def _request(self, url, method): scheme, host = urlparse(url)[:2] scheme = scheme.lower() proxies = getproxies_environment() if scheme in proxies: scheme, host = urlparse(proxies[scheme])[:2] scheme = scheme.lower() kwargs = {} if version_info[1] >= 6: kwargs['timeout'] = self.timeout else: socket.setdefaulttimeout(self.timeout) if scheme == "https": conn = HTTPSConnection(host, **kwargs) else: conn = HTTPConnection(host, **kwargs) headers = {} if method == 'GET': headers['Range'] = 'bytes=0-%s' % self.max_size try: try: conn.request(method.upper(), url_to_bytestring(url), headers=headers) response = conn.getresponse() data = response.read(self.max_size) conn.close() except socket.error, e: raise HTTPException(e.message or e.args[1]) finally: if version_info[1] < 6: socket.setdefaulttimeout(None) contenttype = response.getheader('Content-Type', None) if contenttype: match = re.search('^charset=([a-zA-Z0-9-]+)', contenttype) try: if match: data = data.decode(match.group(1)) elif contenttype.startswith('text/'): data = data.decode('utf-8') except UnicodeDecodeError: guessed = detect(data) if guessed['confidence'] > 0.5: charset = guessed['encoding'] # Common guessing mistake: if charset.startswith('ISO-8859') and '\x92' in data: charset = 'windows-1252' data = unicode(data, charset, errors='replace') return response.status, response.reason, data, response.getheaders()
def _request(self, url, method): scheme, host = urlparse(url)[:2] scheme = scheme.lower() proxies = getproxies_environment() if scheme in proxies: scheme, host = urlparse(proxies[scheme])[:2] scheme = scheme.lower() kwargs = {} if version_info[1] >= 6: kwargs["timeout"] = self.timeout else: socket.setdefaulttimeout(self.timeout) if scheme == "https": conn = HTTPSConnection(host, **kwargs) else: conn = HTTPConnection(host, **kwargs) headers = {} if method == "GET": headers["Range"] = "bytes=0-%s" % self.max_size try: try: conn.request(method.upper(), iri_to_uri(url), headers=headers) response = conn.getresponse() data = response.read(self.max_size) conn.close() except socket.error, e: raise HTTPException(e.message or e.args[1]) finally: if version_info[1] < 6: socket.setdefaulttimeout(None) contenttype = response.getheader("Content-Type", None) if contenttype: match = re.search("^charset=([a-zA-Z0-9-]+)", contenttype) try: if match: data = data.decode(match.group(1)) elif contenttype.startswith("text/"): data = data.decode("utf-8") except UnicodeDecodeError: guessed = detect(data) if guessed["confidence"] > 0.5: charset = guessed["encoding"] # Common guessing mistake: if charset.startswith("ISO-8859") and "\x92" in data: charset = "windows-1252" data = unicode(data, charset, errors="replace") return response.status, response.reason, data, response.getheaders()
def createConn(host, port, scheme): if scheme == "https": if 'https' in urllib.getproxies_environment(): proxies_env = urllib.getproxies_environment() if 'https' in proxies_env.keys(): valProxy = ['https'] else: valProxy = ['http'] netlocProxy = urlparse(valProxy)[2] [__PROXYHOST__, __PROXYPORT__] = netlocProxy.split(":") conn = httplib.HTTPConnection(host=__PROXYHOST__, port=__PROXYPORT__, timeout=SOCKETTIMEOUT) conn.set_tunnel(host=host, port=int(port)) ctx = SSL.Context() ctx.set_allow_unknown_ca(True) ctx.set_verify(SSL.verify_none, 1) conn = M2Crypto.httpslib.ProxyHTTPSConnection(host=__PROXYHOST__, port=__PROXYPORT__) conn.ssl_ctx = ctx conn.putrequest('HEAD', scheme + "://" + ":".join(host, port)) return conn else: return httplib.HTTPSConnection(host=host, port=port, timeout=SOCKETTIMEOUT) else: if 'http' in urllib.getproxies_environment(): valProxy = urllib.getproxies_environment()['http'] netlocProxy = urlparse(valProxy)[2] [__PROXYHOST__, __PROXYPORT__] = netlocProxy.split(":") return httplib.HTTPConnection(host=__PROXYHOST__, port=__PROXYPORT__, timeout=SOCKETTIMEOUT) else: return httplib.HTTPConnection(host=host, port=port, timeout=SOCKETTIMEOUT)
def test_getproxies_environment_prefer_lowercase(self): # Test lowercase preference with removal os.environ['no_proxy'] = '' os.environ['No_Proxy'] = 'localhost' self.assertFalse(urllib.proxy_bypass_environment('localhost')) self.assertFalse(urllib.proxy_bypass_environment('arbitrary')) os.environ['http_proxy'] = '' os.environ['HTTP_PROXY'] = 'http://somewhere:3128' proxies = urllib.getproxies_environment() self.assertEqual({}, proxies) # Test lowercase preference of proxy bypass and correct matching including ports os.environ['no_proxy'] = 'localhost, noproxy.com, my.proxy:1234' os.environ['No_Proxy'] = 'xyz.com' self.assertTrue(urllib.proxy_bypass_environment('localhost')) self.assertTrue(urllib.proxy_bypass_environment('noproxy.com:5678')) self.assertTrue(urllib.proxy_bypass_environment('my.proxy:1234')) self.assertFalse(urllib.proxy_bypass_environment('my.proxy')) self.assertFalse(urllib.proxy_bypass_environment('arbitrary')) # Test lowercase preference with replacement os.environ['http_proxy'] = 'http://somewhere:3128' os.environ['Http_Proxy'] = 'http://somewhereelse:3128' proxies = urllib.getproxies_environment() self.assertEqual('http://somewhere:3128', proxies['http'])
def __init__(self, gitlab_url, gitlab_token): self.dag = dag.DAG() self.gitlab_url = gitlab_url self.gitlab_token = gitlab_token self.gitlab = gitlab.Gitlab(gitlab_url, gitlab_token, api_version=4) # TODO: This should be automatic but it's not (python-gitlab bug) self.gitlab.session.proxies.update(urllib.getproxies_environment()) # Cache group/project to Gitlab project objects. If projects # changes the deamon must be restarted self.projects = {} # cache for triggers self.triggers = {} # Projects which must not be built self.locked_projects = set() self.finished_at = {}
def testUrl(url): ''' Test if we can reach `url`. ''' logger = ObsLightPrintManager.getLogger() logger.info("Testing URL '%s'", url) opener = urllib2.build_opener( urllib2.ProxyHandler(urllib.getproxies_environment())) urllib2.install_opener(opener) try: _test = urllib2.urlopen(url, timeout=SOCKETTIMEOUT) return True except urllib2.URLError, e: message = "Could not reach %s: %s" % (str(url), str(e)) logger.warning(message) return False
def createImage(self): """ Launch the build of an image. """ self.failIsUserNotInUserGroup() timeString = time.strftime("%Y-%m-%d_%Hh%Mm") + str(time.time() % 1).split(".")[1] logFilePath = os.path.join(self.projectDirectory, "build_" + timeString, "buildLog") # cacheDirPath = os.path.join(self.projectDirectory, "cache") proxies = urllib.getproxies_environment() for scheme in proxies.keys(): if scheme == 'http': cmd = "sudo sed -r -i 's,(; *)*proxy =.*,proxy = " + proxies[scheme] + ",' /etc/mic/mic.conf" self.__subprocess(cmd) cmd = "sudo mic --debug --verbose create " + self.getImageType() cmd += " " + self.getKickstartFile() cmd += " --logfile=" + logFilePath # cmd += " --cachedir=" + cacheDirPath cmd += " --outdir=" + self.projectDirectory cmd += " --arch=" + self.__architecture # cmd += " --release=build_" + timeString cmd += " --local-pkgs-path=" + self.localPackagesDirectory self.__subprocess(cmd)
def getproxy(schema = 'http'): p = urllib.getproxies_environment() return p.get(schema,None)
def test_getproxies_environment_keep_no_proxies(self): self.env.set("NO_PROXY", "localhost") proxies = urllib.getproxies_environment() # getproxies_environment use lowered case truncated (no '_proxy') keys self.assertEqual("localhost", proxies["no"])
# Current version of webcheck. VERSION = '1.9.8' # The homepage of webcheck. HOMEPAGE = 'http://ch.tudelft.nl/~arthur/webcheck/' # Whether to consider any URL not starting with the base URL to be external. # This is the state of the -b command line option. BASE_URLS_ONLY = False # Avoid checking external links at all. This is the state of the -a command # line option. AVOID_EXTERNAL_LINKS = False # The proxy configuration. PROXIES = urllib.getproxies_environment() # IO timeout as passed to socket.setdefaulttimeout() # value is a float in seconds None disables the timeout IOTIMEOUT = 10.0 # Output directory. This is the state of the -o command line option. OUTPUT_DIR = '.' # Whether to try to read a state file to continue from. CONTINUE = False # This is the time in seconds to wait between requests. This is the state of # the -w command line option. WAIT_BETWEEN_REQUESTS = 0
import httplib, mimetypes, urllib proxies = urllib.getproxies_environment() def post_multipart(host, selector, fields): """ Post fields and files to an http host as multipart/form-data. fields is a sequence of (name, filename, value) elements for form fields. If filename is None, the field is treated as a regular field; otherwise, the field is uploaded as a file. Return the server's response page. """ return post_multipart_formdata(host, selector, fields)[3] def post_multipart_formdata(host, selector, fields): content_type, body = encode_multipart_formdata(fields) try: realhost = proxies["http"] except KeyError: realhost = host h = httplib.HTTP(realhost) h.putrequest('POST', selector) h.putheader('content-type', content_type) h.putheader('content-length', str(len(body))) h.putheader('host', host) h.endheaders() h.send(body) retcode, retmsg, headers = h.getreply() return retcode, retmsg, headers, h.file.read() def encode_multipart_formdata(fields):
import httplib, mimetypes, urllib proxies = urllib.getproxies_environment() def post_multipart(host, selector, fields): """ Post fields and files to an http host as multipart/form-data. fields is a sequence of (name, filename, value) elements for form fields. If filename is None, the field is treated as a regular field; otherwise, the field is uploaded as a file. Return the server's response page. """ return post_multipart_formdata(host, selector, fields)[3] def post_multipart_formdata(host, selector, fields): content_type, body = encode_multipart_formdata(fields) try: realhost = proxies["http"] except KeyError: realhost = host h = httplib.HTTP(realhost) h.putrequest('POST', selector) h.putheader('content-type', content_type) h.putheader('content-length', str(len(body))) h.putheader('host', host) h.endheaders() h.send(body) retcode, retmsg, headers = h.getreply() return retcode, retmsg, headers, h.file.read()
def test_getproxies_environment_keep_no_proxies(self): self.env.set('NO_PROXY', 'localhost') proxies = urllib.getproxies_environment() # getproxies_environment use lowered case truncated (no '_proxy') keys self.assertEquals('localhost', proxies['no'])
def test_getproxies_environment_keep_no_proxies(self): self.env.set('NO_PROXY', 'localhost') proxies = urllib.getproxies_environment() # getproxies_environment use lowered case truncated (no '_proxy') keys self.assertEqual('localhost', proxies['no'])
# Current version of webcheck. VERSION = "1.9.6" # The homepage of webcheck. HOMEPAGE = "http://ch.tudelft.nl/~arthur/webcheck/" # Whether to consider any URL not starting with the base URL to be external. # This is the state of the -b command line option. BASE_URLS_ONLY = False # Avoid checking external links at all. This is the state of the -a command # line option. AVOID_EXTERNAL_LINKS = False # The proxy configuration. PROXIES = urllib.getproxies_environment() # Output directory. This is the state of the -o command line option. OUTPUT_DIR = "." # This is the time in seconds to wait between requests. This is the state of # the -w command line option. WAIT_BETWEEN_REQUESTS = 0 # Redirect depth, the number of redirects to follow. This is the state of the # -r command line option. REDIRECT_DEPTH = 5 # The list of plugins that will be used to generate the report. PLUGINS = ["sitemap", "urllist",
def proxied(value): netloc = urlparse(value).netloc proxied = bool(getproxies_environment()) and not proxy_bypass(netloc) return (proxied)