def __get_proxies(self): try: mm_http_proxy = config.connection.get_plugin_client_setting('mm_http_proxy', None) mm_http_proxy = mm_http_proxy.replace("\\","") mm_https_proxy = config.connection.get_plugin_client_setting('mm_https_proxy', None) mm_https_proxy = mm_https_proxy.replace("\\","") if mm_https_proxy == None and mm_http_proxy != None: mm_https_proxy = mm_http_proxy if mm_http_proxy != None and mm_https_proxy != None: return { "http": mm_http_proxy, "https": mm_https_proxy } elif mm_http_proxy != None: return { "http": mm_http_proxy } elif mm_https_proxy != None: return { "https": mm_https_proxy } else: return urllib.getproxies() except: return urllib.getproxies()
def getAutoProxy(self): """Fetch the proxy from the the system environment variables """ if urllib.getproxies().has_key('http'): return urllib.getproxies()['http'] else: return ""
def start_schedulers(options): try: from multiprocessing import Process except: sys.stderr.write('Sorry, -K only supported for python 2.6-2.7\n') return processes = [] apps = [(app.strip(), None) for app in options.scheduler.split(',')] if options.scheduler_groups: apps = options.scheduler_groups code = "from gluon.globals import current;current._scheduler.loop()" logging.getLogger().setLevel(options.debuglevel) if options.folder: os.chdir(options.folder) if len(apps) == 1 and not options.with_scheduler: app_, code = get_code_for_scheduler(apps[0], options) if not app_: return print('starting single-scheduler for "%s"...' % app_) run(app_, True, True, None, False, code) return # Work around OS X problem: http://bugs.python.org/issue9405 if PY2: import urllib else: import urllib.request as urllib urllib.getproxies() for app in apps: app_, code = get_code_for_scheduler(app, options) if not app_: continue print('starting scheduler for "%s"...' % app_) args = (app_, True, True, None, False, code) p = Process(target=run, args=args) processes.append(p) print("Currently running %s scheduler processes" % (len(processes))) p.start() ##to avoid bashing the db at the same time time.sleep(0.7) print("Processes started") for p in processes: try: p.join() except (KeyboardInterrupt, SystemExit): print("Processes stopped") except: p.terminate() p.join()
def connect(self): #- Parse proxies pd = urllib.getproxies().get('http', None) if pd is None: sockstype = '' else: sockstype, user, password, hostport = urllib2._parse_proxy(pd) if 'socks' not in sockstype: return httplib.HTTPConnection.connect(self) assert ':' in hostport # if we don't have a port we're screwed host, port = hostport.rsplit(':', 1) port = int(port) for res in socket.getaddrinfo(self.host, self.port, 0, socket.SOCK_STREAM): af, socktype, proto, canonname, sa = res try: self.sock = self._sockettype(af, socktype, proto) self.sock.setproxy(proxytype=getattr(socks, 'PROXY_TYPE_%s' % sockstype.upper()), addr=host, port=port, rdns=False, username=user, password=password) #- The rest is the same as superclass if self.debuglevel > 0: print "connect: (%s, %s)" % (self.host, self.port) self.sock.connect(sa) except socket.error, msg: if self.debuglevel > 0: print 'connect fail:', (self.host, self.port) if self.sock: self.sock.close() self.sock = None continue break
def get_environ_proxies(netloc): """Return a dict of environment proxies.""" get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper()) # First check whether no_proxy is defined. If it is, check that the URL # we're getting isn't in the no_proxy list. no_proxy = get_proxy('no_proxy') if no_proxy: # We need to check whether we match here. We need to see if we match # the end of the netloc, both with and without the port. no_proxy = no_proxy.replace(' ', '').split(',') for host in no_proxy: if netloc.endswith(host) or netloc.split(':')[0].endswith(host): # The URL does match something in no_proxy, so we don't want # to apply the proxies on this URL. return {} # If the system proxy settings indicate that this URL should be bypassed, # don't proxy. if proxy_bypass(netloc): return {} # If we get here, we either didn't have no_proxy set or we're not going # anywhere that no_proxy applies to, and the system settings don't require # bypassing the proxy for the current URL. return getproxies()
def __init__(self, radiodns_services): self._radiodns_services = radiodns_services self._dns = DnsResolver() self._proxy_settings = None self._use_http_proxy = False self._radiovis_client = None self._http_client = HttpClientThread(self) self._http_client.start() self._listeners = [] # Get system proxy server settings (from http_proxy environment variable # or web browser settings). proxies = urllib.getproxies() if "http" in proxies: http_proxy = urlparse.urlparse(proxies['http']) self._proxy_settings = ProxySettings(proxy_type = socks.PROXY_TYPE_HTTP, host = http_proxy.hostname, port = http_proxy.port) self.log("HTTP proxy: " + http_proxy.hostname + ", port " + str(http_proxy.port)) else: self._proxy_settings = None
def urlopen(url, proxies=None, data=None): global _opener if not proxies: proxies = urllib.getproxies() headers = {'User-Agent': UA_STR, 'Accept-Encoding' : 'gzip;q=1.0, deflate;q=0.9, identity;q=0.5'} req = urllib2.Request(url, data, headers) proxy_support = urllib2.ProxyHandler(proxies) if _opener is None: pwd_manager = handlepasswd() handlers = [proxy_support, urllib2.UnknownHandler, HttpWithGzipHandler, urllib2.HTTPBasicAuthHandler(pwd_manager), urllib2.ProxyBasicAuthHandler(pwd_manager), urllib2.HTTPDigestAuthHandler(pwd_manager), urllib2.ProxyDigestAuthHandler(pwd_manager), urllib2.HTTPDefaultErrorHandler, urllib2.HTTPRedirectHandler, ] if hasattr(httplib, 'HTTPS'): handlers.append(HttpsWithGzipHandler) _opener = urllib2.build_opener(*handlers) # print _opener.handlers urllib2.install_opener(_opener) return _opener.open(req)
def do_upload_and_exit(path, url, proxy): f = open(path, 'rb') # mmap the file to reduce the amount of memory required (see bit.ly/2aNENXC) filedata = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) # Get proxies from environment/system proxy_handler = urllib2.ProxyHandler(urllib.getproxies()) if proxy != "": # unless a proxy is explicitly passed, then use that instead proxy_handler = urllib2.ProxyHandler({'https': proxy, 'http': proxy}) opener = urllib2.build_opener(proxy_handler) request = urllib2.Request(url.encode('utf-8'), data=filedata) request.add_header(str('Content-Type'), str('application/zip')) request.get_method = lambda: str('PUT') exit_code = 0 try: url = opener.open(request) if url.getcode() == 200: log('Done uploading') else: raise Exception('Error uploading, expected status code 200, got status code: {0}'.format(url.getcode())) except Exception as e: log(traceback.format_exc()) exit_code = 1 filedata.close() f.close() sys.exit(exit_code)
def __init__(self, proxies=None): if proxies is None: proxies = getproxies() assert hasattr(proxies, "has_key"), "proxies must be a mapping" self.proxies = proxies for type, url in proxies.items(): setattr(self, "%s_open" % type, lambda r, proxy=url, type=type, meth=self.proxy_open: meth(r, proxy, type))
def get_soap_client(wsdlurl): # pragma: no cover (no tests for this function) """Get a SOAP client for performing requests.""" # this function isn't automatically tested because the functions using # it are not automatically tested try: from urllib import getproxies except ImportError: from urllib.request import getproxies # try suds first try: from suds.client import Client return Client(wsdlurl, proxy=getproxies()).service except ImportError: # fall back to using pysimplesoap from pysimplesoap.client import SoapClient return SoapClient(wsdl=wsdlurl, proxy=getproxies())
def request_connection(self): """If we are not behind a proxy, create the connection once and reuse it for all requests. If we are behind a proxy, we need to revert to HTTP 1.0 and use a separate connection for each request. """ # If we haven't done so, determine whether we're behind a proxy. if self.behind_proxy is None: import urllib proxies = urllib.getproxies() if "http" in proxies: self.behind_proxy = True self.proxy = proxies["http"] else: self.behind_proxy = False # Create a new connection or reuse an existing one. if self.behind_proxy: httplib.HTTPConnection._http_vsn = 10 httplib.HTTPConnection._http_vsn_str = "HTTP/1.0" if self.proxy is not None: self.con = httplib.HTTPConnection(self.proxy, self.port) else: # Testsuite has set self.behind_proxy to True to simulate # being behind a proxy. self.con = httplib.HTTPConnection(self.server, self.port) else: httplib.HTTPConnection._http_vsn = 11 httplib.HTTPConnection._http_vsn_str = "HTTP/1.1" if not self.con: self.con = httplib.HTTPConnection(self.server, self.port)
def GetDefaultProxyInfo(method='http'): """Get ProxyInfo from environment. This function is meant to mimic httplib2.proxy_info_from_environment, but get the proxy information from urllib.getproxies instead. urllib can also get proxy information from Windows Internet Explorer settings or MacOSX framework SystemConfiguration. Args: method: protocol string Returns: httplib2 ProxyInfo object or None """ proxy_dict = urllib.getproxies() proxy_url = proxy_dict.get(method, None) if not proxy_url: return None pi = httplib2.proxy_info_from_url(proxy_url, method) # The ProxyInfo object has a bypass_host method that takes the hostname as an # argument and it returns 1 or 0 based on if the hostname should bypass the # proxy or not. We could either build the bypassed hosts list and pass it to # pi.bypass_hosts, or we can just replace the method with the function in # urllib, and completely mimic urllib logic. We do the latter. # Since the urllib.proxy_bypass _function_ (no self arg) is not "bound" to the # class instance, it doesn't receive the self arg when its called. We don't # need to "bind" it via types.MethodType(urllib.proxy_bypass, pi). pi.bypass_host = urllib.proxy_bypass return pi
def _http_get(uri, silent=False): if PYTHON3: opener = urllib2.build_opener(urllib2.ProxyHandler(urllib.request.getproxies())) else: opener = urllib2.build_opener(urllib2.ProxyHandler(urllib.getproxies())) for repo in repos: if 'storage.jcloud.com' in repo: _uri = uri for p in ('/', 'dev', 'master', 'update', 'plugins'): _uri = _uri.lstrip(p).lstrip('/') url = repo + '/' + _uri else: url = repo + '/raw/' + uri try: resp = opener.open(urllib2.Request(url, headers=headers), timeout = 15) body = resp.read() try: f = StringIO(body) gz = gzip.GzipFile(fileobj = f) body = gz.read() except: pass except urllib2.HTTPError as e: if not silent: print('HTTP Error %s when fetching %s' % (e.code, url)) except urllib2.URLError as e: pass else: return body
def urlopen(url, proxies=None, data=None): """ Return connected request object for given url. All errors raise exceptions. """ global _opener if proxies is None: proxies = urllib.getproxies() headers = { 'User-Agent': UA_STR, 'Accept-Encoding' : 'gzip;q=1.0, deflate;q=0.9, identity;q=0.5', } request = urllib2.Request(url, data, headers) proxy_support = urllib2.ProxyHandler(proxies) if _opener is None: # XXX heh, not really protected :) pwd_manager = PasswordManager("WebCleaner", "imadoofus") handlers = [proxy_support, urllib2.UnknownHandler, HttpWithGzipHandler, urllib2.HTTPBasicAuthHandler(pwd_manager), urllib2.ProxyBasicAuthHandler(pwd_manager), urllib2.HTTPDigestAuthHandler(pwd_manager), urllib2.ProxyDigestAuthHandler(pwd_manager), urllib2.HTTPDefaultErrorHandler, urllib2.HTTPRedirectHandler, ] if hasattr(httplib, 'HTTPS'): handlers.append(HttpsWithGzipHandler) _opener = urllib2.build_opener(*handlers) # print _opener.handlers urllib2.install_opener(_opener) return _opener.open(request)
def __init__(self): self.proxies = {} for type, url in getproxies().items(): self.proxies[type] = self._get_proxy(url, type) if not self.proxies: raise NotConfigured
def http_proxy(self): """ Retrieves the operating system http proxy. First, the method scans the environment for variables named http_proxy, in case insensitive way. If both lowercase and uppercase environment variables exist (and disagree), lowercase is preferred. When the method cannot find such environment variables: - for Mac OS X, it will look for proxy information from Mac OS X System Configuration, - for Windows, it will look for proxy information from Windows Systems Registry. .. note:: There is a restriction when looking for proxy information from Mac OS X System Configuration or Windows Systems Registry: in these cases, the Toolkit does not support the use of proxies which require authentication (username and password). """ # Get the dictionary of scheme to proxy server URL mappings; for example: # {"http": "http://*****:*****@74.50.63.111:80", "https": "http://74.50.63.111:443"} # "getproxies" scans the environment for variables named <scheme>_proxy, in case insensitive way. # When it cannot find it, for Mac OS X it looks for proxy information from Mac OSX System Configuration, # and for Windows it looks for proxy information from Windows Systems Registry. # If both lowercase and uppercase environment variables exist (and disagree), lowercase is preferred. # Note the following restriction: "getproxies" does not support the use of proxies which # require authentication (user and password) when looking for proxy information from # Mac OSX System Configuration or Windows Systems Registry. system_proxies = urllib.getproxies() # Get the http proxy when it exists in the dictionary. proxy = system_proxies.get("http") if proxy: # Remove any spurious "http://" from the http proxy string. proxy = proxy.replace("http://", "", 1) return proxy
def using_http_proxy(url): """ Return True if the url will use HTTP proxy. Returns False otherwise. """ up = urlparse(url) return up.scheme.lower() in getproxies() and not proxy_bypass(up.netloc)
def fetch_proxies(self): prefs = self.prefs try: mode = prefs['proxy_mode'] except Exception: TRACE("Couldn't load proxy info from preferences") unhandled_exc_handler() return None if mode == NO_PROXY: return None if mode == AUTO_PROXY: info = urllib.getproxies() if info and 'http' in info: try: parsed = urlparse.urlparse(info['http']) _type, hostname, port = socks.PROXY_TYPE_HTTP, parsed.hostname, parsed.port except Exception: unhandled_exc_handler() return None elif info and 'socks' in info: _type = socks.PROXY_TYPE_SOCKS4 try: split = info['socks'].split('//')[1].split(':', 1) if len(split) == 1: hostname = split[0] port = 1080 else: hostname, port = split port = int(port) except (ValueError, IndexError): return None else: return None return ProxyInfo(_type, hostname, port) if prefs['proxy_type'] == SOCKS4: args = [socks.PROXY_TYPE_SOCKS4] elif prefs['proxy_type'] == SOCKS5: args = [socks.PROXY_TYPE_SOCKS5] else: args = [socks.PROXY_TYPE_HTTP] try: def tsiu(a): if type(a) is unicode: return a.encode('utf-8') return a args += [tsiu(prefs['proxy_server']), int(prefs['proxy_port'])] opt_args = {} if prefs['proxy_requires_auth']: opt_args['proxy_user'] = tsiu(prefs['proxy_username']) opt_args['proxy_pass'] = tsiu(prefs['proxy_password']) return ProxyInfo(*args, **opt_args) except Exception: unhandled_exc_handler() return None
def __init__(self): self._log = logging.getLogger(self.__class__.__name__) # Parent class will disable the middleware when no proxy # is configured in the mining node (raises NotConfigured exception) # Just copied the constructor code and removed the exception self.proxies = {} for key, url in getproxies().items(): self.proxies[key] = self._get_proxy(url, key)
def __init__(self, settings, crawler=None): super(HttpProxyMiddleware, self).__init__(settings) self.proxies = {} for type, url in getproxies().items(): self.proxies[type] = self._get_proxy(url, type) if not self.proxies: raise NotConfigured
def start_schedulers(options): try: from multiprocessing import Process except: sys.stderr.write('Sorry, -K only supported for Python 2.6+\n') return logging.getLogger().setLevel(options.log_level) apps = [ag.split(':') for ag in options.schedulers] if not options.with_scheduler and len(apps) == 1: app, code = get_code_for_scheduler(apps[0], options) if not app: return print('starting single-scheduler for "%s"...' % app) run(app, True, True, None, False, code, False, True) return # Work around OS X problem: http://bugs.python.org/issue9405 if PY2: import urllib else: import urllib.request as urllib urllib.getproxies() processes = [] for app_groups in apps: app, code = get_code_for_scheduler(app_groups, options) if not app: continue print('starting scheduler for "%s"...' % app) args = (app, True, True, None, False, code, False, True) p = Process(target=run, args=args) processes.append(p) print("Currently running %s scheduler processes" % (len(processes))) p.start() ##to avoid bashing the db at the same time time.sleep(0.7) print("Processes started") for p in processes: try: p.join() except (KeyboardInterrupt, SystemExit): print("Processes stopped") except: p.terminate() p.join()
def __init__ (self): """ Initialize the default options. """ super(Configuration, self).__init__() self['trace'] = False self["verbose"] = False self["complete"] = False self["warnings"] = True self["ignorewarnings"] = [] self['quiet'] = False self["anchors"] = False self["externlinks"] = [] self["internlinks"] = [] # on ftp, password is set by Pythons ftplib self["authentication"] = [] self["loginurl"] = None self["loginuserfield"] = "login" self["loginpasswordfield"] = "password" self["loginextrafields"] = {} self["proxy"] = urllib.getproxies() self["recursionlevel"] = -1 self["wait"] = 0 self['sendcookies'] = False self['storecookies'] = False self['cookiefile'] = None self["status"] = False self["status_wait_seconds"] = 5 self["fileoutput"] = [] self['output'] = 'text' self['logger'] = None self["warningregex"] = None self["warningregex_max"] = 5 self["warnsizebytes"] = None self["nntpserver"] = os.environ.get("NNTP_SERVER", None) self["threads"] = 100 # socket timeout in seconds self["timeout"] = 60 self["checkhtml"] = False self["checkcss"] = False self["scanvirus"] = False self["clamavconf"] = clamav.canonical_clamav_conf() self["useragent"] = UserAgent self["debugmemory"] = False self["localwebroot"] = None self["sslverify"] = True self["warnsslcertdaysvalid"] = 14 self["maxrunseconds"] = None self["maxnumurls"] = None self["maxconnectionshttp"] = 10 self["maxconnectionshttps"] = 10 self["maxconnectionsftp"] = 2 self.loggers = {} from ..logger import LoggerClasses for c in LoggerClasses: key = c.LoggerName self[key] = {} self.loggers[key] = c
def make_http_proxy(addr): import urllib proxies = urllib.getproxies() try: proxy_url = proxies["http"] except KeyError: return HTTPProxyServer(addr) url = urlparse(proxy_url) return HTTPProxy2ProxyServer(addr, (url.hostname, url.port or 8080))
def __init__(self, proxies = None): if proxies is None: proxies = getproxies() raise hasattr(proxies, 'has_key') or AssertionError('proxies must be a mapping') self.proxies = proxies for type, url in proxies.items(): setattr(self, '%s_open' % type, lambda r, proxy = url, type = type, meth = self.proxy_open: meth(r, proxy, type)) return
def __init__(self, proxies=None): if proxies is None: proxies = getproxies() assert hasattr(proxies, 'has_key'), "proxies must be a mapping" self.proxies = proxies for type, url in proxies.items(): setattr(self, '%s_open' % type, lambda r, proxy=url, type=type, meth=self.proxy_open: \ meth(r, proxy, type))
def __init__(self, proxies=None): if proxies is None: proxies = getproxies() self.proxies = proxies for type, url in proxies.items(): setattr(self, '%s_open' % type, lambda r, proxy=url, type=type, meth=self.proxy_open: meth( r, proxy, type))
def get_proxy(agent_config, use_system_settings=False): proxy_settings = {} # First we read the proxy configuration from agent.conf proxy_host = agent_config.get('proxy_host', None) if proxy_host is not None and not use_system_settings: proxy_settings['host'] = proxy_host try: proxy_settings['port'] = int(agent_config.get('proxy_port', 3128)) except ValueError: log.error('Proxy port must be an Integer. Defaulting it to 3128') proxy_settings['port'] = 3128 proxy_settings['user'] = agent_config.get('proxy_user', None) proxy_settings['password'] = agent_config.get('proxy_password', None) proxy_settings['system_settings'] = False log.debug("Proxy Settings: %s:%s@%s:%s" % (proxy_settings['user'], "*****", proxy_settings['host'], proxy_settings['port'])) return proxy_settings # If no proxy configuration was specified in agent.conf # We try to read it from the system settings try: import urllib proxies = urllib.getproxies() proxy = proxies.get('https', None) if proxy is not None: try: proxy = proxy.split('://')[1] except Exception: pass px = proxy.split(':') proxy_settings['host'] = px[0] proxy_settings['port'] = px[1] proxy_settings['user'] = None proxy_settings['password'] = None proxy_settings['system_settings'] = True if '@' in proxy_settings['host']: creds = proxy_settings['host'].split('@')[0].split(':') proxy_settings['user'] = creds[0] if len(creds) == 2: proxy_settings['password'] = creds[1] log.debug("Proxy Settings: %s:%s@%s:%s" % (proxy_settings['user'], "*****", proxy_settings['host'], proxy_settings['port'])) return proxy_settings except Exception as e: log.debug( "Error while trying to fetch proxy settings using urllib %s. Proxy is probably not set" % str(e)) log.debug("No proxy configured") return None
def test_get_parameters_passed_ok(self): #We mock the GET Method method for avoid calling network get_vars = {'var': 1, 'dos': 'tres', 'cuatro': 4} post_vars = {'var': 2, 'dos': 'gaticos', 'cuatro': 'monetes'} self.api.call('post_one', self.url_params, get_vars, post_vars) post_mock.assert_called_with("http://www.testingfakeurl.com/es/latest/user1/quickstart/", verify=False, headers=None, data=post_vars, params=get_vars, timeout=15, auth=None, proxies=urllib.getproxies())
def _create_client(server, credential, debug, api_type="products"): cfg = None if api_type in ('projectv2', 'artifact', 'repository', 'scanner', 'scan', 'scanall', 'preheat', 'quota', 'replication', 'robot', 'gc', 'retention'): cfg = v2_swagger_client.Configuration() else: cfg = swagger_client.Configuration() cfg.host = server.endpoint cfg.verify_ssl = server.verify_ssl # support basic auth only for now cfg.username = credential.username cfg.password = credential.password cfg.debug = debug proxies = getproxies() proxy = proxies.get('http', proxies.get('all', None)) if proxy: cfg.proxy = proxy if cfg.username is None and cfg.password is None: # returns {} for auth_settings for anonymous access import types cfg.auth_settings = types.MethodType(lambda self: {}, cfg) return { "chart": client.ChartRepositoryApi(client.ApiClient(cfg)), "products": swagger_client.ProductsApi(swagger_client.ApiClient(cfg)), "projectv2": v2_swagger_client.ProjectApi(v2_swagger_client.ApiClient(cfg)), "artifact": v2_swagger_client.ArtifactApi(v2_swagger_client.ApiClient(cfg)), "preheat": v2_swagger_client.PreheatApi(v2_swagger_client.ApiClient(cfg)), "quota": v2_swagger_client.QuotaApi(v2_swagger_client.ApiClient(cfg)), "repository": v2_swagger_client.RepositoryApi(v2_swagger_client.ApiClient(cfg)), "scan": v2_swagger_client.ScanApi(v2_swagger_client.ApiClient(cfg)), "scanall": v2_swagger_client.ScanAllApi(v2_swagger_client.ApiClient(cfg)), "scanner": v2_swagger_client.ScannerApi(v2_swagger_client.ApiClient(cfg)), "replication": v2_swagger_client.ReplicationApi(v2_swagger_client.ApiClient(cfg)), "robot": v2_swagger_client.RobotApi(v2_swagger_client.ApiClient(cfg)), "gc": v2_swagger_client.GcApi(v2_swagger_client.ApiClient(cfg)), "retention": v2_swagger_client.RetentionApi(v2_swagger_client.ApiClient(cfg)), }.get(api_type, 'Error: Wrong API type')
def sign_in(creds): r = requests.get('https://mavensmate.appspot.com/github', params={ 'username': creds['username'], 'password': creds['password'] }, proxies=urllib.getproxies(), verify=False) r.raise_for_status() return util.parse_rest_response(r.text)
def get_proxies(): proxies = getproxies() filtered_proxies = {} for key, value in proxies.items(): if key.startswith('http://'): if not value.startswith('http://'): filtered_proxies[key] = 'http://{0}'.format(value) else: filtered_proxies[key] = value return filtered_proxies
def get_proxies(): proxies = getproxies() #proxies为引入的getproxies(50行) filtered_proxies = {} #空字典 for key, value in proxies.items(): #遍历字典 if key.startswith('http'): #项的键以http开头 if not value.startswith('http'): #值的开头不为http filtered_proxies[key] = 'http://%s' % value #将值置入字符串,生成新字符串 else: filtered_proxies[key] = value #将值赋给键 return filtered_proxies #返回字典
def get_proxies(): proxies = getproxies() filtered_proxies = {} for key, value in proxies.items(): if key.startswith('http'): if not value.startswith('http'): filtered_proxies[key] = 'http://%s' % value else: filtered_proxies[key] = value return filtered_proxies
def get_soap_client( wsdlurl, timeout=30): # pragma: no cover (not part of normal test suite) """Get a SOAP client for performing requests. The client is cached. The timeout is in seconds.""" # this function isn't automatically tested because the functions using # it are not automatically tested if (wsdlurl, timeout) not in _soap_clients: # try zeep first try: from zeep.transports import Transport transport = Transport(timeout=timeout) from zeep import CachingClient client = CachingClient(wsdlurl, transport=transport).service except ImportError: # fall back to non-caching zeep client try: from zeep import Client client = Client(wsdlurl, transport=transport).service except ImportError: # other implementations require passing the proxy config try: from urllib import getproxies except ImportError: from urllib.request import getproxies # fall back to suds try: from suds.client import Client client = Client(wsdlurl, proxy=getproxies(), timeout=timeout).service except ImportError: # use pysimplesoap as last resort try: from pysimplesoap.client import SoapClient client = SoapClient(wsdl=wsdlurl, proxy=getproxies(), timeout=timeout) except ImportError: raise ImportError( 'No SOAP library (such as zeep) found') _soap_clients[(wsdlurl, timeout)] = client return _soap_clients[(wsdlurl, timeout)]
def _get_proxies(self): proxies = getproxies() proxy = {} if self.proxy: parsed_proxy = urlparse(self.proxy) proxy[parsed_proxy.scheme] = parsed_proxy.geturl() proxies.update(proxy) return proxies
def make_pypi_client(url): http_proxy_url = urllib.getproxies().get("http", "") if http_proxy_url: http_proxy_spec = urllib.splithost(urllib.splittype(http_proxy_url)[1])[0] transport = ProxiedTransport() transport.set_proxy(http_proxy_spec) else: transport = None return Server(url, transport=transport)
def fix_proxy_env(self): import urllib proxy_settings = urllib.getproxies() for ptype in proxy_settings: proxy = proxy_settings[ptype] if '://' not in proxy: proxy = '%s://%s' % (ptype, proxy) protocol, val = proxy.split('://') proxy = '%s://%s@%s' % (protocol, self['proxy_auth'], val) os.environ['%s_proxy' % ptype] = proxy
def _get_client(): # pragma: no cover (no tests for this function) """Get a SOAP client for performing VIES requests.""" # this function isn't automatically tested because the functions using # it are not automatically tested global _vies_client if not _vies_client: try: from urllib import getproxies except ImportError: from urllib.request import getproxies # try suds first try: from suds.client import Client _vies_client = Client(vies_wsdl, proxy=getproxies()).service except ImportError: # fall back to using pysimplesoap from pysimplesoap.client import SoapClient _vies_client = SoapClient(wsdl=vies_wsdl, proxy=getproxies()) return _vies_client
def __init__(self): """ Initialize the default options. """ super(Configuration, self).__init__() ## checking options self["allowedschemes"] = [] self['cookiefile'] = None self['robotstxt'] = True self["debugmemory"] = False self["localwebroot"] = None self["maxfilesizeparse"] = 1 * 1024 * 1024 self["maxfilesizedownload"] = 5 * 1024 * 1024 self["maxnumurls"] = None self["maxrunseconds"] = None self["maxrequestspersecond"] = 10 self["maxhttpredirects"] = 10 self["nntpserver"] = os.environ.get("NNTP_SERVER", None) self["proxy"] = urllib.getproxies() self["sslverify"] = True self["threads"] = 10 self["timeout"] = 60 self["aborttimeout"] = 300 self["recursionlevel"] = -1 self["useragent"] = UserAgent ## authentication self["authentication"] = [] self["loginurl"] = None self["loginuserfield"] = "login" self["loginpasswordfield"] = "password" self["loginextrafields"] = {} ## filtering self["externlinks"] = [] self["ignorewarnings"] = [] self["internlinks"] = [] self["checkextern"] = False ## plugins self["pluginfolders"] = get_plugin_folders() self["enabledplugins"] = [] ## output self['trace'] = False self['quiet'] = False self["verbose"] = False self["warnings"] = True self["fileoutput"] = [] self['output'] = 'text' self["status"] = False self["status_wait_seconds"] = 5 self['logger'] = None self.loggers = {} from ..logger import LoggerClasses for c in LoggerClasses: key = c.LoggerName self[key] = {} self.loggers[key] = c
def make_pypi_client(url): http_proxy_url = urllib.getproxies().get("http", "") if http_proxy_url: http_proxy_spec = urllib.splithost( urllib.splittype(http_proxy_url)[1])[0] transport = ProxiedTransport() transport.set_proxy(http_proxy_spec) else: transport = None return ServerProxy(url, transport=transport)
def proxies(self): """ optional field, might not exist """ try: proxies = self.get_conf("proxies") # If there is proxies section, but empty, it will try to use system proxy if not proxies: return urllib.getproxies() return dict(proxies) except: return None
def get_proxies(self): """avproxy - instance of the AVProxy class""" proxy_url = self.get_smart_proxy_url() if proxy_url: return { 'http': 'http://{}'.format(proxy_url), 'https': 'http://{}'.format(proxy_url), } # This helper function returns a dictionary of scheme to proxy server URL mappings. # It scans the environment for variables named <scheme>_proxy return urllib.getproxies()
def _get_http_proxy(): """Returns an HTTP proxy URL formatted for consumption by SOAPpy. SOAPpy does some fairly low-level HTTP manipulation and needs to be explicitly made aware of HTTP proxy URLs, which also have to be formatted without a schema or path. """ http_proxy = urllib.getproxies().get('http') if http_proxy is None: return None return urlparse.urlparse(http_proxy).netloc
def get_lyrics(artist, song): functions = [get_lyrics_genius, get_lyrics_songlyrics] lyrics = "" proxy = urllib.getproxies() for f in functions: print("bla") lyrics = f(artist, song, proxy) if lyrics != "": break return lyrics.encode('utf-8')
def becomeDaemon(self): """Code below comes from the excellent recipe by Chad J. Schroeder. """ # Workaround for http://bugs.python.org/issue9405 on Mac OS X from platform import system if system() == 'Darwin': from urllib import getproxies getproxies() try: pid = os.fork() except OSError as e: raise Exception("%s [%d]" % (e.strerror, e.errno)) if pid == 0: # The first child. os.setsid() try: pid = os.fork() # Fork a second child. except OSError as e: raise Exception("%s [%d]" % (e.strerror, e.errno)) if pid == 0: # The second child. os.chdir(WORKDIR) os.umask(UMASK) else: # Exit parent (the first child) of the second child. os._exit(0) else: os._exit(0) # Exit parent of the first child. # Iterate through and close all stdin/out/err for fd in range(0, MAXFD): try: os.close(fd) except OSError: # ERROR, fd wasn't open to begin with (ignored) pass os.open(REDIRECT_TO, os.O_RDWR) # standard input (0) # Duplicate standard input to standard output and standard error. os.dup2(0, 1) # standard output (1) os.dup2(0, 2) # standard error (2)
def start_schedulers(options): from multiprocessing import Process apps = [ag.split(':') for ag in options.schedulers] if not options.with_scheduler and len(apps) == 1: app, code = get_code_for_scheduler(options.folder, apps[0]) if not app: return print('starting single-scheduler for "%s"...' % app) run(app, True, True, None, False, code, False, True) return # Work around OS X problem: http://bugs.python.org/issue9405 if PY2: import urllib else: import urllib.request as urllib urllib.getproxies() processes = [] for app_groups in apps: app, code = get_code_for_scheduler(options.folder, app_groups) if not app: continue print('starting scheduler for "%s"...' % app) args = (app, True, True, None, False, code, False, True) p = Process(target=run, args=args) processes.append(p) print("Currently running %s scheduler processes" % (len(processes))) p.start() ##to avoid bashing the db at the same time time.sleep(0.7) print("Processes started") for p in processes: try: p.join() except (KeyboardInterrupt, SystemExit): print("Processes stopped") except: p.terminate() p.join()
def __init__(self, url=None): self._use_datetime = 0 # determine protocol type self.__type, rest = urllib.splittype(url) # find out proxy environment p = get_proxy(urllib.getproxies(), self.__type) if p: opener = urllib2.build_opener(ConnectHTTPHandler(proxy=p), ConnectHTTPSHandler(proxy=p)) urllib2.install_opener(opener)
def __init__(self): self.session = requests.session() self.session.proxies = urllib.getproxies() self.headers = { "Accept": "*/*", "Accept-Encoding": "gzip, deflate", "Accept-Language": "en;q=1, fr;q=0.9, de;q=0.8, ja;q=0.7, nl;q=0.6, it;q=0.5", "Content-Type": "application/x-www-form-urlencoded; charset=utf-8", "X-Robinhood-API-Version": "1.0.0", "Connection": "keep-alive", "User-Agent": "Robinhood/823 (iPhone; iOS 9.1.2; Scale/2.00)" } self.session.headers = self.headers
def get_proxy(): proxy_settings = config.get('proxy', {}) # if nothing was set, use OS-level proxies if not proxy_settings or \ not (proxy_settings.get('http') or proxy_settings.get('https')): proxy_settings = getproxies() # allow anything local... if proxy_settings: set_no_proxy_settings(proxy_settings) return proxy_settings
def _getClient(cls, **options): """Establish a connection and get a `suds.Client` For internal purposes, only """ from suds.client import Client try: from urllib import getproxies except ImportError: from urllib.request import getproxies return Client(cls.GSIS_HOST + cls.GSIS_WSDL, proxy=getproxies(), **options)
def get_lyrics(artist, song): functions = [get_lyrics_genius, get_lyrics_songlyrics] lyrics = "" proxy = urllib.getproxies() for f in functions: print "Searching for %s - %s" % (artist.encode('utf-8'), song.encode('utf-8')) lyrics = f(artist, song, proxy) if lyrics != "": break return lyrics.encode('utf-8')
def analyze_sentiment(text): url = "http://text-processing.com/api/sentiment/" proxy = urllib.getproxies() data_text = requests.post(url, proxies=proxy, data={'text': text}).text # probabilities are in data["probability"][label] # label can be "neg", "neutral" or "pos" try: data = json.loads(data_text) #return data["label"] return data["probability"]["pos"] except ValueError: return None
def get_proxies(): proxies = getproxies() filtered_proxies = {} ## Iterate key-value pair over dictionary ## for key in d: just iterates keys ## for key, value in d.items: key value pair ## http://stackoverflow.com/questions/3294889/iterating-over-dictionaries-using-for-loops-in-python for key, value in proxies.items(): if key.startswith('http'): if not value.startswith('http'): filtered_proxies[key] = 'http://%s' % value else: filtered_proxies[key] = value return filtered_proxies
def get_proxies(): import urllib proxies = urllib.getproxies() result = {} if 'https' not in proxies and \ 'http' in proxies: url = proxies['http'].replace("http://", "https://") result['https'] = papyon.Proxy(url) for type, url in proxies.items(): if type == 'no': continue if type == 'https' and url.startswith('http://'): url = url.replace('http://', 'https://', 1) result[type] = papyon.Proxy(url) return result
def check_vies(number): # pragma: no cover (no tests for this function) """Queries the online European Commission VAT Information Exchange System (VIES) for validity of the provided number. Note that the service has usage limitations (see the VIES website for details). This returns a dict-like object.""" # this function isn't automatically tested because it would require # network access for the tests and unnecessarily load the VIES website number = compact(number) global _vies_client if not _vies_client: from suds.client import Client from urllib import getproxies _vies_client = Client(vies_wsdl, proxy=getproxies()) return _vies_client.service.checkVat(number[:2], number[2:])
def _create_client(server, credential, debug): cfg = swagger_client.Configuration() cfg.host = server.endpoint cfg.verify_ssl = server.verify_ssl # support basic auth only for now cfg.username = credential.username cfg.password = credential.password cfg.debug = debug proxies = getproxies() proxy = proxies.get('http', proxies.get('all', None)) if proxy: cfg.proxy = proxy return swagger_client.ProductsApi(swagger_client.ApiClient(cfg))