def __get_proxies(self):
try:
mm_http_proxy = config.connection.get_plugin_client_setting('mm_http_proxy', None)
mm_http_proxy = mm_http_proxy.replace("\\","")
mm_https_proxy = config.connection.get_plugin_client_setting('mm_https_proxy', None)
mm_https_proxy = mm_https_proxy.replace("\\","")
if mm_https_proxy == None and mm_http_proxy != None:
mm_https_proxy = mm_http_proxy
if mm_http_proxy != None and mm_https_proxy != None:
return {
"http": mm_http_proxy,
"https": mm_https_proxy
}
elif mm_http_proxy != None:
return {
"http": mm_http_proxy
}
elif mm_https_proxy != None:
return {
"https": mm_https_proxy
}
else:
return urllib.getproxies()
except:
return urllib.getproxies()
def getAutoProxy(self):
"""Fetch the proxy from the the system environment variables
"""
if urllib.getproxies().has_key('http'):
return urllib.getproxies()['http']
else:
return ""
def start_schedulers(options):
try:
from multiprocessing import Process
except:
sys.stderr.write('Sorry, -K only supported for python 2.6-2.7\n')
return
processes = []
apps = [(app.strip(), None) for app in options.scheduler.split(',')]
if options.scheduler_groups:
apps = options.scheduler_groups
code = "from gluon.globals import current;current._scheduler.loop()"
logging.getLogger().setLevel(options.debuglevel)
if options.folder:
os.chdir(options.folder)
if len(apps) == 1 and not options.with_scheduler:
app_, code = get_code_for_scheduler(apps[0], options)
if not app_:
return
print('starting single-scheduler for "%s"...' % app_)
run(app_, True, True, None, False, code)
return
# Work around OS X problem: http://bugs.python.org/issue9405
if PY2:
import urllib
else:
import urllib.request as urllib
urllib.getproxies()
for app in apps:
app_, code = get_code_for_scheduler(app, options)
if not app_:
continue
print('starting scheduler for "%s"...' % app_)
args = (app_, True, True, None, False, code)
p = Process(target=run, args=args)
processes.append(p)
print("Currently running %s scheduler processes" % (len(processes)))
p.start()
##to avoid bashing the db at the same time
time.sleep(0.7)
print("Processes started")
for p in processes:
try:
p.join()
except (KeyboardInterrupt, SystemExit):
print("Processes stopped")
except:
p.terminate()
p.join()
def connect(self):
#- Parse proxies
pd = urllib.getproxies().get('http', None)
if pd is None:
sockstype = ''
else:
sockstype, user, password, hostport = urllib2._parse_proxy(pd)
if 'socks' not in sockstype:
return httplib.HTTPConnection.connect(self)
assert ':' in hostport # if we don't have a port we're screwed
host, port = hostport.rsplit(':', 1)
port = int(port)
for res in socket.getaddrinfo(self.host, self.port, 0, socket.SOCK_STREAM):
af, socktype, proto, canonname, sa = res
try:
self.sock = self._sockettype(af, socktype, proto)
self.sock.setproxy(proxytype=getattr(socks, 'PROXY_TYPE_%s' % sockstype.upper()), addr=host, port=port, rdns=False, username=user, password=password)
#- The rest is the same as superclass
if self.debuglevel > 0:
print "connect: (%s, %s)" % (self.host, self.port)
self.sock.connect(sa)
except socket.error, msg:
if self.debuglevel > 0:
print 'connect fail:', (self.host, self.port)
if self.sock:
self.sock.close()
self.sock = None
continue
break
def get_environ_proxies(netloc):
"""Return a dict of environment proxies."""
get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper())
# First check whether no_proxy is defined. If it is, check that the URL
# we're getting isn't in the no_proxy list.
no_proxy = get_proxy('no_proxy')
if no_proxy:
# We need to check whether we match here. We need to see if we match
# the end of the netloc, both with and without the port.
no_proxy = no_proxy.replace(' ', '').split(',')
for host in no_proxy:
if netloc.endswith(host) or netloc.split(':')[0].endswith(host):
# The URL does match something in no_proxy, so we don't want
# to apply the proxies on this URL.
return {}
# If the system proxy settings indicate that this URL should be bypassed,
# don't proxy.
if proxy_bypass(netloc):
return {}
# If we get here, we either didn't have no_proxy set or we're not going
# anywhere that no_proxy applies to, and the system settings don't require
# bypassing the proxy for the current URL.
return getproxies()
def __init__(self, radiodns_services):
self._radiodns_services = radiodns_services
self._dns = DnsResolver()
self._proxy_settings = None
self._use_http_proxy = False
self._radiovis_client = None
self._http_client = HttpClientThread(self)
self._http_client.start()
self._listeners = []
# Get system proxy server settings (from http_proxy environment variable
# or web browser settings).
proxies = urllib.getproxies()
if "http" in proxies:
http_proxy = urlparse.urlparse(proxies['http'])
self._proxy_settings = ProxySettings(proxy_type = socks.PROXY_TYPE_HTTP,
host = http_proxy.hostname,
port = http_proxy.port)
self.log("HTTP proxy: " + http_proxy.hostname +
", port " + str(http_proxy.port))
else:
self._proxy_settings = None
def urlopen(url, proxies=None, data=None):
global _opener
if not proxies:
proxies = urllib.getproxies()
headers = {'User-Agent': UA_STR,
'Accept-Encoding' : 'gzip;q=1.0, deflate;q=0.9, identity;q=0.5'}
req = urllib2.Request(url, data, headers)
proxy_support = urllib2.ProxyHandler(proxies)
if _opener is None:
pwd_manager = handlepasswd()
handlers = [proxy_support,
urllib2.UnknownHandler, HttpWithGzipHandler,
urllib2.HTTPBasicAuthHandler(pwd_manager),
urllib2.ProxyBasicAuthHandler(pwd_manager),
urllib2.HTTPDigestAuthHandler(pwd_manager),
urllib2.ProxyDigestAuthHandler(pwd_manager),
urllib2.HTTPDefaultErrorHandler, urllib2.HTTPRedirectHandler,
]
if hasattr(httplib, 'HTTPS'):
handlers.append(HttpsWithGzipHandler)
_opener = urllib2.build_opener(*handlers)
# print _opener.handlers
urllib2.install_opener(_opener)
return _opener.open(req)
def do_upload_and_exit(path, url, proxy):
f = open(path, 'rb')
# mmap the file to reduce the amount of memory required (see bit.ly/2aNENXC)
filedata = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
# Get proxies from environment/system
proxy_handler = urllib2.ProxyHandler(urllib.getproxies())
if proxy != "":
# unless a proxy is explicitly passed, then use that instead
proxy_handler = urllib2.ProxyHandler({'https': proxy, 'http': proxy})
opener = urllib2.build_opener(proxy_handler)
request = urllib2.Request(url.encode('utf-8'), data=filedata)
request.add_header(str('Content-Type'), str('application/zip'))
request.get_method = lambda: str('PUT')
exit_code = 0
try:
url = opener.open(request)
if url.getcode() == 200:
log('Done uploading')
else:
raise Exception('Error uploading, expected status code 200, got status code: {0}'.format(url.getcode()))
except Exception as e:
log(traceback.format_exc())
exit_code = 1
filedata.close()
f.close()
sys.exit(exit_code)
def __init__(self, proxies=None):
if proxies is None:
proxies = getproxies()
assert hasattr(proxies, "has_key"), "proxies must be a mapping"
self.proxies = proxies
for type, url in proxies.items():
setattr(self, "%s_open" % type, lambda r, proxy=url, type=type, meth=self.proxy_open: meth(r, proxy, type))
def get_soap_client(wsdlurl): # pragma: no cover (no tests for this function)
"""Get a SOAP client for performing requests."""
# this function isn't automatically tested because the functions using
# it are not automatically tested
try:
from urllib import getproxies
except ImportError:
from urllib.request import getproxies
# try suds first
try:
from suds.client import Client
return Client(wsdlurl, proxy=getproxies()).service
except ImportError:
# fall back to using pysimplesoap
from pysimplesoap.client import SoapClient
return SoapClient(wsdl=wsdlurl, proxy=getproxies())
def request_connection(self):
"""If we are not behind a proxy, create the connection once and reuse
it for all requests. If we are behind a proxy, we need to revert to
HTTP 1.0 and use a separate connection for each request.
"""
# If we haven't done so, determine whether we're behind a proxy.
if self.behind_proxy is None:
import urllib
proxies = urllib.getproxies()
if "http" in proxies:
self.behind_proxy = True
self.proxy = proxies["http"]
else:
self.behind_proxy = False
# Create a new connection or reuse an existing one.
if self.behind_proxy:
httplib.HTTPConnection._http_vsn = 10
httplib.HTTPConnection._http_vsn_str = "HTTP/1.0"
if self.proxy is not None:
self.con = httplib.HTTPConnection(self.proxy, self.port)
else: # Testsuite has set self.behind_proxy to True to simulate
# being behind a proxy.
self.con = httplib.HTTPConnection(self.server, self.port)
else:
httplib.HTTPConnection._http_vsn = 11
httplib.HTTPConnection._http_vsn_str = "HTTP/1.1"
if not self.con:
self.con = httplib.HTTPConnection(self.server, self.port)
def GetDefaultProxyInfo(method='http'):
"""Get ProxyInfo from environment.
This function is meant to mimic httplib2.proxy_info_from_environment, but get
the proxy information from urllib.getproxies instead. urllib can also get
proxy information from Windows Internet Explorer settings or MacOSX framework
SystemConfiguration.
Args:
method: protocol string
Returns:
httplib2 ProxyInfo object or None
"""
proxy_dict = urllib.getproxies()
proxy_url = proxy_dict.get(method, None)
if not proxy_url:
return None
pi = httplib2.proxy_info_from_url(proxy_url, method)
# The ProxyInfo object has a bypass_host method that takes the hostname as an
# argument and it returns 1 or 0 based on if the hostname should bypass the
# proxy or not. We could either build the bypassed hosts list and pass it to
# pi.bypass_hosts, or we can just replace the method with the function in
# urllib, and completely mimic urllib logic. We do the latter.
# Since the urllib.proxy_bypass _function_ (no self arg) is not "bound" to the
# class instance, it doesn't receive the self arg when its called. We don't
# need to "bind" it via types.MethodType(urllib.proxy_bypass, pi).
pi.bypass_host = urllib.proxy_bypass
return pi
def _http_get(uri, silent=False):
if PYTHON3:
opener = urllib2.build_opener(urllib2.ProxyHandler(urllib.request.getproxies()))
else:
opener = urllib2.build_opener(urllib2.ProxyHandler(urllib.getproxies()))
for repo in repos:
if 'storage.jcloud.com' in repo:
_uri = uri
for p in ('/', 'dev', 'master', 'update', 'plugins'):
_uri = _uri.lstrip(p).lstrip('/')
url = repo + '/' + _uri
else:
url = repo + '/raw/' + uri
try:
resp = opener.open(urllib2.Request(url, headers=headers), timeout = 15)
body = resp.read()
try:
f = StringIO(body)
gz = gzip.GzipFile(fileobj = f)
body = gz.read()
except:
pass
except urllib2.HTTPError as e:
if not silent:
print('HTTP Error %s when fetching %s' % (e.code, url))
except urllib2.URLError as e:
pass
else:
return body
def urlopen(url, proxies=None, data=None):
"""
Return connected request object for given url.
All errors raise exceptions.
"""
global _opener
if proxies is None:
proxies = urllib.getproxies()
headers = {
'User-Agent': UA_STR,
'Accept-Encoding' : 'gzip;q=1.0, deflate;q=0.9, identity;q=0.5',
}
request = urllib2.Request(url, data, headers)
proxy_support = urllib2.ProxyHandler(proxies)
if _opener is None:
# XXX heh, not really protected :)
pwd_manager = PasswordManager("WebCleaner", "imadoofus")
handlers = [proxy_support,
urllib2.UnknownHandler,
HttpWithGzipHandler,
urllib2.HTTPBasicAuthHandler(pwd_manager),
urllib2.ProxyBasicAuthHandler(pwd_manager),
urllib2.HTTPDigestAuthHandler(pwd_manager),
urllib2.ProxyDigestAuthHandler(pwd_manager),
urllib2.HTTPDefaultErrorHandler,
urllib2.HTTPRedirectHandler,
]
if hasattr(httplib, 'HTTPS'):
handlers.append(HttpsWithGzipHandler)
_opener = urllib2.build_opener(*handlers)
# print _opener.handlers
urllib2.install_opener(_opener)
return _opener.open(request)
def __init__(self):
self.proxies = {}
for type, url in getproxies().items():
self.proxies[type] = self._get_proxy(url, type)
if not self.proxies:
raise NotConfigured
def http_proxy(self):
"""
Retrieves the operating system http proxy.
First, the method scans the environment for variables named http_proxy, in case insensitive way.
If both lowercase and uppercase environment variables exist (and disagree), lowercase is preferred.
When the method cannot find such environment variables:
- for Mac OS X, it will look for proxy information from Mac OS X System Configuration,
- for Windows, it will look for proxy information from Windows Systems Registry.
.. note:: There is a restriction when looking for proxy information from
Mac OS X System Configuration or Windows Systems Registry:
in these cases, the Toolkit does not support the use of proxies
which require authentication (username and password).
"""
# Get the dictionary of scheme to proxy server URL mappings; for example:
# {"http": "http://*****:*****@74.50.63.111:80", "https": "http://74.50.63.111:443"}
# "getproxies" scans the environment for variables named <scheme>_proxy, in case insensitive way.
# When it cannot find it, for Mac OS X it looks for proxy information from Mac OSX System Configuration,
# and for Windows it looks for proxy information from Windows Systems Registry.
# If both lowercase and uppercase environment variables exist (and disagree), lowercase is preferred.
# Note the following restriction: "getproxies" does not support the use of proxies which
# require authentication (user and password) when looking for proxy information from
# Mac OSX System Configuration or Windows Systems Registry.
system_proxies = urllib.getproxies()
# Get the http proxy when it exists in the dictionary.
proxy = system_proxies.get("http")
if proxy:
# Remove any spurious "http://" from the http proxy string.
proxy = proxy.replace("http://", "", 1)
return proxy
def using_http_proxy(url):
"""
Return True if the url will use HTTP proxy.
Returns False otherwise.
"""
up = urlparse(url)
return up.scheme.lower() in getproxies() and not proxy_bypass(up.netloc)
def fetch_proxies(self):
prefs = self.prefs
try:
mode = prefs['proxy_mode']
except Exception:
TRACE("Couldn't load proxy info from preferences")
unhandled_exc_handler()
return None
if mode == NO_PROXY:
return None
if mode == AUTO_PROXY:
info = urllib.getproxies()
if info and 'http' in info:
try:
parsed = urlparse.urlparse(info['http'])
_type, hostname, port = socks.PROXY_TYPE_HTTP, parsed.hostname, parsed.port
except Exception:
unhandled_exc_handler()
return None
elif info and 'socks' in info:
_type = socks.PROXY_TYPE_SOCKS4
try:
split = info['socks'].split('//')[1].split(':', 1)
if len(split) == 1:
hostname = split[0]
port = 1080
else:
hostname, port = split
port = int(port)
except (ValueError, IndexError):
return None
else:
return None
return ProxyInfo(_type, hostname, port)
if prefs['proxy_type'] == SOCKS4:
args = [socks.PROXY_TYPE_SOCKS4]
elif prefs['proxy_type'] == SOCKS5:
args = [socks.PROXY_TYPE_SOCKS5]
else:
args = [socks.PROXY_TYPE_HTTP]
try:
def tsiu(a):
if type(a) is unicode:
return a.encode('utf-8')
return a
args += [tsiu(prefs['proxy_server']), int(prefs['proxy_port'])]
opt_args = {}
if prefs['proxy_requires_auth']:
opt_args['proxy_user'] = tsiu(prefs['proxy_username'])
opt_args['proxy_pass'] = tsiu(prefs['proxy_password'])
return ProxyInfo(*args, **opt_args)
except Exception:
unhandled_exc_handler()
return None
def __init__(self):
self._log = logging.getLogger(self.__class__.__name__)
# Parent class will disable the middleware when no proxy
# is configured in the mining node (raises NotConfigured exception)
# Just copied the constructor code and removed the exception
self.proxies = {}
for key, url in getproxies().items():
self.proxies[key] = self._get_proxy(url, key)
def __init__(self, settings, crawler=None):
super(HttpProxyMiddleware, self).__init__(settings)
self.proxies = {}
for type, url in getproxies().items():
self.proxies[type] = self._get_proxy(url, type)
if not self.proxies:
raise NotConfigured
def start_schedulers(options):
try:
from multiprocessing import Process
except:
sys.stderr.write('Sorry, -K only supported for Python 2.6+\n')
return
logging.getLogger().setLevel(options.log_level)
apps = [ag.split(':') for ag in options.schedulers]
if not options.with_scheduler and len(apps) == 1:
app, code = get_code_for_scheduler(apps[0], options)
if not app:
return
print('starting single-scheduler for "%s"...' % app)
run(app, True, True, None, False, code, False, True)
return
# Work around OS X problem: http://bugs.python.org/issue9405
if PY2:
import urllib
else:
import urllib.request as urllib
urllib.getproxies()
processes = []
for app_groups in apps:
app, code = get_code_for_scheduler(app_groups, options)
if not app:
continue
print('starting scheduler for "%s"...' % app)
args = (app, True, True, None, False, code, False, True)
p = Process(target=run, args=args)
processes.append(p)
print("Currently running %s scheduler processes" % (len(processes)))
p.start()
##to avoid bashing the db at the same time
time.sleep(0.7)
print("Processes started")
for p in processes:
try:
p.join()
except (KeyboardInterrupt, SystemExit):
print("Processes stopped")
except:
p.terminate()
p.join()
def __init__ (self):
"""
Initialize the default options.
"""
super(Configuration, self).__init__()
self['trace'] = False
self["verbose"] = False
self["complete"] = False
self["warnings"] = True
self["ignorewarnings"] = []
self['quiet'] = False
self["anchors"] = False
self["externlinks"] = []
self["internlinks"] = []
# on ftp, password is set by Pythons ftplib
self["authentication"] = []
self["loginurl"] = None
self["loginuserfield"] = "login"
self["loginpasswordfield"] = "password"
self["loginextrafields"] = {}
self["proxy"] = urllib.getproxies()
self["recursionlevel"] = -1
self["wait"] = 0
self['sendcookies'] = False
self['storecookies'] = False
self['cookiefile'] = None
self["status"] = False
self["status_wait_seconds"] = 5
self["fileoutput"] = []
self['output'] = 'text'
self['logger'] = None
self["warningregex"] = None
self["warningregex_max"] = 5
self["warnsizebytes"] = None
self["nntpserver"] = os.environ.get("NNTP_SERVER", None)
self["threads"] = 100
# socket timeout in seconds
self["timeout"] = 60
self["checkhtml"] = False
self["checkcss"] = False
self["scanvirus"] = False
self["clamavconf"] = clamav.canonical_clamav_conf()
self["useragent"] = UserAgent
self["debugmemory"] = False
self["localwebroot"] = None
self["sslverify"] = True
self["warnsslcertdaysvalid"] = 14
self["maxrunseconds"] = None
self["maxnumurls"] = None
self["maxconnectionshttp"] = 10
self["maxconnectionshttps"] = 10
self["maxconnectionsftp"] = 2
self.loggers = {}
from ..logger import LoggerClasses
for c in LoggerClasses:
key = c.LoggerName
self[key] = {}
self.loggers[key] = c
def make_http_proxy(addr):
import urllib
proxies = urllib.getproxies()
try:
proxy_url = proxies["http"]
except KeyError:
return HTTPProxyServer(addr)
url = urlparse(proxy_url)
return HTTPProxy2ProxyServer(addr, (url.hostname, url.port or 8080))
def __init__(self, proxies = None):
if proxies is None:
proxies = getproxies()
raise hasattr(proxies, 'has_key') or AssertionError('proxies must be a mapping')
self.proxies = proxies
for type, url in proxies.items():
setattr(self, '%s_open' % type, lambda r, proxy = url, type = type, meth = self.proxy_open: meth(r, proxy, type))
return
def __init__(self, proxies=None):
if proxies is None:
proxies = getproxies()
assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
self.proxies = proxies
for type, url in proxies.items():
setattr(self, '%s_open' % type,
lambda r, proxy=url, type=type, meth=self.proxy_open: \
meth(r, proxy, type))
def __init__(self, proxies=None):
if proxies is None:
proxies = getproxies()
self.proxies = proxies
for type, url in proxies.items():
setattr(self,
'%s_open' % type,
lambda r, proxy=url, type=type, meth=self.proxy_open: meth(
r, proxy, type))
def get_proxy(agent_config, use_system_settings=False):
proxy_settings = {}
# First we read the proxy configuration from agent.conf
proxy_host = agent_config.get('proxy_host', None)
if proxy_host is not None and not use_system_settings:
proxy_settings['host'] = proxy_host
try:
proxy_settings['port'] = int(agent_config.get('proxy_port', 3128))
except ValueError:
log.error('Proxy port must be an Integer. Defaulting it to 3128')
proxy_settings['port'] = 3128
proxy_settings['user'] = agent_config.get('proxy_user', None)
proxy_settings['password'] = agent_config.get('proxy_password', None)
proxy_settings['system_settings'] = False
log.debug("Proxy Settings: %s:%s@%s:%s" %
(proxy_settings['user'], "*****", proxy_settings['host'],
proxy_settings['port']))
return proxy_settings
# If no proxy configuration was specified in agent.conf
# We try to read it from the system settings
try:
import urllib
proxies = urllib.getproxies()
proxy = proxies.get('https', None)
if proxy is not None:
try:
proxy = proxy.split('://')[1]
except Exception:
pass
px = proxy.split(':')
proxy_settings['host'] = px[0]
proxy_settings['port'] = px[1]
proxy_settings['user'] = None
proxy_settings['password'] = None
proxy_settings['system_settings'] = True
if '@' in proxy_settings['host']:
creds = proxy_settings['host'].split('@')[0].split(':')
proxy_settings['user'] = creds[0]
if len(creds) == 2:
proxy_settings['password'] = creds[1]
log.debug("Proxy Settings: %s:%s@%s:%s" %
(proxy_settings['user'], "*****", proxy_settings['host'],
proxy_settings['port']))
return proxy_settings
except Exception as e:
log.debug(
"Error while trying to fetch proxy settings using urllib %s. Proxy is probably not set"
% str(e))
log.debug("No proxy configured")
return None
def make_http_proxy(addr):
import urllib
proxies = urllib.getproxies()
try:
proxy_url = proxies["http"]
except KeyError:
return HTTPProxyServer(addr)
url = urlparse(proxy_url)
return HTTPProxy2ProxyServer(addr, (url.hostname, url.port or 8080))
def test_get_parameters_passed_ok(self):
#We mock the GET Method method for avoid calling network
get_vars = {'var': 1, 'dos': 'tres', 'cuatro': 4}
post_vars = {'var': 2, 'dos': 'gaticos', 'cuatro': 'monetes'}
self.api.call('post_one', self.url_params, get_vars, post_vars)
post_mock.assert_called_with("http://www.testingfakeurl.com/es/latest/user1/quickstart/",
verify=False, headers=None, data=post_vars, params=get_vars,
timeout=15, auth=None,
proxies=urllib.getproxies())
def __init__(self, proxies = None):
if proxies is None:
proxies = getproxies()
raise hasattr(proxies, 'has_key') or AssertionError('proxies must be a mapping')
self.proxies = proxies
for type, url in proxies.items():
setattr(self, '%s_open' % type, lambda r, proxy = url, type = type, meth = self.proxy_open: meth(r, proxy, type))
return
def _create_client(server, credential, debug, api_type="products"):
cfg = None
if api_type in ('projectv2', 'artifact', 'repository', 'scanner', 'scan',
'scanall', 'preheat', 'quota', 'replication', 'robot',
'gc', 'retention'):
cfg = v2_swagger_client.Configuration()
else:
cfg = swagger_client.Configuration()
cfg.host = server.endpoint
cfg.verify_ssl = server.verify_ssl
# support basic auth only for now
cfg.username = credential.username
cfg.password = credential.password
cfg.debug = debug
proxies = getproxies()
proxy = proxies.get('http', proxies.get('all', None))
if proxy:
cfg.proxy = proxy
if cfg.username is None and cfg.password is None:
# returns {} for auth_settings for anonymous access
import types
cfg.auth_settings = types.MethodType(lambda self: {}, cfg)
return {
"chart":
client.ChartRepositoryApi(client.ApiClient(cfg)),
"products":
swagger_client.ProductsApi(swagger_client.ApiClient(cfg)),
"projectv2":
v2_swagger_client.ProjectApi(v2_swagger_client.ApiClient(cfg)),
"artifact":
v2_swagger_client.ArtifactApi(v2_swagger_client.ApiClient(cfg)),
"preheat":
v2_swagger_client.PreheatApi(v2_swagger_client.ApiClient(cfg)),
"quota":
v2_swagger_client.QuotaApi(v2_swagger_client.ApiClient(cfg)),
"repository":
v2_swagger_client.RepositoryApi(v2_swagger_client.ApiClient(cfg)),
"scan":
v2_swagger_client.ScanApi(v2_swagger_client.ApiClient(cfg)),
"scanall":
v2_swagger_client.ScanAllApi(v2_swagger_client.ApiClient(cfg)),
"scanner":
v2_swagger_client.ScannerApi(v2_swagger_client.ApiClient(cfg)),
"replication":
v2_swagger_client.ReplicationApi(v2_swagger_client.ApiClient(cfg)),
"robot":
v2_swagger_client.RobotApi(v2_swagger_client.ApiClient(cfg)),
"gc":
v2_swagger_client.GcApi(v2_swagger_client.ApiClient(cfg)),
"retention":
v2_swagger_client.RetentionApi(v2_swagger_client.ApiClient(cfg)),
}.get(api_type, 'Error: Wrong API type')
def sign_in(creds):
r = requests.get('https://mavensmate.appspot.com/github',
params={
'username': creds['username'],
'password': creds['password']
},
proxies=urllib.getproxies(),
verify=False)
r.raise_for_status()
return util.parse_rest_response(r.text)
def get_proxies():
proxies = getproxies()
filtered_proxies = {}
for key, value in proxies.items():
if key.startswith('http://'):
if not value.startswith('http://'):
filtered_proxies[key] = 'http://{0}'.format(value)
else:
filtered_proxies[key] = value
return filtered_proxies
def get_proxies():
proxies = getproxies() #proxies为引入的getproxies（50行）
filtered_proxies = {} #空字典
for key, value in proxies.items(): #遍历字典
if key.startswith('http'): #项的键以http开头
if not value.startswith('http'): #值的开头不为http
filtered_proxies[key] = 'http://%s' % value #将值置入字符串，生成新字符串
else:
filtered_proxies[key] = value #将值赋给键
return filtered_proxies #返回字典
def get_proxies():
proxies = getproxies()
filtered_proxies = {}
for key, value in proxies.items():
if key.startswith('http'):
if not value.startswith('http'):
filtered_proxies[key] = 'http://%s' % value
else:
filtered_proxies[key] = value
return filtered_proxies
def get_soap_client(
wsdlurl,
timeout=30): # pragma: no cover (not part of normal test suite)
"""Get a SOAP client for performing requests. The client is cached. The
timeout is in seconds."""
# this function isn't automatically tested because the functions using
# it are not automatically tested
if (wsdlurl, timeout) not in _soap_clients:
# try zeep first
try:
from zeep.transports import Transport
transport = Transport(timeout=timeout)
from zeep import CachingClient
client = CachingClient(wsdlurl, transport=transport).service
except ImportError:
# fall back to non-caching zeep client
try:
from zeep import Client
client = Client(wsdlurl, transport=transport).service
except ImportError:
# other implementations require passing the proxy config
try:
from urllib import getproxies
except ImportError:
from urllib.request import getproxies
# fall back to suds
try:
from suds.client import Client
client = Client(wsdlurl,
proxy=getproxies(),
timeout=timeout).service
except ImportError:
# use pysimplesoap as last resort
try:
from pysimplesoap.client import SoapClient
client = SoapClient(wsdl=wsdlurl,
proxy=getproxies(),
timeout=timeout)
except ImportError:
raise ImportError(
'No SOAP library (such as zeep) found')
_soap_clients[(wsdlurl, timeout)] = client
return _soap_clients[(wsdlurl, timeout)]
def _get_proxies(self):
proxies = getproxies()
proxy = {}
if self.proxy:
parsed_proxy = urlparse(self.proxy)
proxy[parsed_proxy.scheme] = parsed_proxy.geturl()
proxies.update(proxy)
return proxies
def make_pypi_client(url):
http_proxy_url = urllib.getproxies().get("http", "")
if http_proxy_url:
http_proxy_spec = urllib.splithost(urllib.splittype(http_proxy_url)[1])[0]
transport = ProxiedTransport()
transport.set_proxy(http_proxy_spec)
else:
transport = None
return Server(url, transport=transport)
def get_proxies():
proxies = getproxies()
filtered_proxies = {}
for key, value in proxies.items():
if key.startswith('http'):
if not value.startswith('http'):
filtered_proxies[key] = 'http://%s' % value
else:
filtered_proxies[key] = value
return filtered_proxies
def fix_proxy_env(self):
import urllib
proxy_settings = urllib.getproxies()
for ptype in proxy_settings:
proxy = proxy_settings[ptype]
if '://' not in proxy:
proxy = '%s://%s' % (ptype, proxy)
protocol, val = proxy.split('://')
proxy = '%s://%s@%s' % (protocol, self['proxy_auth'], val)
os.environ['%s_proxy' % ptype] = proxy
def _get_client(): # pragma: no cover (no tests for this function)
"""Get a SOAP client for performing VIES requests."""
# this function isn't automatically tested because the functions using
# it are not automatically tested
global _vies_client
if not _vies_client:
try:
from urllib import getproxies
except ImportError:
from urllib.request import getproxies
# try suds first
try:
from suds.client import Client
_vies_client = Client(vies_wsdl, proxy=getproxies()).service
except ImportError:
# fall back to using pysimplesoap
from pysimplesoap.client import SoapClient
_vies_client = SoapClient(wsdl=vies_wsdl, proxy=getproxies())
return _vies_client
def __init__(self):
"""
Initialize the default options.
"""
super(Configuration, self).__init__()
## checking options
self["allowedschemes"] = []
self['cookiefile'] = None
self['robotstxt'] = True
self["debugmemory"] = False
self["localwebroot"] = None
self["maxfilesizeparse"] = 1 * 1024 * 1024
self["maxfilesizedownload"] = 5 * 1024 * 1024
self["maxnumurls"] = None
self["maxrunseconds"] = None
self["maxrequestspersecond"] = 10
self["maxhttpredirects"] = 10
self["nntpserver"] = os.environ.get("NNTP_SERVER", None)
self["proxy"] = urllib.getproxies()
self["sslverify"] = True
self["threads"] = 10
self["timeout"] = 60
self["aborttimeout"] = 300
self["recursionlevel"] = -1
self["useragent"] = UserAgent
## authentication
self["authentication"] = []
self["loginurl"] = None
self["loginuserfield"] = "login"
self["loginpasswordfield"] = "password"
self["loginextrafields"] = {}
## filtering
self["externlinks"] = []
self["ignorewarnings"] = []
self["internlinks"] = []
self["checkextern"] = False
## plugins
self["pluginfolders"] = get_plugin_folders()
self["enabledplugins"] = []
## output
self['trace'] = False
self['quiet'] = False
self["verbose"] = False
self["warnings"] = True
self["fileoutput"] = []
self['output'] = 'text'
self["status"] = False
self["status_wait_seconds"] = 5
self['logger'] = None
self.loggers = {}
from ..logger import LoggerClasses
for c in LoggerClasses:
key = c.LoggerName
self[key] = {}
self.loggers[key] = c
def make_pypi_client(url):
http_proxy_url = urllib.getproxies().get("http", "")
if http_proxy_url:
http_proxy_spec = urllib.splithost(
urllib.splittype(http_proxy_url)[1])[0]
transport = ProxiedTransport()
transport.set_proxy(http_proxy_spec)
else:
transport = None
return ServerProxy(url, transport=transport)
def proxies(self):
""" optional field, might not exist
"""
try:
proxies = self.get_conf("proxies")
# If there is proxies section, but empty, it will try to use system proxy
if not proxies:
return urllib.getproxies()
return dict(proxies)
except:
return None
def get_proxies(self):
"""avproxy - instance of the AVProxy class"""
proxy_url = self.get_smart_proxy_url()
if proxy_url:
return {
'http': 'http://{}'.format(proxy_url),
'https': 'http://{}'.format(proxy_url),
}
# This helper function returns a dictionary of scheme to proxy server URL mappings.
# It scans the environment for variables named <scheme>_proxy
return urllib.getproxies()
def _get_http_proxy():
"""Returns an HTTP proxy URL formatted for consumption by SOAPpy.
SOAPpy does some fairly low-level HTTP manipulation and needs to be
explicitly made aware of HTTP proxy URLs, which also have to be
formatted without a schema or path.
"""
http_proxy = urllib.getproxies().get('http')
if http_proxy is None:
return None
return urlparse.urlparse(http_proxy).netloc
def get_lyrics(artist, song):
functions = [get_lyrics_genius, get_lyrics_songlyrics]
lyrics = ""
proxy = urllib.getproxies()
for f in functions:
print("bla")
lyrics = f(artist, song, proxy)
if lyrics != "":
break
return lyrics.encode('utf-8')
def becomeDaemon(self):
"""Code below comes from the excellent recipe by Chad J. Schroeder.
"""
# Workaround for http://bugs.python.org/issue9405 on Mac OS X
from platform import system
if system() == 'Darwin':
from urllib import getproxies
getproxies()
try:
pid = os.fork()
except OSError as e:
raise Exception("%s [%d]" % (e.strerror, e.errno))
if pid == 0: # The first child.
os.setsid()
try:
pid = os.fork() # Fork a second child.
except OSError as e:
raise Exception("%s [%d]" % (e.strerror, e.errno))
if pid == 0: # The second child.
os.chdir(WORKDIR)
os.umask(UMASK)
else:
# Exit parent (the first child) of the second child.
os._exit(0)
else:
os._exit(0) # Exit parent of the first child.
# Iterate through and close all stdin/out/err
for fd in range(0, MAXFD):
try:
os.close(fd)
except OSError: # ERROR, fd wasn't open to begin with (ignored)
pass
os.open(REDIRECT_TO, os.O_RDWR) # standard input (0)
# Duplicate standard input to standard output and standard error.
os.dup2(0, 1) # standard output (1)
os.dup2(0, 2) # standard error (2)
def start_schedulers(options):
from multiprocessing import Process
apps = [ag.split(':') for ag in options.schedulers]
if not options.with_scheduler and len(apps) == 1:
app, code = get_code_for_scheduler(options.folder, apps[0])
if not app:
return
print('starting single-scheduler for "%s"...' % app)
run(app, True, True, None, False, code, False, True)
return
# Work around OS X problem: http://bugs.python.org/issue9405
if PY2:
import urllib
else:
import urllib.request as urllib
urllib.getproxies()
processes = []
for app_groups in apps:
app, code = get_code_for_scheduler(options.folder, app_groups)
if not app:
continue
print('starting scheduler for "%s"...' % app)
args = (app, True, True, None, False, code, False, True)
p = Process(target=run, args=args)
processes.append(p)
print("Currently running %s scheduler processes" % (len(processes)))
p.start()
##to avoid bashing the db at the same time
time.sleep(0.7)
print("Processes started")
for p in processes:
try:
p.join()
except (KeyboardInterrupt, SystemExit):
print("Processes stopped")
except:
p.terminate()
p.join()
def __init__(self, url=None):
self._use_datetime = 0
# determine protocol type
self.__type, rest = urllib.splittype(url)
# find out proxy environment
p = get_proxy(urllib.getproxies(), self.__type)
if p:
opener = urllib2.build_opener(ConnectHTTPHandler(proxy=p),
ConnectHTTPSHandler(proxy=p))
urllib2.install_opener(opener)
def __init__(self):
self.session = requests.session()
self.session.proxies = urllib.getproxies()
self.headers = {
"Accept": "*/*",
"Accept-Encoding": "gzip, deflate",
"Accept-Language": "en;q=1, fr;q=0.9, de;q=0.8, ja;q=0.7, nl;q=0.6, it;q=0.5",
"Content-Type": "application/x-www-form-urlencoded; charset=utf-8",
"X-Robinhood-API-Version": "1.0.0",
"Connection": "keep-alive",
"User-Agent": "Robinhood/823 (iPhone; iOS 9.1.2; Scale/2.00)"
}
self.session.headers = self.headers
def get_proxy():
proxy_settings = config.get('proxy', {})
# if nothing was set, use OS-level proxies
if not proxy_settings or \
not (proxy_settings.get('http') or proxy_settings.get('https')):
proxy_settings = getproxies()
# allow anything local...
if proxy_settings:
set_no_proxy_settings(proxy_settings)
return proxy_settings
def _getClient(cls, **options):
"""Establish a connection and get a `suds.Client`
For internal purposes, only
"""
from suds.client import Client
try:
from urllib import getproxies
except ImportError:
from urllib.request import getproxies
return Client(cls.GSIS_HOST + cls.GSIS_WSDL,
proxy=getproxies(),
**options)
def get_lyrics(artist, song):
functions = [get_lyrics_genius, get_lyrics_songlyrics]
lyrics = ""
proxy = urllib.getproxies()
for f in functions:
print "Searching for %s - %s" % (artist.encode('utf-8'),
song.encode('utf-8'))
lyrics = f(artist, song, proxy)
if lyrics != "":
break
return lyrics.encode('utf-8')
def analyze_sentiment(text):
url = "http://text-processing.com/api/sentiment/"
proxy = urllib.getproxies()
data_text = requests.post(url, proxies=proxy, data={'text': text}).text
# probabilities are in data["probability"][label]
# label can be "neg", "neutral" or "pos"
try:
data = json.loads(data_text)
#return data["label"]
return data["probability"]["pos"]
except ValueError:
return None
def get_proxies():
proxies = getproxies()
filtered_proxies = {}
## Iterate key-value pair over dictionary
## for key in d: just iterates keys
## for key, value in d.items: key value pair
## http://stackoverflow.com/questions/3294889/iterating-over-dictionaries-using-for-loops-in-python
for key, value in proxies.items():
if key.startswith('http'):
if not value.startswith('http'):
filtered_proxies[key] = 'http://%s' % value
else:
filtered_proxies[key] = value
return filtered_proxies
def get_proxies():
import urllib
proxies = urllib.getproxies()
result = {}
if 'https' not in proxies and \
'http' in proxies:
url = proxies['http'].replace("http://", "https://")
result['https'] = papyon.Proxy(url)
for type, url in proxies.items():
if type == 'no': continue
if type == 'https' and url.startswith('http://'):
url = url.replace('http://', 'https://', 1)
result[type] = papyon.Proxy(url)
return result
def check_vies(number): # pragma: no cover (no tests for this function)
"""Queries the online European Commission VAT Information Exchange
System (VIES) for validity of the provided number. Note that the
service has usage limitations (see the VIES website for details).
This returns a dict-like object."""
# this function isn't automatically tested because it would require
# network access for the tests and unnecessarily load the VIES website
number = compact(number)
global _vies_client
if not _vies_client:
from suds.client import Client
from urllib import getproxies
_vies_client = Client(vies_wsdl, proxy=getproxies())
return _vies_client.service.checkVat(number[:2], number[2:])
def _create_client(server, credential, debug):
cfg = swagger_client.Configuration()
cfg.host = server.endpoint
cfg.verify_ssl = server.verify_ssl
# support basic auth only for now
cfg.username = credential.username
cfg.password = credential.password
cfg.debug = debug
proxies = getproxies()
proxy = proxies.get('http', proxies.get('all', None))
if proxy:
cfg.proxy = proxy
return swagger_client.ProductsApi(swagger_client.ApiClient(cfg))
