def __init__(self, url, username=None, password=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
super().__init__(url, username, password, timeout)
# Setting ssl key verificationas false
context = ssl._create_stdlib_context(check_hostname=False)
unverified_handler = HTTPSHandler(context=context, check_hostname=False)
install_opener(build_opener(unverified_handler))
def set_proxy(proxy, user=None, password=''):
"""
Set the HTTP proxy for Python to download through.
If ``proxy`` is None then tries to set proxy from environment or system
settings.
:param proxy: The HTTP proxy server to use. For example:
'http://proxy.example.com:3128/'
:param user: The username to authenticate with. Use None to disable
authentication.
:param password: The password to authenticate with.
"""
from nltk import compat
if proxy is None:
# Try and find the system proxy settings
try:
proxy = getproxies()['http']
except KeyError:
raise ValueError('Could not detect default proxy settings')
# Set up the proxy handler
proxy_handler = ProxyHandler({'https': proxy, 'http': proxy})
opener = build_opener(proxy_handler)
if user is not None:
# Set up basic proxy authentication if provided
password_manager = HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(realm=None, uri=proxy, user=user, passwd=password)
opener.add_handler(ProxyBasicAuthHandler(password_manager))
opener.add_handler(ProxyDigestAuthHandler(password_manager))
# Overide the existing url opener
install_opener(opener)
def debug():
"""
Activate debugging on urllib2.
"""
if six.PY2:
handler = HTTPSHandler(debuglevel=1)
opener = build_opener(handler)
install_opener(opener)
else:
http_client.HTTPConnection.debuglevel = 1
def build_request_with_data(url, data, api_key, method):
"""Build a request with the received method."""
http_redirect_with_data_handler = HTTPRedirectWithDataHandler(method=method)
opener = build_opener(http_redirect_with_data_handler)
install_opener(opener)
url = make_url(url, api_key=api_key, args=None)
request = Request(url, headers={'Content-Type': 'application/json'}, data=json.dumps(data))
request_method = request.get_method()
if request_method != method:
request.get_method = lambda: method
return opener, request
def quick_test():
"""Quick test/example of CacheHandler"""
from httplib2 import FileCache
logging.basicConfig(level=logging.DEBUG)
store = FileCache(".cache")
opener = request.build_opener(CacheHandler(store))
request.install_opener(opener)
response = request.urlopen("http://www.google.com/")
print(response.headers)
print("Response:", response.read()[:100], '...\n')
response.reload(store)
print(response.headers)
print("After reload:", response.read()[:100], '...\n')
def open_with_basic_auth(url, auth):
"""
opens an url protected with basic http authentication
:param url: string - the url to open
:param auth:
:return:
"""
user, passwd = auth
p = six.moves.urllib.request.HTTPPasswordMgrWithDefaultRealm()
p.add_password(None, url, user, passwd)
auth_handler = six.moves.urllib.request.HTTPBasicAuthHandler(p)
opener = request.build_opener(auth_handler)
request.install_opener(opener)
return opener.open(url)
def url_get(base_url, password_mgr=None, pathspec=None, params=None):
"""Make contact with the uri provided and return any contents."""
# Uses system proxy settings if they exist.
proxy = urlrequest.ProxyHandler()
if password_mgr is not None:
auth = urlrequest.HTTPDigestAuthHandler(password_mgr)
urlopener = urlrequest.build_opener(proxy, auth)
else:
urlopener = urlrequest.build_opener(proxy)
urlrequest.install_opener(urlopener)
full_url = build_url(base_url, pathspec=pathspec, params=params)
response = urlopener.open(full_url)
content = response.read()
response.close()
return content
def run(self, args, opts):
try:
import setuptools
except ImportError:
raise UsageError("setuptools not installed")
request.install_opener(request.build_opener(HTTPRedirectHandler))
if opts.list_targets:
for name, target in _get_targets().items():
print("%-20s %s" % (name, target['url']))
return
if opts.list_projects:
target = _get_target(opts.list_projects)
req = request.Request(_url(target, 'listprojects.json'))
_add_auth_header(req, target)
f = request.urlopen(req)
projects = json.loads(f.read())['projects']
print(os.linesep.join(projects))
return
tmpdir = None
if opts.build_egg: # build egg only
egg, tmpdir = _build_egg()
_log("Writing egg to %s" % opts.build_egg)
shutil.copyfile(egg, opts.build_egg)
else: # buld egg and deploy
target_name = _get_target_name(args)
target = _get_target(target_name)
project = _get_project(target, opts)
version = _get_version(target, opts)
if opts.egg:
_log("Using egg: %s" % opts.egg)
egg = opts.egg
else:
_log("Packing version %s" % version)
egg, tmpdir = _build_egg()
if not _upload_egg(target, egg, project, version):
self.exitcode = 1
if tmpdir:
if opts.debug:
_log("Output dir not removed: %s" % tmpdir)
else:
shutil.rmtree(tmpdir)
def _make_opener(self, realm, base_url, username, password):
"""uTorrent API need HTTP Basic Auth and cookie support for token verify."""
auth_handler = HTTPBasicAuthHandler()
auth_handler.add_password(realm=realm,
uri=base_url,
user=username,
passwd=password)
opener = build_opener(auth_handler)
install_opener(opener)
cookie_jar = CookieJar()
cookie_handler = HTTPCookieProcessor(cookie_jar)
handlers = [auth_handler, cookie_handler]
opener = build_opener(*handlers)
return opener
def do_call(self, url, data=None, method=None,
calltimeout=constants.SOCKET_TIMEOUT):
"""Send requests to server.
Send HTTPS call, get response in JSON.
Convert response into Python Object and return it.
"""
if self.url:
url = self.url + url
if "xx/sessions" not in url:
LOG.debug('Request URL: %(url)s\n'
'Call Method: %(method)s\n'
'Request Data: %(data)s\n',
{'url': url,
'method': method,
'data': data})
opener = urlreq.build_opener(urlreq.HTTPCookieProcessor(self.cookie))
urlreq.install_opener(opener)
result = None
try:
req = urlreq.Request(url, data, self.headers)
if method:
req.get_method = lambda: method
res_temp = urlreq.urlopen(req, timeout=calltimeout)
res = res_temp.read().decode("utf-8")
LOG.debug('Response Data: %(res)s.', {'res': res})
except Exception as err:
LOG.error(_LE('\nBad response from server: %(url)s.'
' Error: %(err)s'), {'url': url, 'err': err})
res = '{"error":{"code":%s,' \
'"description":"Connect server error"}}' \
% constants.ERROR_CONNECT_TO_SERVER
try:
result = jsonutils.loads(res)
except Exception as err:
err_msg = (_('JSON transfer error: %s.') % err)
LOG.error(err_msg)
raise exception.InvalidInput(reason=err_msg)
return result
def _get_initial_token(url):
"""
Create initial connection to get authentication token for future
requests.
Returns a string to be used in subsequent connections with the
X-CSRFToken header or the empty string if we didn't find any token in
the cookies.
"""
cookiejar = CookieJar()
opener = build_opener(HTTPCookieProcessor(cookiejar))
install_opener(opener)
opener.open(url)
for cookie in cookiejar:
if cookie.name == 'csrftoken':
return cookie.value
return ''
def prepare(self):
"""
Read options for uploading, check that they're sane
"""
super(BlazeMeterUploader, self).prepare()
self.client.address = self.settings.get("address", self.client.address)
self.client.data_address = self.settings.get("data-address", self.client.data_address)
self.client.timeout = dehumanize_time(self.settings.get("timeout", self.client.timeout))
self.send_interval = dehumanize_time(self.settings.get("send-interval", self.send_interval))
self.browser_open = self.settings.get("browser-open", self.browser_open)
token = self.settings.get("token", "")
proxy_settings = self.engine.config.get("settings").get("proxy")
if proxy_settings:
if proxy_settings.get("address"):
proxy_url = urlsplit(proxy_settings.get("address"))
username = proxy_settings.get("username")
pwd = proxy_settings.get("password")
if username and pwd:
proxy_uri = "%s://%s:%s@%s" % (proxy_url.scheme, username, pwd, proxy_url.netloc)
else:
proxy_uri = "%s://%s" % (proxy_url.scheme, proxy_url.netloc)
proxy_handler = ProxyHandler({"https": proxy_uri, "http": proxy_uri})
opener = build_opener(proxy_handler)
install_opener(opener)
if not token:
self.log.warning("No BlazeMeter API key provided, will upload anonymously")
self.client.token = token
self.client.active_session_id = self.parameters.get("session-id", None)
self.client.test_id = self.parameters.get("test-id", None)
self.client.user_id = self.parameters.get("user-id", None)
self.client.data_signature = self.parameters.get("signature", None)
if not self.client.test_id:
test_name = self.parameters.get("test", "Taurus Test") # TODO: provide a way to put datetime into test name
try:
self.client.ping() # to check connectivity and auth
if token:
self.test_id = self.client.test_by_name(test_name, {"type": "external"})
except HTTPError:
self.log.error("Cannot reach online results storage, maybe the address/token is wrong")
raise
def __init__(self, go_server, path=''):
self.go_server = go_server
self.logger = logging.getLogger(__name__+'.URL')
self.contents = []
full_url = go_server.url + path
if self.go_server.user and self.go_server.password:
self.logger.debug("logging in as %s"%self.go_server.user)
passman = HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, full_url,
self.go_server.user,
self.go_server.password)
install_opener(build_opener(HTTPBasicAuthHandler(passman)))
self.logger.debug("reading url %s"%full_url)
file_handle = urlopen(full_url)
if six.PY2:
for line in file_handle:
self.contents.append(line)
else:
for line in file_handle:
self.contents.append(line.decode())
self.logger.debug('line count: %d'%len(self.contents))
file_handle.close()
path_parts = urlsplit(full_url).path.split('/')
last = path_parts[-1]
# /path/to/something/
if last is '':
path_parts.pop()
last = path_parts[-1]
self.path_parts = path_parts
self.file_name = last
self.file_path = '/'.join(path_parts[0:-1])
def _get_initial_token(url):
"""
Create initial connection to get authentication token for future
requests.
Returns a string to be used in subsequent connections with the
X-CSRFToken header or the empty string if we didn't find any token in
the cookies.
"""
logging.info("Getting initial CSRF token.")
cookiejar = CookieJar()
opener = build_opener(HTTPCookieProcessor(cookiejar))
install_opener(opener)
opener.open(url)
for cookie in cookiejar:
if cookie.name == "csrftoken":
logging.info("Found CSRF token.")
return cookie.value
logging.warn("Did not find the CSRF token.")
return ""
def call(self, url, data=None, method=None):
"""Send requests to server.
Send HTTPS call, get response in JSON.
Convert response into Python Object and return it.
"""
if "xx/sessions" not in url:
LOG.debug('Request URL: %(url)s\n'
'Call Method: %(method)s\n'
'Request Data: %(data)s\n',
{'url': url,
'method': method,
'data': data})
opener = urlreq.build_opener(urlreq.HTTPCookieProcessor(self.cookie))
urlreq.install_opener(opener)
try:
req = urlreq.Request(url, data, self.headers)
if method:
req.get_method = lambda: method
res_temp = urlreq.urlopen(req, timeout=constants.SOCKET_TIMEOUT)
res = res_temp.read().decode("utf-8")
LOG.debug('Response Data: %(res)s.', {'res': res})
except Exception as err:
LOG.error(_LE('Bad response from server: %s.') % err)
raise err
try:
res_json = jsonutils.loads(res)
except Exception as err:
err_msg = (_('JSON transfer error: %s.') % err)
LOG.error(err_msg)
raise exception.InvalidShare(reason=err_msg)
return res_json
def do_call(self, url, data=None, method=None):
"""Send requests to server.
Send HTTPS call, get response in JSON.
Convert response into Python Object and return it.
"""
if "xx/sessions" not in url:
LOG.debug(
"Request URL: %(url)s\n" "Call Method: %(method)s\n" "Request Data: %(data)s\n",
{"url": url, "method": method, "data": data},
)
opener = urlreq.build_opener(urlreq.HTTPCookieProcessor(self.cookie))
urlreq.install_opener(opener)
result = None
try:
req = urlreq.Request(url, data, self.headers)
if method:
req.get_method = lambda: method
res_temp = urlreq.urlopen(req, timeout=constants.SOCKET_TIMEOUT)
res = res_temp.read().decode("utf-8")
LOG.debug("Response Data: %(res)s.", {"res": res})
except Exception as err:
LOG.error(_LE("\nBad response from server: %(url)s." " Error: %(err)s"), {"url": url, "err": err})
res = '{"error":{"code":%s,' '"description":"Connect server error"}}' % constants.ERROR_CONNECT_TO_SERVER
try:
result = jsonutils.loads(res)
except Exception as err:
err_msg = _("JSON transfer error: %s.") % err
LOG.error(err_msg)
raise exception.InvalidInput(reason=err_msg)
return result
def __init__(self, url, cookie_file=None, username=None, password=None,
api_token=None, agent=None, session=None, disable_proxy=False,
auth_callback=None, otp_token_callback=None,
verify_ssl=True, save_cookies=True):
if not url.endswith('/'):
url += '/'
self.url = url + 'api/'
self.save_cookies = save_cookies
if self.save_cookies:
self.cookie_jar, self.cookie_file = create_cookie_jar(
cookie_file=cookie_file)
try:
self.cookie_jar.load(ignore_expires=True)
except IOError:
pass
else:
self.cookie_jar = CookieJar()
self.cookie_file = None
# Get the cookie domain from the url. If the domain
# does not contain a '.' (e.g. 'localhost'), we assume
# it is a local domain and suffix it (See RFC 2109).
parsed_url = urlparse(url)
self.domain = parsed_url[1].partition(':')[0] # Remove Port.
if self.domain.count('.') < 1:
self.domain = '%s.local' % self.domain
if session:
cookie = Cookie(
version=0,
name=RB_COOKIE_NAME,
value=session,
port=None,
port_specified=False,
domain=self.domain,
domain_specified=True,
domain_initial_dot=True,
path=parsed_url[2],
path_specified=True,
secure=False,
expires=None,
discard=False,
comment=None,
comment_url=None,
rest={'HttpOnly': None})
self.cookie_jar.set_cookie(cookie)
if self.save_cookies:
self.cookie_jar.save()
if username:
# If the username parameter is given, we have to clear the session
# cookie manually or it will override the username:password
# combination retrieved from the authentication callback.
try:
self.cookie_jar.clear(self.domain, parsed_url[2],
RB_COOKIE_NAME)
except KeyError:
pass
# Set up the HTTP libraries to support all of the features we need.
password_mgr = ReviewBoardHTTPPasswordMgr(self.url,
username,
password,
api_token,
auth_callback,
otp_token_callback)
self.preset_auth_handler = PresetHTTPAuthHandler(self.url,
password_mgr)
handlers = []
if not verify_ssl:
context = ssl._create_unverified_context()
handlers.append(HTTPSHandler(context=context))
if disable_proxy:
handlers.append(ProxyHandler({}))
handlers += [
HTTPCookieProcessor(self.cookie_jar),
ReviewBoardHTTPBasicAuthHandler(password_mgr),
HTTPDigestAuthHandler(password_mgr),
self.preset_auth_handler,
ReviewBoardHTTPErrorProcessor(),
]
if agent:
self.agent = agent
else:
self.agent = ('RBTools/' + get_package_version()).encode('utf-8')
opener = build_opener(*handlers)
opener.addheaders = [
(b'User-agent', self.agent),
]
install_opener(opener)
self._cache = None
self._urlopen = urlopen
def __init__(self,
url,
cookie_file=None,
username=None,
password=None,
api_token=None,
agent=None,
session=None,
disable_proxy=False,
auth_callback=None,
otp_token_callback=None,
verify_ssl=True,
save_cookies=True,
ext_auth_cookies=None):
if not url.endswith('/'):
url += '/'
self.url = url + 'api/'
self.save_cookies = save_cookies
self.ext_auth_cookies = ext_auth_cookies
if self.save_cookies:
self.cookie_jar, self.cookie_file = create_cookie_jar(
cookie_file=cookie_file)
try:
self.cookie_jar.load(ignore_expires=True)
except IOError:
pass
else:
self.cookie_jar = CookieJar()
self.cookie_file = None
if self.ext_auth_cookies:
try:
self.cookie_jar.load(ext_auth_cookies, ignore_expires=True)
except IOError as e:
logging.critical(
'There was an error while loading a '
'cookie file: %s', e)
pass
# Get the cookie domain from the url. If the domain
# does not contain a '.' (e.g. 'localhost'), we assume
# it is a local domain and suffix it (See RFC 2109).
parsed_url = urlparse(url)
self.domain = parsed_url[1].partition(':')[0] # Remove Port.
if self.domain.count('.') < 1:
self.domain = '%s.local' % self.domain
if session:
cookie = Cookie(version=0,
name=RB_COOKIE_NAME,
value=session,
port=None,
port_specified=False,
domain=self.domain,
domain_specified=True,
domain_initial_dot=True,
path=parsed_url[2],
path_specified=True,
secure=False,
expires=None,
discard=False,
comment=None,
comment_url=None,
rest={'HttpOnly': None})
self.cookie_jar.set_cookie(cookie)
if self.save_cookies:
self.cookie_jar.save()
if username:
# If the username parameter is given, we have to clear the session
# cookie manually or it will override the username:password
# combination retrieved from the authentication callback.
try:
self.cookie_jar.clear(self.domain, parsed_url[2],
RB_COOKIE_NAME)
except KeyError:
pass
# Set up the HTTP libraries to support all of the features we need.
password_mgr = ReviewBoardHTTPPasswordMgr(self.url, username, password,
api_token, auth_callback,
otp_token_callback)
self.preset_auth_handler = PresetHTTPAuthHandler(
self.url, password_mgr)
handlers = []
if not verify_ssl:
context = ssl._create_unverified_context()
handlers.append(HTTPSHandler(context=context))
if disable_proxy:
handlers.append(ProxyHandler({}))
handlers += [
HTTPCookieProcessor(self.cookie_jar),
ReviewBoardHTTPBasicAuthHandler(password_mgr),
HTTPDigestAuthHandler(password_mgr),
self.preset_auth_handler,
ReviewBoardHTTPErrorProcessor(),
]
if agent:
self.agent = agent
else:
self.agent = ('RBTools/' + get_package_version()).encode('utf-8')
opener = build_opener(*handlers)
opener.addheaders = [
(str('User-agent'), str(self.agent)),
]
install_opener(opener)
self._cache = None
self._urlopen = urlopen
def setUp(self):
""" Set up the mock objects to do our unit tests """
my_opener = urllib2.build_opener(MyHTTPHandler)
urllib2.install_opener(my_opener)
self.p = dlipower.PowerSwitch(hostname='lpc.digital-loggers.com')
def getPageHTML2(
sURL, dSendHeaders = {}, tProxy = ('', -1 ), doFrames = True, bUnZip = True ):
#
"""
This one returns dReceiveHeaders and the page HTML.
gets page HTML (content) and other stuff,
but with synchronous (blocking) operation.
If cannot get page, returns empty string.
"""
#
from String.Get import getUnZipped
from String.Test import isGzipped
#from Utils.Both2n3 import Request, urlopen, build_opener, install_opener, URLError
from Web.Address import UrlMustHaveSchemeAndPath
from Web.HTML import oFindFrameBeg, oFindiFrameEnd, oFindiFrameBeg, \
oFindFrameSpec
from Web.Zip import getCompressedOffChunks
#
sURL = UrlMustHaveSchemeAndPath( sURL )
#
sHTML = ''
dReceiveHeaders = {}
#
sProxy = ''
#
sRealURL = sURL
#
sReason = sErrorCode = sComment = ''
#
if tProxy != ('', -1 ) and isDotQuadPortTuple( tProxy ):
#
# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/456195
#
sProxy = '%s:%s' % tProxy
#
opener = build_opener(
ConnectHTTPHandler( proxy = sProxy ), ConnectHTTPSHandler( proxy = sProxy ) )
#
install_opener(opener)
#
#
try:
#
oRequest = Request( sURL, headers = dSendHeaders )
#
oPage = urlopen( oRequest )
#
except URLError:
#
error, msg, traceback = exc_info()
#
if hasattr(msg, 'reason'):
sComment = 'We failed to reach the server.'
sReason = msg.reason
elif hasattr(msg, 'code'):
sComment = 'The server couldn\'t fulfill the request.'
sErrorCode = str( msg.code )
#
except SocketError:
#
error, msg, traceback = exc_info()
#
sComment = 'We hit a socket error on open.'
sReason = str( msg )
#
except Exception:
#
error, msg, traceback = exc_info()
#
sComment = 'We hit some unknown error.'
sReason = str( msg )
#
else:
#
try:
sHTML = oPage.read()
dReceiveHeaders = oPage.info()
sRealURL = oPage.geturl()
except (ValueError, SocketError):
error, msg, traceback = exc_info()
sComment = 'We hit a socket error on read.'
sReason = str( msg )
#
#
dResult = dict(
sReason = sReason,
sErrorCode = sErrorCode,
sComment = sComment,
sRealURL = sRealURL )
#
if bUnZip and (
dReceiveHeaders.get( 'content-encoding' ) == 'gzip' or
isGzipped( sHTML ) ):
#
bUnZipped, sHTML = getUnZipped( sHTML )
#
if not bUnZipped and \
dReceiveHeaders.get( 'transfer-encoding' ) == 'chunked':
#
sHTML = getCompressedOffChunks( sHTML )
#
bUnZipped, sHTML = getUnZipped( sHTML )
#
#
#
if sHTML and doFrames:
#
liFrames = oFindiFrameBeg.findall( sHTML )
lFrames = oFindFrameBeg.findall( sHTML )
#
sCookie = dReceiveHeaders.get( 'cookie' )
#
if liFrames:
#
liFrameSpecs = [ lFrame[ 1 ] for lFrame in
[ oFindFrameSpec.split( sFrame ) for sFrame in liFrames ]
if len( lFrame ) > 1 ]
#
sHTML = _getFrameText(
sHTML, liFrameSpecs, sURL, dSendHeaders, sCookie, tProxy,
oFindiFrameBeg, oFindiFrameEnd )
#
#
if lFrames:
#
lParts = oFindFrameBeg.split( sHTML )
#
if lParts: del lParts[ 0 ]
#
lFrameSpecs = [ lFrame[ 1 ] for lFrame in
[ oFindFrameSpec.split( sPart ) for sPart in lParts ]
if len( lFrame ) > 1 ]
#
sHTML = _getFrameText(
sHTML, lFrameSpecs, sURL, dSendHeaders, sCookie, tProxy, oFindFrameBeg )
#
#
return sHTML, dReceiveHeaders, dResult
def download_scripts(proxies=None, install_dir=None):
import visdom
print("Downloading scripts. It might take a while.")
# location in which to download stuff:
if install_dir is None:
install_dir = os.path.dirname(visdom.__file__)
# all files that need to be downloaded:
b = 'https://unpkg.com/'
bb = '%[email protected]/dist/' % b
ext_files = {
# - js
'%[email protected]/dist/jquery.min.js' % b: 'jquery.min.js',
'%[email protected]/dist/js/bootstrap.min.js' % b: 'bootstrap.min.js',
'%[email protected]/umd/react.production.min.js' % b: 'react-react.min.js',
'%[email protected]/umd/react-dom.production.min.js' % b: 'react-dom.min.js', # noqa
'%[email protected]/dist/react-modal.min.js' % b: 'react-modal.min.js', # noqa
'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_SVG': # noqa
'mathjax-MathJax.js',
# here is another url in case the cdn breaks down again.
# https://raw.githubusercontent.com/plotly/plotly.js/master/dist/plotly.min.js
'https://cdn.plot.ly/plotly-latest.min.js': 'plotly-plotly.min.js',
# Stanford Javascript Crypto Library for Password Hashing
'%[email protected]/sjcl.js' % b: 'sjcl.js',
# - css
'%[email protected]/css/styles.css' % b: 'react-resizable-styles.css', # noqa
'%[email protected]/css/styles.css' % b: 'react-grid-layout-styles.css', # noqa
'%scss/bootstrap.min.css' % bb: 'bootstrap.min.css',
# - fonts
'%[email protected]' % b: 'classnames',
'%[email protected]' % b: 'layout_bin_packer',
'%sfonts/glyphicons-halflings-regular.eot' % bb:
'glyphicons-halflings-regular.eot',
'%sfonts/glyphicons-halflings-regular.woff2' % bb:
'glyphicons-halflings-regular.woff2',
'%sfonts/glyphicons-halflings-regular.woff' % bb:
'glyphicons-halflings-regular.woff',
'%sfonts/glyphicons-halflings-regular.ttf' % bb:
'glyphicons-halflings-regular.ttf',
'%sfonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular' % bb: # noqa
'glyphicons-halflings-regular.svg#glyphicons_halflingsregular',
}
# make sure all relevant folders exist:
dir_list = [
'%s' % install_dir,
'%s/static' % install_dir,
'%s/static/js' % install_dir,
'%s/static/css' % install_dir,
'%s/static/fonts' % install_dir,
]
for directory in dir_list:
if not os.path.exists(directory):
os.makedirs(directory)
# set up proxy handler:
from six.moves.urllib import request
from six.moves.urllib.error import HTTPError, URLError
handler = request.ProxyHandler(proxies) if proxies is not None \
else request.BaseHandler()
opener = request.build_opener(handler)
request.install_opener(opener)
built_path = os.path.join(here, 'static/version.built')
is_built = False
if os.path.exists(built_path):
with open(built_path, 'r') as build_file:
build_version = build_file.read().strip()
if build_version == visdom.__version__:
is_built = True
else:
os.remove(built_path)
# download files one-by-one:
for (key, val) in ext_files.items():
# set subdirectory:
sub_dir = 'fonts'
if '.js' in key:
sub_dir = 'js'
if '.css' in key:
sub_dir = 'css'
# download file:
filename = '%s/static/%s/%s' % (install_dir, sub_dir, val)
if not os.path.exists(filename) or not is_built:
req = request.Request(key,
headers={'User-Agent': 'Chrome/30.0.0.0'})
try:
data = opener.open(req).read()
with open(filename, 'wb') as fwrite:
fwrite.write(data)
except HTTPError as exc:
logging.error('Error {} while downloading {}'.format(
exc.code, key))
except URLError as exc:
logging.error('Error {} while downloading {}'.format(
exc.reason, key))
if not is_built:
with open(built_path, 'w+') as build_file:
build_file.write(visdom.__version__)
def _post(self, url,
param_dict={},
files={},
securityHandler=None,
additional_headers={},
custom_handlers=[],
proxy_url=None,
proxy_port=80,
compress=True,
out_folder=None,
file_name=None):
"""
Performs a POST operation on a URL.
Inputs:
param_dict - key/value pair of values
ex: {"foo": "bar"}
files - key/value pair of file objects where the key is
the input name and the value is the file path
ex: {"file": r"c:\temp\myfile.zip"}
securityHandler - object that handles the token or other site
security. It must inherit from the base security class.
ex: arcrest.AGOLSecurityHandler("SomeUsername", "SOMEPASSWORD")
additional_headers - are additional key/value headers that a user
wants to pass during the operation.
ex: {"accept-encoding": "gzip"}
custom_handlers - this is additional web operation handlers as a
list of objects.
Ex: [CustomAuthHandler]
proxy_url - url of the proxy
proxy_port - default 80, port number of the proxy
compress - default true, determines if gzip should be used of not for
the web operation.
out_folder - if the URL requested returns a file, this will be the
disk save location
file_name - if the operation returns a file and the file name is not
given in the header or a user wishes to override the return saved
file name, provide value here.
Output:
returns dictionary or string depending on web operation.
"""
self._last_method = "POST"
headers = {}
opener = None
return_value = None
handlers = [RedirectHandler()]
param_dict, handler, cj = self._processHandler(securityHandler, param_dict)
if handler is not None:
handlers.append(handler)
if cj is not None:
handlers.append(request.HTTPCookieProcessor(cj))
if isinstance(custom_handlers, list) and \
len(custom_handlers) > 0:
for h in custom_handlers:
handlers.append(h)
if compress:
headers['Accept-Encoding'] = 'gzip'
else:
headers['Accept-Encoding'] = ''
for k,v in additional_headers.items():
headers[k] = v
del k,v
opener = request.build_opener(*handlers)
request.install_opener(opener)
opener.addheaders = [(k,v) for k,v in headers.items()]
if len(files) == 0:
data = urlencode(param_dict)
if self.PY3:
data = data.encode('ascii')
opener.data = data
resp = opener.open(url.encode('ascii'), data=data)
else:
mpf = MultiPartForm(param_dict=param_dict,
files=files)
req = request.Request(url.encode('ascii'))
body = mpf.make_result
req.add_header('User-agent', self.useragent)
req.add_header('Content-type', mpf.get_content_type())
req.add_header('Content-length', len(body))
req.data = body
resp = request.urlopen(req)
del body, mpf
self._last_code = resp.getcode()
self._last_url = resp.geturl()
return_value = self._process_response(resp=resp,
out_folder=out_folder)
if isinstance(return_value, dict):
if "error" in return_value and \
'message' in return_value['error']:
if return_value['error']['message'].lower() == 'request not made over ssl':
if url.startswith('http://'):
url = url.replace('http://', 'https://')
return self._post(url,
param_dict,
files,
securityHandler,
additional_headers,
custom_handlers,
proxy_url,
proxy_port,
compress,
out_folder,
file_name)
return return_value
else:
return return_value
return return_value
# -*- coding: utf-8 -*-
"""Exposes urllib imports with additional request handlers."""
from __future__ import (absolute_import, division, print_function,
unicode_literals, with_statement)
# pylint: disable=import-error
from six.moves.urllib import error, parse, request # noqa: F401
from watchmaker.utils.urllib.request_handlers import HAS_BOTO3, S3Handler
if HAS_BOTO3:
request.install_opener(request.build_opener(S3Handler))
from six.moves.urllib import request
from docutils import nodes
from docutils.utils import relative_path
import sphinx
from sphinx.locale import _
from sphinx.builders.html import INVENTORY_FILENAME
handlers = [request.ProxyHandler(), request.HTTPRedirectHandler(), request.HTTPHandler()]
try:
handlers.append(request.HTTPSHandler)
except AttributeError:
pass
request.install_opener(request.build_opener(*handlers))
UTF8StreamReader = codecs.lookup("utf-8")[2]
def read_inventory_v1(f, uri, join):
f = UTF8StreamReader(f)
invdata = {}
line = next(f)
projname = line.rstrip()[11:]
line = next(f)
version = line.rstrip()[11:]
for line in f:
name, type, location = line.rstrip().split(None, 2)
location = join(uri, location)
# version 1 did not add anchors to the location
def conn(self, req):
'''URL connection wrappercatching common exceptions and retrying where necessary
param: connreq can be either a url string or a request object
'''
sr = self.__sec, 'Connection Manager'
self.setRequest(req)
req_str = self.getRequestStr()
#if self.auth is set it should have been added to the request header... might be legacy where that hasn't happened
if self.auth:
self.addRequestHeader("Authorization", self.auth)
request.install_opener(self.opener(purl=self.openerstrs_ntlm))
retry = INIT_MAX_RETRY_ATTEMPTS
while retry > 0:
retry -= 1
try:
handle = request.urlopen(self.getRequest()) #,data)
if handle:
if handle.geturl() != req_str:
msg = 'Redirect Warning'
#cannot easily mask redirected url so logging original
LM.info(
msg, LM._LogExtra(*sr,
exc=None,
url=req_str,
rty=0))
return handle
#self.setResponse(handle)
#break
except HTTPError as he:
last_exc = he
if re.search('429', str(he)):
msg = 'RateLimit Error {0}. Sleep awaiting 429 expiry. Attempt {1}'.format(
he, MAX_RETRY_ATTEMPTS - retry)
LM.error(msg,
LM._LogExtra(*sr, exc=he, url=req_str, rty=retry))
LDSAPI.sleepIncr(retry)
continue
elif retry:
# I'm leaving this code here to test with because LDS was
# somehow throwing exceptions as well as redirecting
#
#if re.search('301',str(he)):
# msg = 'Redirect Error {0}'.format(he)
# #if we have a valid response and its a 301 see if it contains a redirect-to
# if handle and handle.geturl():
# retry = 1
# self.setRequest(handle.geturl()) #TODO reauth?
# msg += '. Attempting alternate connect'
# else:
# retry = 0
# LM.error(msg,LM._LogExtra(*sr,exc=he,url=self.getRequestStr(),rty=0))
# continue
if re.search('401|500', str(he)):
msg = 'HTTP Error {0} Returns {1}. Attempt {2}'.format(
req_str, he, MAX_RETRY_ATTEMPTS - retry)
LM.error(
msg,
LM._LogExtra(*sr, exc=he, url=req_str, rty=retry))
continue
elif re.search('403', str(he)):
msg = 'HTTP Error {0} Returns {1}. Attempt {2} (consider proxy)'.format(
req_str, he, MAX_RETRY_ATTEMPTS - retry)
LM.error(
msg,
LM._LogExtra(*sr, exc=he, url=req_str, rty=retry))
continue
elif re.search('502', str(he)):
msg = 'Proxy Error {0} Returns {1}. Attempt {2}'.format(
req_str, he, MAX_RETRY_ATTEMPTS - retry)
LM.error(
msg,
LM._LogExtra(*sr, exc=he, url=req_str, rty=retry))
continue
elif re.search('410', str(he)):
msg = 'Layer removed {0} Returns {1}. Attempt {2}'.format(
req_str, he, MAX_RETRY_ATTEMPTS - retry)
LM.error(
msg,
LM._LogExtra(*sr, exc=he, url=req_str, rty=retry))
retry = 0
continue
else:
msg = 'Error with request {0} returns {1}'.format(
req_str, he)
LM.error(
msg,
LM._LogExtra(*sr, exc=he, url=req_str, rty=retry))
continue
else:
#Retries have been exhausted, raise the active httpexception
raise HTTPError(he.msg + msg)
except HttpResponseError as rd:
LM.warning('Disconnect. {}'.format(rd),
LM._LogExtra(*sr, exc=rd, url=req_str, rty=retry))
LDSAPI.sleepIncr(retry)
continue
except URLError as ue:
LM.warning('URL error on connect {}'.format(ue),
LM._LogExtra(*sr, exc=ue, url=req_str, rty=retry))
if re.search('Connection refused|violation of protocol',
str(ue)):
LDSAPI.sleepIncr(retry)
continue
#raise ue
except ConnectionError as ce:
LM.warning('Error on connection. {}'.format(ce),
LM._LogExtra(*sr, exc=ce, url=req_str, rty=retry))
LDSAPI.sleepIncr(retry)
continue
except ValueError as ve:
LM.error('Value error on connect {}'.format(ve),
LM._LogExtra(*sr, exc=ve, url=req_str, rty=retry))
raise ve
except Exception as xe:
LM.error('Other error on connect {}'.format(xe),
LM._LogExtra(*sr, exc=xe, url=req_str, rty=retry))
raise xe
else:
raise last_exc
def download_scripts(proxies=None, install_dir=None):
print("Downloading scripts. It might take a while.")
# location in which to download stuff:
if install_dir is None:
import visdom
install_dir = os.path.dirname(visdom.__file__)
# all files that need to be downloaded:
b = 'https://unpkg.com/'
bb = '%[email protected]/dist/' % b
ext_files = {
'%[email protected]/dist/jquery.min.js' % b: 'jquery.min.js',
'%[email protected]/dist/js/bootstrap.min.js' % b: 'bootstrap.min.js',
'%[email protected]/css/styles.css' % b: 'react-resizable-styles.css',
'%[email protected]/css/styles.css' % b: 'react-grid-layout-styles.css',
'%[email protected]/dist/react.min.js' % b: 'react-react.min.js',
'%[email protected]/dist/react-dom.min.js' % b: 'react-dom.min.js',
'%[email protected]' % b: 'classnames',
'%[email protected]' % b: 'layout_bin_packer',
'https://raw.githubusercontent.com/STRML/react-grid-layout/0.14.0/dist/' +
'react-grid-layout.min.js': 'react-grid-layout.min.js',
'https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_SVG':
'mathjax-MathJax.js',
# here is another url in case the cdn breaks down again.
# https://raw.githubusercontent.com/plotly/plotly.js/master/dist/plotly.min.js
'https://cdn.plot.ly/plotly-latest.min.js':
'plotly-plotly.min.js',
'%scss/bootstrap.min.css' % bb: 'bootstrap.min.css',
'%sfonts/glyphicons-halflings-regular.eot' % bb:
'glyphicons-halflings-regular.eot',
'%sfonts/glyphicons-halflings-regular.woff2' % bb:
'glyphicons-halflings-regular.woff2',
'%sfonts/glyphicons-halflings-regular.woff' % bb:
'glyphicons-halflings-regular.woff',
'%sfonts/glyphicons-halflings-regular.ttf' % bb:
'glyphicons-halflings-regular.ttf',
'%sfonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular' % bb:
'glyphicons-halflings-regular.svg#glyphicons_halflingsregular',
}
# make sure all relevant folders exist:
dir_list = [
'%s' % install_dir,
'%s/static' % install_dir,
'%s/static/js' % install_dir,
'%s/static/css' % install_dir,
'%s/static/fonts' % install_dir,
]
for directory in dir_list:
if not os.path.exists(directory):
os.makedirs(directory)
# set up proxy handler:
from six.moves.urllib import request
from six.moves.urllib.error import HTTPError, URLError
handler = request.ProxyHandler(proxies) if proxies is not None \
else request.BaseHandler()
opener = request.build_opener(handler)
request.install_opener(opener)
# download files one-by-one:
for (key, val) in ext_files.items():
# set subdirectory:
sub_dir = 'fonts'
if '.js' in key:
sub_dir = 'js'
if '.css' in key:
sub_dir = 'css'
# download file:
filename = '%s/static/%s/%s' % (install_dir, sub_dir, val)
if not os.path.exists(filename):
req = request.Request(key, headers={'User-Agent': 'Chrome/30.0.0.0'})
try:
data = opener.open(req).read()
with open(filename, 'wb') as fwrite:
fwrite.write(data)
except (HTTPError, URLError) as exc:
logging.error('Error {} while downloading {}'.format(exc.code, key))
def download_scripts(proxies=None, install_dir=None):
print("Downloading scripts. It might take a while.")
# location in which to download stuff:
if install_dir is None:
import visdom
install_dir = os.path.dirname(visdom.__file__)
# all files that need to be downloaded:
b = 'https://unpkg.com/'
bb = '%[email protected]/dist/' % b
ext_files = {
'%[email protected]/dist/jquery.min.js' % b:
'jquery.min.js',
'%[email protected]/dist/js/bootstrap.min.js' % b:
'bootstrap.min.js',
'%[email protected]/css/styles.css' % b:
'react-resizable-styles.css',
'%[email protected]/css/styles.css' % b:
'react-grid-layout-styles.css',
'%[email protected]/dist/react-modal.min.js' % b:
'react-modal.min.js',
'%[email protected]/dist/react.min.js' % b:
'react-react.min.js',
'%[email protected]/dist/react-dom.min.js' % b:
'react-dom.min.js',
'%[email protected]' % b:
'classnames',
'%[email protected]' % b:
'layout_bin_packer',
'https://raw.githubusercontent.com/STRML/react-grid-layout/0.14.0/dist/' + 'react-grid-layout.min.js':
'react-grid-layout.min.js',
'https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_SVG':
'mathjax-MathJax.js',
# here is another url in case the cdn breaks down again.
# https://raw.githubusercontent.com/plotly/plotly.js/master/dist/plotly.min.js
'https://cdn.plot.ly/plotly-latest.min.js':
'plotly-plotly.min.js',
'%scss/bootstrap.min.css' % bb:
'bootstrap.min.css',
'%sfonts/glyphicons-halflings-regular.eot' % bb:
'glyphicons-halflings-regular.eot',
'%sfonts/glyphicons-halflings-regular.woff2' % bb:
'glyphicons-halflings-regular.woff2',
'%sfonts/glyphicons-halflings-regular.woff' % bb:
'glyphicons-halflings-regular.woff',
'%sfonts/glyphicons-halflings-regular.ttf' % bb:
'glyphicons-halflings-regular.ttf',
'%sfonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular' % bb:
'glyphicons-halflings-regular.svg#glyphicons_halflingsregular',
}
# make sure all relevant folders exist:
dir_list = [
'%s' % install_dir,
'%s/static' % install_dir,
'%s/static/js' % install_dir,
'%s/static/css' % install_dir,
'%s/static/fonts' % install_dir,
]
for directory in dir_list:
if not os.path.exists(directory):
os.makedirs(directory)
# set up proxy handler:
from six.moves.urllib import request
from six.moves.urllib.error import HTTPError, URLError
handler = request.ProxyHandler(proxies) if proxies is not None \
else request.BaseHandler()
opener = request.build_opener(handler)
request.install_opener(opener)
# download files one-by-one:
for (key, val) in ext_files.items():
# set subdirectory:
sub_dir = 'fonts'
if '.js' in key:
sub_dir = 'js'
if '.css' in key:
sub_dir = 'css'
# download file:
filename = '%s/static/%s/%s' % (install_dir, sub_dir, val)
if not os.path.exists(filename):
req = request.Request(key,
headers={'User-Agent': 'Chrome/30.0.0.0'})
try:
data = opener.open(req).read()
with open(filename, 'wb') as fwrite:
fwrite.write(data)
except HTTPError as exc:
logging.error('Error {} while downloading {}'.format(
exc.code, key))
except URLError as exc:
logging.error('Error {} while downloading {}'.format(
exc.reason, key))
def setUp(self):
self.http_mock = Mock(side_effect=[])
opener = OpenerDirector()
opener.add_handler(TestHandler(self.http_mock))
install_opener(opener)
def install(buildout=None, pwd_path=None):
pwdsf = StringIO()
combined_creds = []
github_creds = None
creds = []
local_pwd_path = ''
github_repos = None
if buildout is not None:
local_pwd_path = os.path.join(
buildout['buildout']['directory'],
'.httpauth')
if 'github-repos' in buildout['buildout']:
github_repos = buildout['buildout']['github-repos'].split('\n')
system_pwd_path = os.path.join(
os.path.expanduser('~'),
'.buildout',
'.httpauth')
def combine_cred_file(file_path, combined_creds):
if file_path is None or not os.path.exists(file_path):
return
cred_file = open(file_path)
combined_creds += [l.strip()
for l in cred_file.readlines() if l.strip()]
cred_file.close()
# combine all the possible .httpauth files together
combine_cred_file(pwd_path, combined_creds)
combine_cred_file(local_pwd_path, combined_creds)
combine_cred_file(system_pwd_path, combined_creds)
pwdsf_len = pwdsf.write(u"\n".join(combined_creds))
pwdsf.seek(0)
if not pwdsf_len:
pwdsf = None
log.warn('Could not load authentication information')
try:
auth_handler = CredHandler()
github_creds = get_github_credentials()
new_handlers = []
if github_creds:
new_handlers.append(GithubHandler(github_creds, github_repos))
if pwdsf:
for l, row in enumerate(csv.reader(pwdsf)):
if len(row) == 3:
realm, uris, user = (el.strip() for el in row)
password = prompt_passwd(realm, user)
elif len(row) == 4:
realm, uris, user, password = (el.strip() for el in row)
else:
raise RuntimeError(
"Authentication file cannot be parsed %s:%s" % (
pwd_path, l + 1))
creds.append((realm, uris, user, password))
log.debug('Added credentials %r, %r' % (realm, uris))
auth_handler.add_password(realm, uris, user, password)
if creds:
new_handlers.append(auth_handler)
if creds or github_creds:
download.url_opener = URLOpener(creds, github_creds, github_repos)
if new_handlers:
if url_opener is not None:
handlers = url_opener.handlers[:]
handlers[:0] = new_handlers
else:
handlers = new_handlers
urllib_request.build_opener = lambda *a: original_build_opener(*handlers)
urllib_request.install_opener(urllib_request.build_opener())
finally:
if pwdsf:
pwdsf.close()
def _mule_fixup():
from six.moves.urllib.request import install_opener
install_opener(None)
def tearDown(self):
# Uninstall custom opener
install_opener(None)
def setup(self):
# Can't import this at the top because it's circular.
# XXX Someone smarter than me, please figure out the right
# XXX way to do this.
from spambayes.FileCorpus import ExpiryFileCorpus, FileMessageFactory
username = options["globals", "proxy_username"]
password = options["globals", "proxy_password"]
server = options["globals", "proxy_server"]
if server.find(":") != -1:
server, port = server.split(':', 1)
else:
port = 8080
if server:
# Build a new opener that uses a proxy requiring authorization
proxy_support = urllib2.ProxyHandler({"http" : \
"http://%s:%s@%s:%d" % \
(username, password,
server, port)})
opener = urllib2.build_opener(proxy_support,
urllib2.HTTPHandler)
else:
# Build a new opener without any proxy information.
opener = urllib2.build_opener(urllib2.HTTPHandler)
# Install it
urllib2.install_opener(opener)
# Setup the cache for retrieved urls
age = options["URLRetriever", "x-cache_expiry_days"]*24*60*60
dir = options["URLRetriever", "x-cache_directory"]
if not os.path.exists(dir):
# Create the directory.
if options["globals", "verbose"]:
print("Creating URL cache directory", file=sys.stderr)
os.makedirs(dir)
self.urlCorpus = ExpiryFileCorpus(age, FileMessageFactory(),
dir, cacheSize=20)
# Kill any old information in the cache
self.urlCorpus.removeExpiredMessages()
# Setup caches for unretrievable urls
self.bad_url_cache_name = os.path.join(dir, "bad_urls.pck")
self.http_error_cache_name = os.path.join(dir, "http_error_urls.pck")
if os.path.exists(self.bad_url_cache_name):
try:
self.bad_urls = pickle_read(self.bad_url_cache_name)
except (IOError, ValueError):
# Something went wrong loading it (bad pickle,
# probably). Start afresh.
if options["globals", "verbose"]:
print("Bad URL pickle, using new.", file=sys.stderr)
self.bad_urls = {"url:non_resolving": (),
"url:non_html": (),
"url:unknown_error": ()}
else:
if options["globals", "verbose"]:
print("URL caches don't exist: creating")
self.bad_urls = {"url:non_resolving": (),
"url:non_html": (),
"url:unknown_error": ()}
if os.path.exists(self.http_error_cache_name):
try:
self.http_error_urls = pickle_read(self.http_error_cache_name)
except (IOError, ValueError):
# Something went wrong loading it (bad pickle,
# probably). Start afresh.
if options["globals", "verbose"]:
print("Bad HHTP error pickle, using new.", file=sys.stderr)
self.http_error_urls = {}
else:
self.http_error_urls = {}
def _mule_fixup():
from six.moves.urllib.request import install_opener
install_opener(None)
def download_scripts(proxies=None, install_dir=None):
import visdom
print("Downloading scripts. It might take a while.")
# location in which to download stuff:
if install_dir is None:
install_dir = os.path.dirname(visdom.__file__)
# all files that need to be downloaded:
b = 'https://unpkg.com/'
bb = '%[email protected]/dist/' % b
ext_files = {
# - js
'%[email protected]/dist/jquery.min.js' % b: 'jquery.min.js',
'%[email protected]/dist/js/bootstrap.min.js' % b: 'bootstrap.min.js',
'%[email protected]/umd/react.production.min.js' % b: 'react-react.min.js',
'%[email protected]/umd/react-dom.production.min.js' % b: 'react-dom.min.js', # noqa
'%[email protected]/dist/react-modal.min.js' % b: 'react-modal.min.js', # noqa
'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_SVG': # noqa
'mathjax-MathJax.js',
# here is another url in case the cdn breaks down again.
# https://raw.githubusercontent.com/plotly/plotly.js/master/dist/plotly.min.js
'https://cdn.plot.ly/plotly-latest.min.js': 'plotly-plotly.min.js',
# Stanford Javascript Crypto Library for Password Hashing
'%[email protected]/sjcl.js' % b: 'sjcl.js',
# - css
'%[email protected]/css/styles.css' % b: 'react-resizable-styles.css', # noqa
'%[email protected]/css/styles.css' % b: 'react-grid-layout-styles.css', # noqa
'%scss/bootstrap.min.css' % bb: 'bootstrap.min.css',
# - fonts
'%[email protected]' % b: 'classnames',
'%[email protected]' % b: 'layout_bin_packer',
'%sfonts/glyphicons-halflings-regular.eot' % bb:
'glyphicons-halflings-regular.eot',
'%sfonts/glyphicons-halflings-regular.woff2' % bb:
'glyphicons-halflings-regular.woff2',
'%sfonts/glyphicons-halflings-regular.woff' % bb:
'glyphicons-halflings-regular.woff',
'%sfonts/glyphicons-halflings-regular.ttf' % bb:
'glyphicons-halflings-regular.ttf',
'%sfonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular' % bb: # noqa
'glyphicons-halflings-regular.svg#glyphicons_halflingsregular',
}
# make sure all relevant folders exist:
dir_list = [
'%s' % install_dir,
'%s/static' % install_dir,
'%s/static/js' % install_dir,
'%s/static/css' % install_dir,
'%s/static/fonts' % install_dir,
]
for directory in dir_list:
if not os.path.exists(directory):
os.makedirs(directory)
# set up proxy handler:
from six.moves.urllib import request
from six.moves.urllib.error import HTTPError, URLError
handler = request.ProxyHandler(proxies) if proxies is not None \
else request.BaseHandler()
opener = request.build_opener(handler)
request.install_opener(opener)
built_path = os.path.join(here, 'static/version.built')
is_built = visdom.__version__ == 'no_version_file'
if os.path.exists(built_path):
with open(built_path, 'r') as build_file:
build_version = build_file.read().strip()
if build_version == visdom.__version__:
is_built = True
else:
os.remove(built_path)
# download files one-by-one:
for (key, val) in ext_files.items():
# set subdirectory:
sub_dir = 'fonts'
if '.js' in key:
sub_dir = 'js'
if '.css' in key:
sub_dir = 'css'
# download file:
filename = '%s/static/%s/%s' % (install_dir, sub_dir, val)
if not os.path.exists(filename) or not is_built:
req = request.Request(key,
headers={'User-Agent': 'Chrome/30.0.0.0'})
try:
data = opener.open(req).read()
with open(filename, 'wb') as fwrite:
fwrite.write(data)
except HTTPError as exc:
logging.error('Error {} while downloading {}'.format(
exc.code, key))
except URLError as exc:
logging.error('Error {} while downloading {}'.format(
exc.reason, key))
if not is_built:
with open(built_path, 'w+') as build_file:
build_file.write(visdom.__version__)
def unload(buildout=None):
urllib_request.build_opener = original_build_opener
urllib_request.install_opener(urllib_request.build_opener())
