def download_scripts(proxies=None, install_dir=None): import visdom print("Downloading scripts. It might take a while.") # location in which to download stuff: if install_dir is None: install_dir = os.path.dirname(visdom.__file__) # all files that need to be downloaded: b = 'https://unpkg.com/' bb = '%[email protected]/dist/' % b ext_files = { # - js '%[email protected]/dist/jquery.min.js' % b: 'jquery.min.js', '%[email protected]/dist/js/bootstrap.min.js' % b: 'bootstrap.min.js', '%[email protected]/umd/react.production.min.js' % b: 'react-react.min.js', '%[email protected]/umd/react-dom.production.min.js' % b: 'react-dom.min.js', # noqa '%[email protected]/dist/react-modal.min.js' % b: 'react-modal.min.js', # noqa 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_SVG': # noqa 'mathjax-MathJax.js', # here is another url in case the cdn breaks down again. # https://raw.githubusercontent.com/plotly/plotly.js/master/dist/plotly.min.js 'https://cdn.plot.ly/plotly-latest.min.js': 'plotly-plotly.min.js', # Stanford Javascript Crypto Library for Password Hashing '%[email protected]/sjcl.js' % b: 'sjcl.js', # - css '%[email protected]/css/styles.css' % b: 'react-resizable-styles.css', # noqa '%[email protected]/css/styles.css' % b: 'react-grid-layout-styles.css', # noqa '%scss/bootstrap.min.css' % bb: 'bootstrap.min.css', # - fonts '%[email protected]' % b: 'classnames', '%[email protected]' % b: 'layout_bin_packer', '%sfonts/glyphicons-halflings-regular.eot' % bb: 'glyphicons-halflings-regular.eot', '%sfonts/glyphicons-halflings-regular.woff2' % bb: 'glyphicons-halflings-regular.woff2', '%sfonts/glyphicons-halflings-regular.woff' % bb: 'glyphicons-halflings-regular.woff', '%sfonts/glyphicons-halflings-regular.ttf' % bb: 'glyphicons-halflings-regular.ttf', '%sfonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular' % bb: # noqa 'glyphicons-halflings-regular.svg#glyphicons_halflingsregular', } # make sure all relevant folders exist: dir_list = [ '%s' % install_dir, '%s/static' % install_dir, '%s/static/js' % install_dir, '%s/static/css' % install_dir, '%s/static/fonts' % install_dir, ] for directory in dir_list: if not os.path.exists(directory): os.makedirs(directory) # set up proxy handler: from six.moves.urllib import request from six.moves.urllib.error import HTTPError, URLError handler = request.ProxyHandler(proxies) if proxies is not None \ else request.BaseHandler() opener = request.build_opener(handler) request.install_opener(opener) built_path = os.path.join(here, 'static/version.built') is_built = visdom.__version__ == 'no_version_file' if os.path.exists(built_path): with open(built_path, 'r') as build_file: build_version = build_file.read().strip() if build_version == visdom.__version__: is_built = True else: os.remove(built_path) # download files one-by-one: for (key, val) in ext_files.items(): # set subdirectory: sub_dir = 'fonts' if '.js' in key: sub_dir = 'js' if '.css' in key: sub_dir = 'css' # download file: filename = '%s/static/%s/%s' % (install_dir, sub_dir, val) if not os.path.exists(filename) or not is_built: req = request.Request(key, headers={'User-Agent': 'Chrome/30.0.0.0'}) try: data = opener.open(req).read() with open(filename, 'wb') as fwrite: fwrite.write(data) except HTTPError as exc: logging.error('Error {} while downloading {}'.format( exc.code, key)) except URLError as exc: logging.error('Error {} while downloading {}'.format( exc.reason, key)) if not is_built: with open(built_path, 'w+') as build_file: build_file.write(visdom.__version__)
def _get(url, param_dict={}, securityHandler=None, additional_headers=[], handlers=[], proxy_url=None, proxy_port=None, compress=True, custom_handlers=[], out_folder=None, file_name=None): """ Performs a GET operation Inputs: Output: returns dictionary, string or None """ CHUNK = 4056 param_dict, handler, cj = _processHandler(securityHandler, param_dict) headers = [] + additional_headers if compress: headers.append(('Accept-encoding', 'gzip')) else: headers.append(('Accept-encoding', '')) headers.append(('User-Agent', _useragent)) if len(param_dict.keys()) == 0: param_dict = None if handlers is None: handlers = [] if handler is not None: handlers.append(handler) handlers.append(RedirectHandler()) if cj is not None: handlers.append(request.HTTPCookieProcessor(cj)) if proxy_url is not None: if proxy_port is None: proxy_port = 80 proxies = {"http":"http://%s:%s" % (proxy_url, proxy_port), "https":"https://%s:%s" % (proxy_url, proxy_port)} proxy_support = request.ProxyHandler(proxies) handlers.append(proxy_support) opener = request.build_opener(*handlers) opener.addheaders = headers if param_dict is None: resp = opener.open(url, data=param_dict) elif len(str(urlencode(param_dict))) + len(url) >= 1999: resp = opener.open(url, data=urllib.urlencode(param_dict)) else: format_url = url + "?%s" % urlencode(param_dict) resp = opener.open(fullurl=format_url) # Get some headers from the response maintype = _mainType(resp) contentDisposition = resp.headers.get('content-disposition') contentEncoding = resp.headers.get('content-encoding') contentType = resp.headers.get('content-Type').split(';')[0].lower() contentLength = resp.headers.get('content-length') if maintype.lower() in ('image', 'application/x-zip-compressed') or \ contentType == 'application/x-zip-compressed' or \ (contentDisposition is not None and \ contentDisposition.lower().find('attachment;') > -1): # "application", fname = _get_file_name( contentDisposition=contentDisposition, url=url) if out_folder is None: out_folder = tempfile.gettempdir() if contentLength is not None: max_length = int(contentLength) if max_length < CHUNK: CHUNK = max_length file_name = os.path.join(out_folder, fname) with open(file_name, 'wb') as writer: for data in _chunk(response=resp, size=CHUNK): writer.write(data) writer.flush() writer.flush() del writer return file_name else: read = "" for data in _chunk(response=resp, size=CHUNK): try: read += data.decode('ascii') except: read += data.decode('utf-8') del data try: results = json.loads(read) if 'error' in results: if 'message' in results['error']: if results['error']['message'] == 'Request not made over ssl': if url.startswith('http://'): url = url.replace('http://', 'https://') return _do_get(url, param_dict, securityHandler, additional_headers, handlers, proxy_url, proxy_port, compress, custom_handlers, out_folder, file_name) return results except: return read
import codecs import posixpath from os import path import re from six import iteritems from six.moves.urllib import request from docutils import nodes from docutils.utils import relative_path import sphinx from sphinx.locale import _ from sphinx.builders.html import INVENTORY_FILENAME handlers = [ request.ProxyHandler(), request.HTTPRedirectHandler(), request.HTTPHandler() ] try: handlers.append(request.HTTPSHandler) except AttributeError: pass request.install_opener(request.build_opener(*handlers)) UTF8StreamReader = codecs.lookup('utf-8')[2] def read_inventory_v1(f, uri, join): f = UTF8StreamReader(f)
def download_scripts(proxies=None, install_dir=None): print("Downloading scripts. It might take a while.") # location in which to download stuff: if install_dir is None: import visdom install_dir = os.path.dirname(visdom.__file__) # all files that need to be downloaded: b = 'https://unpkg.com/' bb = '%[email protected]/dist/' % b ext_files = { '%[email protected]/dist/jquery.min.js' % b: 'jquery.min.js', '%[email protected]/dist/js/bootstrap.min.js' % b: 'bootstrap.min.js', '%[email protected]/css/styles.css' % b: 'react-resizable-styles.css', '%[email protected]/css/styles.css' % b: 'react-grid-layout-styles.css', '%[email protected]/dist/react-modal.min.js' % b: 'react-modal.min.js', '%[email protected]/dist/react.min.js' % b: 'react-react.min.js', '%[email protected]/dist/react-dom.min.js' % b: 'react-dom.min.js', '%[email protected]' % b: 'classnames', '%[email protected]' % b: 'layout_bin_packer', 'https://raw.githubusercontent.com/STRML/react-grid-layout/0.14.0/dist/' + 'react-grid-layout.min.js': 'react-grid-layout.min.js', 'https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_SVG': 'mathjax-MathJax.js', # here is another url in case the cdn breaks down again. # https://raw.githubusercontent.com/plotly/plotly.js/master/dist/plotly.min.js 'https://cdn.plot.ly/plotly-latest.min.js': 'plotly-plotly.min.js', '%scss/bootstrap.min.css' % bb: 'bootstrap.min.css', '%sfonts/glyphicons-halflings-regular.eot' % bb: 'glyphicons-halflings-regular.eot', '%sfonts/glyphicons-halflings-regular.woff2' % bb: 'glyphicons-halflings-regular.woff2', '%sfonts/glyphicons-halflings-regular.woff' % bb: 'glyphicons-halflings-regular.woff', '%sfonts/glyphicons-halflings-regular.ttf' % bb: 'glyphicons-halflings-regular.ttf', '%sfonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular' % bb: 'glyphicons-halflings-regular.svg#glyphicons_halflingsregular', } # make sure all relevant folders exist: dir_list = [ '%s' % install_dir, '%s/static' % install_dir, '%s/static/js' % install_dir, '%s/static/css' % install_dir, '%s/static/fonts' % install_dir, ] for directory in dir_list: if not os.path.exists(directory): os.makedirs(directory) # set up proxy handler: from six.moves.urllib import request from six.moves.urllib.error import HTTPError, URLError handler = request.ProxyHandler(proxies) if proxies is not None \ else request.BaseHandler() opener = request.build_opener(handler) request.install_opener(opener) # download files one-by-one: for (key, val) in ext_files.items(): # set subdirectory: sub_dir = 'fonts' if '.js' in key: sub_dir = 'js' if '.css' in key: sub_dir = 'css' # download file: filename = '%s/static/%s/%s' % (install_dir, sub_dir, val) if not os.path.exists(filename): req = request.Request(key, headers={'User-Agent': 'Chrome/30.0.0.0'}) try: data = opener.open(req).read() with open(filename, 'wb') as fwrite: fwrite.write(data) except HTTPError as exc: logging.error('Error {} while downloading {}'.format( exc.code, key)) except URLError as exc: logging.error('Error {} while downloading {}'.format( exc.reason, key))
print('If you get error "ImportError: No module named \'six\'" install six:\n'+\ '$ sudo pip install six') print('To enable your free eval account and get CUSTOMER, YOURZONE and ' + \ 'YOURPASS, please contact [email protected]') import sys import ssl ssl._create_default_https_context = ssl._create_unverified_context if sys.version_info[0] == 2: import six from six.moves.urllib import request opener = request.build_opener( request.ProxyHandler({ 'http': 'http://*****:*****@zproxy.lum-superproxy.io:22225', 'https': 'http://*****:*****@zproxy.lum-superproxy.io:22225' })) content = opener.open( 'https://www.gofundme.com/f/rich-de-croce-memorial-fund').read() with open('test.html', 'w') as file: file.write(str(content)) print('Done.') if sys.version_info[0] == 3: import urllib.request opener = urllib.request.build_opener( urllib.request.ProxyHandler({ 'http':