def __init__(self, _=None):
"""Build self._urllib_opener"""
proxy = super().__str__()
if proxy == "None":
self._urllib_opener = build_opener()
return
components = list(re.match(self._match_regexp, proxy).groups())
self.scheme, self.host, self.port = components
self.components = components
if self.scheme == "socks4":
socks4_handler = SocksiPyHandler(socks.PROXY_TYPE_SOCKS4,
self.host,
int(self.port))
self._urllib_opener = build_opener(socks4_handler)
elif self.scheme == "socks5":
socks5_handler = SocksiPyHandler(socks.PROXY_TYPE_SOCKS5,
self.host,
int(self.port))
self._urllib_opener = build_opener(socks5_handler)
else:
proxy_handler = ProxyHandler({'http': proxy, 'https': proxy})
self._urllib_opener = build_opener(proxy_handler)
def _create_opener(self): opener = urllibrequest.build_opener() if self.useProxy: if self.proxy_protocol == "socks4": opener = urllibrequest.build_opener(SocksiPyHandler(socks.SOCKS4, self.proxy_host, self.proxy_port, username=self.proxy_username, password=self.proxy_password)) if self.proxy_protocol == "socks5": opener = urllibrequest.build_opener(SocksiPyHandler(socks.SOCKS5, self.proxy_host, self.proxy_port, username=self.proxy_username, password=self.proxy_password)) return opener
def open(self, url, data=None, headers=None, cookies=None, timeout=60):
self.log.debug('%s request to %s\nHeaders: %s\nTimeout: %s',
'GET' if data else 'POST', url, headers, timeout)
cookies = cookies or http.cookiejar.LWPCookieJar()
opener = urllib.request.build_opener(
SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, self.host,
self.socks_port),
urllib.request.HTTPCookieProcessor(cookies))
status_code = None
if headers:
opener.addheaders = [item for item in list(headers.items())]
try:
response = opener.open(url, data, timeout)
except http.client.IncompleteRead as e:
response = e.partial
except Exception as e:
if hasattr(e, 'code'):
status_code = e.code
self.log.debug('Failed to open %s, %s', url, e)
return Response(None, None, cookies)
try:
text = response.read()
except Exception as e:
text = None
self.log.debug('Unable to read response content: %r', e)
try:
status_code = status_code or response.getcode()
except Exception as e:
status_code = None
self.log.debug('Error during getting response status code: %r', e)
return Response(text, status_code, cookies)
def check_proxy(proxy, p=False):
"""Checks the status of the proxy object.
Args:
proxy: A proxy to check.
p: A boolean variable that indicates whether to print or return the value.
Returns: The status of the proxy. If there was a failure,
the reason is given back to the caller as an exception string.
"""
if isinstance(proxy, dict):
proxy = Proxy(proto=proxy['proto'],
host=proxy['host'],
port=proxy['port'],
username=proxy['username'],
password=proxy['password'])
try:
opener = urllib2.build_opener(
SocksiPyHandler(scheme_mapping[proxy.proto], proxy.host,
int(proxy.port)))
opener.open("http://httpbin.org/ip").read()
except socks.ProxyConnectionError as e:
if p:
print '{} is \033[91m offline \033[0m'.format(proxy)
else:
return ('offline', str(e))
else:
if p:
print '{} is \033[91m online \033[0m'.format(proxy)
else:
return 'online'
def api(self, command, args=None):
"""
Main API function.
Returns:
returns 'False' if invalid command or if no APIKey or Secret
is specified (if command is "private").
returns {"error":"<error message>"} if API error.
"""
logging.info("Poloniex command: {}, args: {}".format(command, args))
args = args or {}
if command in PUBLIC_COMMANDS:
url = 'https://poloniex.com/public?'
args['command'] = command
# prevent urllib from complaining when using a proxy
context = ssl.create_default_context()
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
opener = build_opener(
SocksiPyHandler(socks.PROXY_TYPE_SOCKS5,
constants.PROXY_ADDR,
constants.PROXY_PORT,
True,
context=context))
ret = opener.open(Request(url + urlencode(args)))
return json.loads(ret.read().decode(encoding='UTF-8'))
else:
return False
def down_web_page_html(url, headers=None, proxy=None, retry=3, data=None):
print('下载如下链接：', url)
# headers = {'User-agent': user_agent}
request = urllib.request.Request(
url,
data=urllib.parse.urlencode(data).encode('utf-8'),
headers=headers)
handler = urllib.request.HTTPCookieProcessor()
opener = urllib.request.build_opener(handler)
if proxy:
# proxy_params = {urllib.parse.urlparse(url).scheme: proxy}
proxy_params = {'https': proxy}
# opener.add_handler(urllib.request.ProxyHandler(proxy_params))
opener.add_handler(SocksiPyHandler(socks.SOCKS5, "127.0.0.1",
1080))
try:
response = opener.open(request)
html = response.read()
code = response.code
except urllib.request.URLError as e:
print('下载遇到错误：', e.reason)
html = ''
if hasattr(e, 'code'):
code = e.code
if retry > 0 and 400 <= code <= 600:
retry -= 1
time.sleep(3)
WebPageDown.down_web_page_html(url, headers, proxy,
retry - 1)
else:
code = None
return {'html': html, 'code': code}
def Test_SS_speed(self, ss_list):
result_list = []
os.system('pkill sslocal')
for i in range(len(ss_list)):
server_port = "108" + str(i)
command = "sslocal -s " + "\"" + ss_list[i][
'server'] + "\"" + " -p " + "\"" + ss_list[i][
'server_port'] + "\"" + " -l " + server_port + " -k " + "\"" + ss_list[
i]['password'] + "\"" + " -m " + "\"" + ss_list[i][
'method'] + "\"" + " 2>1 &"
# print command
os.system(command)
time.sleep(2)
opener = urllib2.build_opener(
SocksiPyHandler(socks.SOCKS5, "127.0.0.1", int(server_port)))
start_time = time.time()
try:
if opener.open('https://www.tumblr.com/').getcode() == 200:
end_time = time.time()
speed = end_time - start_time
else:
speed = 10000000
except Exception:
speed = 10000000
pass
ss_list[i]['speed'] = speed
result_list.append(ss_list[i])
time.sleep(1)
os.system("pkill sslocal")
return result_list
def download_image(url, save_name, proxy=False):
try:
fp = ''
if proxy is True:
opener = urllib2.build_opener(
SocksiPyHandler(socks.SOCKS5, "127.0.0.1", 9999))
fp = opener.open(url, timeout=40)
else:
fp = urllib2.urlopen(url, timeout=40)
data = fp.read()
fp.close()
print url + ' downloading...'
fid = open(save_name, 'w+b')
fid.write(data)
flag = True
flsize = os.path.getsize(save_name)
print 'file size:%d ' % flsize
if flsize < 10:
flag = False
######
except Exception:
print url + ' downloading io error... ', sys.exc_info()[0]
flag = False
return flag
def checkproxy(port, type):
if type == 'http':
handler = socks.HTTP
elif type == 'socks':
handler = socks.SOCKS5
elif type == 'http0':
proxy = urllib.request.ProxyHandler(
{'http': PROXYHOST + ':' + str(port)})
else:
print("[unsupported type]")
return
if type == 'http0':
opener = urllib.request.build_opener(proxy)
else:
opener = urllib.request.build_opener(
SocksiPyHandler(handler, PROXYHOST, port))
try:
response = opener.open(TESTURI, timeout=TIMEOUT).read()
except Exception as e:
print("[%5s]" % ('dead'))
return
print("[%5s]" % ('alive'))
def AuthorizeTokens(client_id, client_secret, authorization_code, redirect_uri, google_account_base_url):
"""Obtains OAuth access token and refresh token.
This uses the application portion of the "OAuth2 for Installed Applications"
flow at https://developers.google.com/accounts/docs/OAuth2InstalledApp#handlingtheresponse
Args:
client_id: Client ID obtained by registering your app.
client_secret: Client secret obtained by registering your app.
authorization_code: code generated by Google Accounts after user grants
permission.
Returns:
The decoded response from the Google Accounts server, as a dict. Expected
fields include 'access_token', 'expires_in', and 'refresh_token'.
"""
params = {}
params['client_id'] = client_id
params['client_secret'] = client_secret
params['code'] = authorization_code
params['redirect_uri'] = redirect_uri
params['grant_type'] = 'authorization_code'
request_url = AccountsUrl(command='o/oauth2/token', google_account_base_url=google_account_base_url)
print request_url, redirect_uri
from sockshandler import SocksiPyHandler
import socks
opener = urllib2.build_opener(SocksiPyHandler(socks.SOCKS5, "localhost", proxyport=1080))
response = opener.open(request_url, urllib.urlencode(params)).read()
print response
return json.loads(response)
def test_socks_connection(url, proxy_host, proxy_port):
"""Return HTTP code from opening URL via SOCKS proxy"""
opener = urllib.request.build_opener(
SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, proxy_host, proxy_port))
resp = opener.open(url)
try:
return resp.code
finally:
resp.close()
def get_page(ResRefererEx):
global PK_status
global htmlshash
# Use a proxy if declared in config file
try:
# Use a HTTP proxy
if proxy_type in 'http':
http_proxy
proxy_support = urllib.request.ProxyHandler({'http': http_proxy})
opener = urllib.request.build_opener(proxy_support)
urllib.request.install_opener(opener)
# Use a SOCKS5 proxy
elif proxy_type in 'socks':
socks_proxy_server
socks_proxy_port
opener = urllib.request.build_opener(
SocksiPyHandler(socks.SOCKS5, socks_proxy_server,
int(socks_proxy_port), True))
urllib.request.install_opener(opener)
except NameError:
pass
try:
request = urllib.request.Request(
url=ResRefererEx,
# Force user-agent
headers={
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'
})
response = urllib.request.urlopen(request, timeout=5)
resp_code = response.getcode()
htmldata = str(response.read().decode('utf-8'))
if resp_code == 200:
try:
# If page contains tracking_file_request
if RegRequest2.finditer(htmldata):
PK_status = 'UP'
# Create SHA256 hash of HTML page content
htmlshash = hashlib.sha256(
htmldata.encode('utf-8')).hexdigest()
else:
PK_status = 'Probably removed'
except:
err = sys.exc_info()
print(err)
pass
else:
PK_status = 'DOWN'
except:
#err = sys.exc_info()[1]
err = sys.exc_info()[1]
PK_status = ('can\'t connect (' + str(err) + ')')
pass
def check_proxy(pip, timeout):
try:
opener = urllib.request.build_opener(
SocksiPyHandler(socks.SOCKS5, pip[0:pip.find(":")],
int(pip[pip.find(":") + 1:len(pip)])))
opener.open("https://api.ipify.org", timeout=timeout)
proxys.add(pip)
except Exception as e:
badproxys.add(pip)
return proxys
def request_webpage_via_ss_proxy(url, proxy):
"""
使用SOCKS5代理进行http请求
proxy[0] 代理IP
proxy[1] 代理端口
"""
import socks
from sockshandler import SocksiPyHandler
opener = urllib.request.build_opener(SocksiPyHandler(socks.SOCKS5, proxy[0], int(proxy[1])))
return opener.open(url, timeout=6)
def build_urllib2_proxy_handler(url):
handler = None
if url:
m = re.match(
r'^(?P<type>\w+)://(?P<ip>\d+\.\d+\.\d+\.\d+)(:(?P<port>\d+))?',
url)
if m and m.group('type').startswith('socks5'):
handler = SocksiPyHandler(
socks.SOCKS5, m.group('ip'),
int(m.group('port')) if m.group('port') else 1080)
return handler
def test_server(port):
opener = urllib2.build_opener(SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, 'localhost', port))
try:
st = time.time()
content = opener.open("https://www.youtube.com", timeout=10).read()
if not content:
return -1
else:
return time.time() - st
except:
return -1
def test_access_2():
import urllib
import socks
from sockshandler import SocksiPyHandler
try:
opener = urllib.request.build_opener(SocksiPyHandler(socks.SOCKS5, "127.0.0.1", 1080))
x = opener.open("http://www.google.com/", timeout=5)
return x.getcode()
except:
return -1
def __init__(self, port_offset):
self.port_offset = int(port_offset)
if not 0 <= self.port_offset < 1000000:
raise Error('Invalid port_offset %s' % self.port_offset)
self.socks_port = base_socks_port + self.port_offset
self.control_port = base_control_port + self.port_offset
self.path = tors_dir + "/" + str(self.port_offset)
self.opener = None
makedir(self.path)
self.opener = urllib2.build_opener(
SocksiPyHandler(socks.SOCKS5, "127.0.0.1", self.socks_port))
def change_proxy():
if USE_PROXY and PROXY_PORTS:
print("Changing proxy..")
port = random.choice(PROXY_PORTS)
debug("Using proxy port #%d" % port)
socks_handler = SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, PROXY_HOST,
port)
opener = urllib.request.build_opener(socks_handler)
urllib.request.install_opener(opener)
return port
return None
def latency(url):
"""测试三个外网的get的平均延时"""
t_start = time.time()
try:
opener = urllib2.build_opener(
SocksiPyHandler(socks.SOCKS5, "127.0.0.1", 1080))
time.sleep(2)
s = opener.open(url)
# print s.read()
t_end = time.time()
return t_end - t_start
except:
return 9999999
def getJsonReponse(tweetCriteria, refreshCursor, cookieJar):
url = "https://twitter.com/i/search/timeline?f=realtime&q=%s&src=typd&max_position=%s"
urlGetData = ''
if hasattr(tweetCriteria, 'querySearch'):
urlGetData += tweetCriteria.querySearch
if hasattr(tweetCriteria, 'username'):
urlGetData += ' from:' + tweetCriteria.username
if hasattr(tweetCriteria, 'since'):
urlGetData += ' since:' + tweetCriteria.since
if hasattr(tweetCriteria, 'until'):
urlGetData += ' until:' + tweetCriteria.until
if hasattr(tweetCriteria, 'topTweets'):
if tweetCriteria.topTweets:
url = "https://twitter.com/i/search/tweets?q=%s&src=typd&max_position=%s"
url = url % (urllib.quote(urlGetData), refreshCursor)
headers = [('Host', "twitter.com"),
('User-Agent', "Mozilla/6.0 (Windows NT 6.1; Win64; x64)"),
('Accept',
"application/json, text/javascript, */*; q=0.01"),
('Accept-Language', "de,en-US;q=0.7,en;q=0.3"),
('X-Requested-With', "XMLHttpRequest"), ('Referer', url),
('Connection', "keep-alive")]
opener = urllib2.build_opener(
urllib2.HTTPCookieProcessor(cookieJar),
SocksiPyHandler(socks.SOCKS5, "127.0.0.1", 1080))
opener.addheaders = headers
try:
response = opener.open(url)
jsonResponse = response.read()
except Exception as e:
if e == KeyboardInterrupt:
sys.exit()
else:
return None
try:
dataJson = json.loads(jsonResponse)
except:
return []
return dataJson
def attack2(host1, port):
try:
opener = urllib2.build_opener(
SocksiPyHandler(socks.SOCKS5, "127.0.0.1", port, True))
opener.addheaders = [('User-Agent', random.choice(useragents))]
opener.addheaders = [('Cache-Control', 'no-cache')]
opener.addheaders = [('Accept-Charset',
'ISO-8859-1,utf-8;q=0.7,*;q=0.7')]
opener.addheaders = [('Referer',
random.choice(referers) + str(randstr))]
opener.addheaders = [('Keep-Alive', random.randint(110, 120))]
opener.addheaders = [('Connection', 'keep-alive')]
opener.addheaders = [('Host', host1)]
except:
print(' Error2')
def __setProxy4Urllib(self):
if self.proxy_flag is False and self.__proxyConfigExists():
# set proxy for urllib2
self.log.info('setting proxy %s %s:%s', protocol, server, port)
from sockshandler import SocksiPyHandler
global proxy
proxy = SocksiPyHandler(protocol, server, int(port))
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
self.proxy_flag = True
else:
self.log.info(
'Proxy has bean configured: protocol=%d server=%s port=%s',
protocol, server, port)
def test_proxy_by_urllib2():
import urllib2
import socks
from sockshandler import SocksiPyHandler
headers = {'Host': 'app-api.pixiv.net', 'User-Agent': 'PixivIOSApp/6.0.9', 'Accept-Language': 'zh',
'Authorization': 'Bearer %s' % 'uMzgoLH6TysDAvu8594IvustucSHz0hwThlMYG3uHY0', 'Accept-Encoding': 'gzip'}
httpHandler = urllib2.HTTPHandler(debuglevel=1)
httpsHandler = urllib2.HTTPSHandler(debuglevel=1)
opener = urllib2.build_opener(httpHandler, httpsHandler, SocksiPyHandler(socks.SOCKS5, "127.0.0.1", 1080))
urllib2.install_opener(opener)
req = urllib2.Request("https://app-api.pixiv.net/v1/illust/detail?illust_id=55418",
headers=headers)
x = urllib2.urlopen(req)
print (x.read())
def crawler(self, counter):
ip = ''
port = 1080
url = ''
filename = "images/MJ%d.jpg" % counter
opener = urllib2.build_opener(SocksiPyHandler(socks.SOCKS5, ip, port))
opener.addheaders = [(
'User-Agent',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36'
)]
response = opener.open(url)
htmlData = response.read()
f = open(filename, 'w')
f.write(htmlData)
f.close()
return filename
def install_proxy(host, port, proxy_type=socks.HTTP):
"""
:param host: proxy host
:param port: proxy ip
:param proxy_type: proxy type
SOCKS4 = 1
SOCKS5 = 2
HTTP = 3
"""
if proxy_type not in (socks.SOCKS5, socks.SOCKS4, socks.HTTP):
raise RuntimeError("Not support proxy type")
proxy = urllib2.ProxyHandler({'http': host + ":" + port}) if proxy_type == 1 \
else SocksiPyHandler(proxy_type, host, port)
opener = urllib2.build_opener(urllib2.HTTPHandler(debuglevel=1),
urllib2.HTTPSHandler(debuglevel=1),
proxy)
urllib2.install_opener(opener)
def __loadCookies(self):
if self.alt_cookiejar is not None:
self.alt_cookiejar.init_agent(new_headers=self.browserHeaders)
self.cj = self.alt_cookiejar
else:
self.cj = http.cookiejar.LWPCookieJar(
) # This is a subclass of FileCookieJar
# that has useful load and save methods
if self.cj is not None:
if os.path.isfile(self.COOKIEFILE):
try:
self.__updateCookiesFromFile()
# self.log.info("Loading CookieJar")
except:
self.log.critical("Cookie file is corrupt/damaged?")
try:
os.remove(self.COOKIEFILE)
except FileNotFoundError:
pass
if http.cookiejar is not None:
# self.log.info("Installing CookieJar")
self.log.debug(self.cj)
cookieHandler = urllib.request.HTTPCookieProcessor(self.cj)
args = (cookieHandler, Handlers.HTTPRedirectHandler)
if self.credHandler:
print("Have cred handler. Building opener using it")
args += (self.credHandler, )
if self.use_socks:
print("Using Socks handler")
if not HAVE_SOCKS:
raise RuntimeError(
"SOCKS Use specified, and no socks installed!")
args = (SocksiPyHandler(socks.SOCKS5, "127.0.0.1",
9050), ) + args
self.opener = urllib.request.build_opener(*args)
#self.opener.addheaders = [('User-Agent', 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)')]
self.opener.addheaders = self.browserHeaders
#urllib2.install_opener(self.opener)
for cookie in self.cj:
self.log.debug(cookie)
def parse_proxy_url(proxy_url):
proto = None
host = None
port = None
login = None
password = None
if not proxy_url:
return
proto_parsed = proxy_url.split("://")[0].lower()
if proto_parsed == "socks5":
proto = socks.SOCKS5
elif proto_parsed == "socks4":
proto = socks.SOCKS4
elif proto_parsed == "http":
proto = socks.HTTP
elif proto_parsed == "https":
proto = socks.HTTP
host_string = proxy_url.split("://")[1]
if '@' in host_string:
ary = host_string.split("@")
user_string = ary[0]
host_string = ary[1]
ary = user_string.split(":")
login = ary[0]
password = ary[1]
if host_string:
ary = host_string.split(":")
host = ary[0]
port = ary[1]
return SocksiPyHandler(proxytype=proto,
proxyaddr=host,
proxyport=int(port),
username=login,
password=password,
rdns=True)
def get_opener(config):
proxy = config.get('proxy')
if not proxy:
return urllib2.build_opener()
url = urlparse.urlparse(proxy)
if url.scheme == 'socks5':
m = re.match(r'^([^:]+):([^@]+)@([^:]+):(.+)$', url.netloc)
if m is None:
print('Proxy URL should look like this: socks5://user:password@host:port', file=sys.stderr)
exit(2)
user, passwd, host, port = m.groups()
try:
from sockshandler import SocksiPyHandler
opener = urllib2.build_opener(SocksiPyHandler(socks.SOCKS5, host, int(port), True, user, passwd))
return opener
except ImportError:
print('Install PySocks to use a socks5 proxy.', file=sys.stderr)
exit(5)
print('Unsupported proxy type: %s' % url.scheme, file=sys.stderr)
exit(3)
def get_url_title(url):
enc = ['utf8', 'iso-8869-1', 'shift-jis']
title = ''
headers = {
'User-Agent': USER_AGENT
}
try:
req = urllib.request.Request(url, data=None, headers=headers)
host = urllib.request.urlparse(url).netloc.split(':')[0]
if PROXY_HOST and PROXY_PORT and re.match(PROXY_REGEX, host):
opener = urllib.request.build_opener(SocksiPyHandler(socks.SOCKS5, PROXY_HOST, PROXY_PORT))
resp = opener.open(req, timeout=5)
else:
resp = urllib.request.urlopen(req, timeout=5)
except urllib.error.HTTPError as e:
return 'Status: ' + str(e.code)
except urllib.error.URLError as e:
return 'Error: ' + str(e.reason)
except Exception as e:
return
# get the site's title, only in html content
if 'content-type' in resp.headers and 'html' in resp.headers['content-type']:
# try the charset set in the html header first, if there is one
if 'charset=' in resp.headers['content-type']:
enc = enc + [resp.headers['content-type'].split('charset=')[-1]]
# read up to 1mb
chunk = resp.read(1024 * 1024)
parser = TitleParser()
for e in enc:
try:
decoded_chunk = chunk.decode(e, 'ignore')
parser.feed(decoded_chunk)
if parser.done:
title = parser.title
parser.close()
if len(title) > 0:
esc = parser.unescape(title)
return 'Title: ' + esc.strip()
except Exception as ex:
pass
# no title, try to output some other useful data
info = []
if 'content-type' in resp.headers:
info.append('Type: ' + resp.headers['content-type'].split(';')[0])
if 'content-length' in resp.headers:
info.append('Size: ' + humanify(int(resp.headers['content-length'])))
if 'last-modified' in resp.headers:
d = resp.headers['last-modified']
try:
parsed_date = dateutil.parser.parse(d)
d = parsed_date.strftime('%F %T') + ' ' + parsed_date.tzname()
except ValueError:
pass
info.append('Modified: ' + d)
return ', '.join(info)