def session_request(self,
method,
url,
params=None,
data=None,
headers=None,
cookies=None,
files=None,
auth=None,
timeout=conf.timeout if 'timeout' in conf else None,
allow_redirects=True,
proxies=None,
hooks=None,
stream=None,
verify=False,
cert=None,
json=None):
# Create the Request.
merged_cookies = merge_cookies(
merge_cookies(RequestsCookieJar(), self.cookies),
cookies or conf.cookie if 'cookie' in conf else None)
req = Request(
method=method.upper(),
url=url,
headers=merge_setting(
headers, conf.http_headers if 'http_headers' in conf else {}),
files=files,
data=data or {},
json=json,
params=params or {},
auth=auth,
cookies=merged_cookies,
hooks=hooks,
)
prep = self.prepare_request(req)
proxies = proxies or conf.proxies if 'proxies' in conf else {}
settings = self.merge_environment_settings(prep.url, proxies, stream,
verify, cert)
# Send the request.
send_kwargs = {
'timeout': timeout,
'allow_redirects': allow_redirects,
}
send_kwargs.update(settings)
resp = self.send(prep, **send_kwargs)
if resp.encoding == 'ISO-8859-1':
encodings = get_encodings_from_content(resp.text)
if encodings:
encoding = encodings[0]
else:
encoding = resp.apparent_encoding
resp.encoding = encoding
return resp
def test_precedence(self):
content = '''
<?xml version="1.0" encoding="XML"?>
<meta charset="HTML5">
<meta http-equiv="Content-type" content="text/html;charset=HTML4" />
'''.strip()
assert get_encodings_from_content(content) == ['HTML5', 'HTML4', 'XML']
def guess_response_encoding(resp):
'''
Guess the content encoding of a requests response.
Note: there's a performance issue due to chardet.
'''
# first try the encoding supplied by responce header and content
encs = get_encodings_from_content(resp.content) or []
for enc in encs:
try:
resp.content.decode(enc)
LOG.info('Detected encoding %s from response content.', enc)
return enc
except UnicodeDecodeError:
LOG.debug('Encoding from response content doesn\'t work.')
enc = get_encoding_from_headers(resp.headers)
if enc:
try:
resp.content.decode(enc)
LOG.info('Detected encoding %s from response header.', enc)
return enc
except UnicodeDecodeError:
LOG.debug('Encoding from response header doesn\'t work.')
# neither encoding works, we have to go the hard way.
start = clock()
g = detect(resp.content)
LOG.info('Detected encoding %s with cofidence of %g in %gs.' % (g['encoding'], g['confidence'], clock() - start))
return g['encoding']
def test_precedence(self):
content = '''
<?xml version="1.0" encoding="XML"?>
<meta charset="HTML5">
<meta http-equiv="Content-type" content="text/html;charset=HTML4" />
'''.strip()
assert get_encodings_from_content(content) == ['HTML5', 'HTML4', 'XML']
def encoding(rsp):
"""
encoding of Response.content.
if Response.encoding is None, encoding will be guessed
by header or content or chardet if avaibable.
"""
# content is unicode
if isinstance(rsp.content, six.text_type):
return 'unicode'
# Try charset from content-type
encoding = get_encoding_from_headers(rsp.headers)
if encoding == 'ISO-8859-1':
encoding = None
# Try charset from content
if not encoding and get_encodings_from_content:
encoding = get_encodings_from_content(rsp.content)
encoding = encoding and encoding[0] or None
# Fallback to auto-detected encoding.
if not encoding and chardet is not None:
encoding = chardet.detect(rsp.content)['encoding']
if encoding and encoding.lower() == 'gb2312':
encoding = 'gb18030'
encoding = encoding or 'utf-8'
return encoding
def encoding(self):
if hasattr(self, "_encoding"):
return self._encoding
# content is unicode
if isinstance(self.content, unicode):
return "unicode"
# Try charset from content-type
encoding = get_encoding_from_headers(self.headers)
if encoding == "ISO-8859-1":
encoding = None
# Try charset from content
if not encoding:
encoding = get_encodings_from_content(self.content)
encoding = encoding and encoding[0] or None
# Fallback to auto-detected encoding.
if not encoding and chardet is not None:
encoding = chardet.detect(self.content)["encoding"]
if encoding and encoding.lower() == "gb2312":
encoding = "gb18030"
self._encoding = encoding or "utf-8"
return self._encoding
def find_encoding(content, headers=None):
# content is unicode
if isinstance(content, str):
return 'utf-8'
encoding = None
# Try charset from content-type
if headers:
encoding = get_encoding_from_headers(headers)
if encoding == 'ISO-8859-1':
encoding = None
# Fallback to auto-detected encoding.
if not encoding and chardet is not None:
encoding = chardet.detect(content)['encoding']
# Try charset from content
if not encoding:
try:
encoding = get_encodings_from_content(content)
encoding = encoding and encoding[0] or None
except:
if isinstance(content, bytes):
return encoding or 'utf-8'
if encoding and encoding.lower() == 'gb2312':
encoding = 'gb18030'
return encoding or 'latin_1'
def encoding(self):
if hasattr(self, '_encoding'):
return self._encoding
# content is unicode
if isinstance(self.content, unicode):
return 'unicode'
# Try charset from content-type
encoding = get_encoding_from_headers(self.headers)
if encoding == 'ISO-8859-1':
encoding = None
# Try charset from content
if not encoding:
encoding = get_encodings_from_content(self.content)
encoding = encoding and encoding[0] or None
# Fallback to auto-detected encoding.
if not encoding and chardet is not None:
encoding = chardet.detect(self.content)['encoding']
if encoding and encoding.lower() == 'gb2312':
encoding = 'gb18030'
self._encoding = encoding or 'utf-8'
return self._encoding
def encoding(self):
if hasattr(self, '_encoding'):
return self._encoding
# content is unicode
if isinstance(self.content, unicode):
return 'unicode'
# Try charset from content-type
encoding = get_encoding_from_headers(self.headers)
if encoding == 'ISO-8859-1':
encoding = None
# Try charset from content
if not encoding:
encoding = get_encodings_from_content(self.content)
encoding = encoding and encoding[0] or None
# Fallback to auto-detected encoding.
if not encoding and chardet is not None:
encoding = chardet.detect(self.content)['encoding']
if encoding and encoding.lower() == 'gb2312':
encoding = 'gb18030'
self._encoding = encoding or 'utf-8'
return self._encoding
def find_encoding(content, headers=None):
# content is unicode
if isinstance(content, unicode):
return 'unicode'
encoding = None
# Try charset from content-type
if headers:
encoding = get_encoding_from_headers(headers)
if encoding == 'ISO-8859-1':
encoding = None
# Try charset from content
if not encoding:
encoding = get_encodings_from_content(content)
encoding = encoding and encoding[0] or None
# Fallback to auto-detected encoding.
if not encoding and chardet is not None:
encoding = chardet.detect(content)['encoding']
if encoding and encoding.lower() == 'gb2312':
encoding = 'gb18030'
return encoding or 'latin_1'
def session_request(self,
method,
url,
params=None,
data=None,
headers=None,
cookies=None,
files=None,
auth=None,
timeout=None,
allow_redirects=True,
proxies=None,
hooks=None,
stream=None,
verify=False,
cert=None,
json=None):
conf = CONF.get("requests", {})
if timeout is None and "timeout" in conf:
timeout = conf["timeout"]
merged_cookies = merge_cookies(
merge_cookies(RequestsCookieJar(), self.cookies), cookies
or (conf.cookie if "cookie" in conf else None))
req = Request(
method=method.upper(),
url=url,
headers=merge_setting(headers,
conf["headers"] if "headers" in conf else {}),
files=files,
data=data or {},
json=json,
params=params or {},
auth=auth,
cookies=merged_cookies,
hooks=hooks,
)
prep = self.prepare_request(req)
proxies = proxies or (conf["proxies"] if "proxies" in conf else {})
settings = self.merged_environment_settings(prep.url, proxies, stream,
verify, cert)
send_kwargs = {
"timeout": timeout,
"allow_redirects": allow_redirects,
}
send_kwargs.update(settings)
resp = self.send(prep, **send_kwargs)
if resp.encoding == "ISO-8859-1":
encodings = get_encodings_from_content(resp.text)
if encodings:
encoding = encodings[0]
else:
encoding = resp.apparent_encoding
resp.encoding = encoding
return resp
def _fetchContent(self):
r = requests.get(self.url)
if get_encodings_from_content(r.content):
self.encoding = get_encodings_from_content(r.content)[0]
else:
from contextlib import closing
from urllib.request import urlopen
with closing(urlopen(self.url)) as f:
self.encoding = f.info().getparam("charset")
# Set System default Codeing
reload(sys)
sys.setdefaultencoding(self.encoding)
content = r.content.decode(self.encoding)
return content
def _fetchContent(self):
r = requests.get(self.url)
if get_encodings_from_content(r.content):
self.encoding = get_encodings_from_content(r.content)[0]
else:
from contextlib import closing
from urllib2 import urlopen
with closing(urlopen(self.url)) as f:
self.encoding = f.info().getparam("charset")
# Set System default Codeing
reload(sys)
sys.setdefaultencoding(self.encoding)
content = r.content.decode(self.encoding)
return content
def encoding(self):
"""
encoding of Response.content.
if Response.encoding is None, encoding will be guessed
by header or content or chardet if available.
"""
if hasattr(self, '_encoding'):
return self._encoding
# content is unicode
if isinstance(self.content, six.text_type):
return 'unicode'
# Try charset from content-type
encoding = get_encoding_from_headers(self.headers)
if encoding == 'ISO-8859-1':
encoding = None
# Try charset from content
if not encoding and get_encodings_from_content:
if six.PY3:
encoding = get_encodings_from_content(
utils.pretty_unicode(self.content[:100]))
else:
encoding = get_encodings_from_content(self.content)
encoding = encoding and encoding[0] or None
# Fallback to auto-detected encoding.
if not encoding and chardet is not None:
encoding = chardet.detect(self.content)['encoding']
if encoding and encoding.lower() == 'gb2312':
encoding = 'gb18030'
self._encoding = encoding or 'utf-8'
return self._encoding
def html_encoding(response_inst):
encoding = getattr(response_inst, 'encoding', None)
if encoding is False: # 不使用编码, 可能是非文本页面
return
if not encoding:
html_text = response_inst.response.text[1:] # 去除bom标记
encodings = get_encodings_from_content(html_text)
if encodings:
encoding = encodings[0]
if not encoding:
encoding = get_encoding_from_headers(response_inst.response.headers)
response_inst.response.encoding = encoding or Public_Constant.default_html_encoding
def encoding(self):
"""
encoding of Response.content.
if Response.encoding is None, encoding will be guessed
by header or content or chardet if available.
"""
if hasattr(self, '_encoding'):
return self._encoding
# content is unicode
if isinstance(self.content, six.text_type):
return 'unicode'
# Try charset from content-type
encoding = get_encoding_from_headers(self.headers)
if encoding == 'ISO-8859-1':
encoding = None
# Try charset from content
if not encoding and get_encodings_from_content:
if six.PY3:
encoding = get_encodings_from_content(utils.pretty_unicode(self.content[:100]))
else:
encoding = get_encodings_from_content(self.content)
encoding = encoding and encoding[0] or None
# Fallback to auto-detected encoding.
if not encoding and chardet is not None:
encoding = chardet.detect(self.content[:600])['encoding']
if encoding and encoding.lower() == 'gb2312':
encoding = 'gb18030'
self._encoding = encoding or 'utf-8'
return self._encoding
def session_request(self, method, url,
params=None, data=None, headers=None, cookies=None, files=None, auth=None,
timeout=10,
allow_redirects=True, proxies=None, hooks=None, stream=None, verify=False, cert=None, json=None):
# Create the Request.
merged_cookies = merge_cookies(merge_cookies(RequestsCookieJar(), self.cookies),
cookies)
default_header = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"
}
req = Request(
method=method.upper(),
url=url,
headers=merge_setting(headers, default_header),
files=files,
data=data or {},
json=json,
params=params or {},
auth=auth,
cookies=merged_cookies,
hooks=hooks,
)
prep = self.prepare_request(req)
proxies = proxies or {}
settings = self.merge_environment_settings(
prep.url, proxies, stream, verify, cert
)
# Send the request.
send_kwargs = {
'timeout': TIMEOUT,
'allow_redirects': allow_redirects,
}
send_kwargs.update(settings)
resp = self.send(prep, **send_kwargs)
if resp.encoding == 'ISO-8859-1':
encodings = get_encodings_from_content(resp.text)
if encodings:
encoding = encodings[0]
else:
encoding = resp.apparent_encoding
resp.encoding = encoding
return resp
def guess_response_encoding(headers, content):
'''
Guess the content encoding of a requests response content and header.
Note: there's a performance issue due to chardet.
'''
# first try the encoding supplied by responce header and content
encs = get_encodings_from_content(content) or []
for enc in encs:
try:
content.decode(enc)
LOG.info('Detected encoding %s from response content.', enc)
return enc
except UnicodeDecodeError:
LOG.debug('Encoding from response content doesn\'t work.')
except Exception, e:
LOG.error('Decode failed, %s' % e)
def procdata_getencoding(seed,headers,content): code = utils.get_encoding_from_headers(headers) if code: if code.lower() == 'gbk' or code.lower() == 'gb2312': code = 'gbk' elif code.lower() == 'utf-8': code = 'utf-8' else: code = None if code == None: code = utils.get_encodings_from_content(content) print "content",seed,code if code: code = code[0] if code.lower() == 'gbk' or code.lower() == 'gb2312': code = 'gbk' return code
def procdata_getencoding(seed, headers, content):
code = utils.get_encoding_from_headers(headers)
if code:
if code.lower() == 'gbk' or code.lower() == 'gb2312':
code = 'gbk'
elif code.lower() == 'utf-8':
code = 'utf-8'
else:
code = None
if code == None:
code = utils.get_encodings_from_content(content)
print "content", seed, code
if code:
code = code[0]
if code.lower() == 'gbk' or code.lower() == 'gb2312':
code = 'gbk'
return code
def guess_content_encoding(content):
'''
Guess the encoding for plain content.
Note: there's a performance issue due to chardet.
'''
# first try the encoding supplied by content
encs = get_encodings_from_content(content) or []
for enc in encs:
try:
content.decode(enc)
LOG.info('Detected encoding %s from content.', enc)
return enc
except UnicodeDecodeError:
LOG.debug('Encoding from content doesn\'t work.')
# neither encoding works, we have to go the hard way.
start = clock()
g = detect(content)
LOG.info('Detected encoding %s with cofidence of %g in %gs.' % (g['encoding'], g['confidence'], clock() - start))
return g['encoding']
def guess_content_encoding(content):
'''
Guess the encoding for plain content.
Note: there's a performance issue due to chardet.
'''
# first try the encoding supplied by content
encs = get_encodings_from_content(content) or []
for enc in encs:
try:
content.decode(enc)
LOG.info('Detected encoding %s from content.', enc)
return enc
except UnicodeDecodeError:
LOG.debug('Encoding from content doesn\'t work.')
# neither encoding works, we have to go the hard way.
start = clock()
g = detect(content)
LOG.info('Detected encoding %s with cofidence of %g in %gs.' %
(g['encoding'], g['confidence'], clock() - start))
return g['encoding']
def filter_encoding(self,seed, headers,content):
code = utils.get_encoding_from_headers(headers)
if code:
if code.lower() == 'gbk' or code.lower() == 'gb2312':
code = 'gbk'
return True
elif code.lower() == 'utf-8' or code.lower() == 'utf8':
code = 'utf8'
# as for utf8, we should check the content
else: # 'ISO-8859-1' and so on,
code = None
# chinese website may also miss the content-encoding header, so detect the content
if code == None:
codes = utils.get_encodings_from_content(content)
if codes:
for code in codes:
if code.lower() in [ 'gbk','gb2312']:
return True
elif code.lower() == 'utf8' or code.lower() == 'utf-8':
code = 'utf8'
break
if code != 'utf8':
return False
# here handle utf8
# to detect any chinese char win
try:
ucon = content.decode('utf8')
for uchar in ucon:
i = ord(uchar)
if i >= 0x4e00 and i <= 0x9fa5:
return True
except Exception, e:
print url, e
pass
def on_incoming(self, msg):
if not msg.type == msg.CHANNEL:
return
# Catching all exceptions without alerting, as there is just so much crap that can go wrong with web stuff. Also, I'm lazy.
try:
urls = self.url_re.findall(msg.body)
for url in urls:
# Catch edge case where url is in brackets
while url.startswith('(') and url.endswith(')'):
url = url[1:-1]
head = requests.head(url, allow_redirects=True)
# work on the URL we were redirected to, if any
url = head.url
message = ""
content_type = head.headers['content-type']
# HTML websites
if 'text/html' in content_type:
# Set up any required request headers
req_headers = {}
# TODO: Accept-Language header from config
req = requests.get(url, headers=req_headers, timeout=5)
if 'charset' not in content_type:
# requests only looks at headers to detect the encoding, we must find the charset ourselves
# we can't use req.content because regex doesn't work on bytestrings apparently
encodings = get_encodings_from_content(req.text)
if encodings:
req.encoding = encodings[0]
soup = BeautifulSoup(req.text)
# Look for the <title> tag or an <h1>, whichever is first
title = soup.find(['title', 'h1'])
if title is None:
return
title = self.utils.tag_to_string(title)
title = ' '.join(title.split())
message = "Title: " + title
# Other resources
else:
content_length = head.headers.get('content-length', '')
if content_length.isdigit():
size = self.sizeof_fmt(int(content_length))
else:
size = "Unknown size"
# Searches for the last segment of the URL (the filename)
filename = re.search(r'/([^/]+)/?$', url).groups(1)[0]
message = "{}: {} ({})".format(filename, content_type,
size)
self.bot.privmsg(msg.channel, message)
except Exception as exception:
print("Link Info Exception!")
print(type(exception), exception)
def test_none(self):
encodings = get_encodings_from_content('')
assert not len(encodings)
def geturl(url,
m='get',
headers={},
ip=None,
retries=0,
encoding=None,
data=None,
*args,
**kwargs):
"""基于requests第三方库，对其进一步封装，返回response对象
1.自动设定User-Agent
2.自动判断html编码方式（url链接为文件的话无法自动判断）
参数：
url: 请求的网址
m：请求的方式，默认get
headers: 设置头文件，默认添加User-Agent
ip：代理,输入字符串格式 [ip:port]
retries：请求错误之后的重试次数，显示错误信息
encoding： 网页的编码格式，默认不编码
data：向服务器发送的数据，注意修改方式
其他：可以传入基于requests的其他参数
返回值：
requests的response对象
"""
us = {'User-Agent': user_agent()}
if headers.get('User-Agent') == None:
headers.update(us)
# 代理ip的处理
proxies = None
if ip:
proxies = {'http': ip, 'https': ip, 'socks5': ip}
# 请求网页的主函数
# retry 请求响应错误时重试
@retry(retries=retries)
def _response():
request_method = getattr(requests, m) # 字符串变成函数变量名
rm = request_method(url,
headers=headers,
proxies=proxies,
*args,
**kwargs)
return rm
r = _response()
# 重试失败的话返回错误信息
if False in r:
return r[1]
# 优先采用encoding参数，如果不存在则自动判断
if encoding != None:
r.encoding = encoding
else:
# 一般的下载文件都很大，判断编码方式过于占用时间
# 故先判断url链接是否为文件，不是文件才分析编码
if "text/html" in r.headers['Content-Type']:
encoding = get_encodings_from_content(r.text)
if encoding:
r.encoding = encoding[0]
return r
def session_request(self,
method,
url,
params=None,
data=None,
headers=None,
cookies=None,
files=None,
auth=None,
timeout=None,
allow_redirects=True,
proxies=None,
hooks=None,
stream=None,
verify=False,
cert=None,
json=None):
# Create the Request.
merged_cookies = merge_cookies(
merge_cookies(RequestsCookieJar(), self.cookies), cookies)
default_header = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"
}
req = Request(
method=method.upper(),
url=url,
headers=merge_setting(headers, default_header),
files=files,
data=data or {},
json=json,
params=params or {},
auth=auth,
cookies=merged_cookies,
hooks=hooks,
)
prep = self.prepare_request(req)
raw = ''
if prep.body:
raw = "{}\n{}\n\n{}".format(
prep.method + ' ' + prep.url,
'\n'.join('{}: {}'.format(k, v) for k, v in prep.headers.items()),
prep.body)
else:
raw = "{}\n{}".format(
prep.method + ' ' + prep.url,
'\n'.join('{}: {}'.format(k, v) for k, v in prep.headers.items()))
proxies = proxies or {}
if PROXY_CONFIG_BOOL and not proxies:
proxies = PROXY_CONFIG
settings = self.merge_environment_settings(prep.url, proxies, stream,
verify, cert)
# Send the request.
send_kwargs = {
'timeout': timeout or TIMEOUT,
'allow_redirects': allow_redirects,
}
send_kwargs.update(settings)
resp = self.send(prep, **send_kwargs)
if resp.encoding == 'ISO-8859-1':
encodings = get_encodings_from_content(resp.text)
if encodings:
encoding = encodings[0]
else:
encoding = resp.apparent_encoding
resp.encoding = encoding
resp.raw = raw
return resp
def test_none(self):
encodings = get_encodings_from_content('')
assert not len(encodings)
def test_pragmas(self, content):
encodings = get_encodings_from_content(content)
assert len(encodings) == 1
assert encodings[0] == 'UTF-8'
def session_request(self,
method,
url,
params=None,
data=None,
headers=None,
cookies=None,
files=None,
auth=None,
timeout=None,
allow_redirects=True,
proxies=None,
hooks=None,
stream=None,
verify=False,
cert=None,
json=None):
# In order to remove headers that are set to None
def _merge_retain_none(request_setting,
session_setting,
dict_class=OrderedDict):
if session_setting is None:
return request_setting
if request_setting is None:
return session_setting
# Bypass if not a dictionary (e.g. verify)
if not (isinstance(session_setting, Mapping)
and isinstance(request_setting, Mapping)):
return request_setting
merged_setting = dict_class(to_key_val_list(session_setting))
merged_setting.update(to_key_val_list(request_setting))
return merged_setting
# Create the Request.
merged_cookies = merge_cookies(
merge_cookies(RequestsCookieJar(), self.cookies), cookies
or (conf.cookie if 'cookie' in conf else None))
if conf.random_agent:
conf.http_headers[HTTP_HEADER.USER_AGENT] = choice(conf.agents)
req = Request(
method=method.upper(),
url=url,
headers=_merge_retain_none(
headers, conf.http_headers if 'http_headers' in conf else {}),
files=files,
data=data or {},
json=json,
params=params or {},
auth=auth,
cookies=merged_cookies,
hooks=hooks,
)
prep = self.prepare_request(req)
# proxies = proxies or (conf.proxies if 'proxies' in conf else {})
if proxies is None:
proxies = conf.proxies if 'proxies' in conf else {}
settings = self.merge_environment_settings(prep.url, proxies, stream,
verify, cert)
timeout = timeout or conf.get("timeout", None)
if timeout:
timeout = float(timeout)
# Send the request.
send_kwargs = {
'timeout': timeout,
'allow_redirects': allow_redirects,
}
send_kwargs.update(settings)
resp = self.send(prep, **send_kwargs)
if resp.encoding == 'ISO-8859-1':
encodings = get_encodings_from_content(resp.text)
if encodings:
encoding = encodings[0]
else:
encoding = resp.apparent_encoding
resp.encoding = encoding
return resp
def test_pragmas(self, content):
encodings = get_encodings_from_content(content)
assert len(encodings) == 1
assert encodings[0] == 'UTF-8'
def session_request(self, method, url,
params=None, data=None, headers=None, cookies=None, files=None, auth=None,
timeout=None,
allow_redirects=True, proxies=None, hooks=None, stream=None, verify=False, cert=None, json=None):
# Create the Request.
merged_cookies = merge_cookies(merge_cookies(RequestsCookieJar(), self.cookies),
cookies)
default_header = {
"User-Agent": conf.agent,
"Connection": "close"
}
req = Request(
method=method.upper(),
url=url,
headers=merge_setting(headers, default_header),
files=files,
data=data or {},
json=json,
params=params or {},
auth=auth,
cookies=merged_cookies,
hooks=hooks,
)
prep = self.prepare_request(req)
raw = ''
p = urlparse(url)
_headers = copy.deepcopy(prep.headers)
if "Host" not in _headers:
_headers["Host"] = p.netloc
if prep.body:
raw = "{}\n{}\n\n{}\n\n".format(
prep.method + ' ' + prep.url + ' HTTP/1.1',
'\n'.join('{}: {}'.format(k, v) for k, v in _headers.items()),
prep.body)
else:
raw = "{}\n{}\n\n".format(
prep.method + ' ' + prep.url + ' HTTP/1.1',
'\n'.join('{}: {}'.format(k, v) for k, v in _headers.items()))
proxies = proxies or {}
if conf["proxy_config_bool"] and not proxies:
proxies = conf["proxy"]
if "socks4" in proxies.keys():
_tmp_str = "socks4://" + proxies["socks4"]
_tmp_proxy = {
"http": _tmp_str,
"https": _tmp_str
}
proxies = _tmp_proxy
elif "socks5" in proxies.keys():
_tmp_str = "socks5://" + proxies["socks5"]
_tmp_proxy = {
"http": _tmp_str,
"https": _tmp_str
}
proxies = _tmp_proxy
# prep.url = prep.url.encode('utf-8', errors='ignore').decode('utf-8', errors='ignore')
# fix https://github.com/boy-hack/w13scan/issues/64
settings = self.merge_environment_settings(
prep.url, proxies, stream, verify, cert
)
# Send the request.
send_kwargs = {
'timeout': timeout or conf["timeout"],
'allow_redirects': allow_redirects,
}
send_kwargs.update(settings)
resp = self.send(prep, **send_kwargs)
if resp.encoding == 'ISO-8859-1':
encodings = get_encodings_from_content(resp.text)
if encodings:
encoding = encodings[0]
else:
encoding = resp.apparent_encoding
resp.encoding = encoding
setattr(resp, 'reqinfo', raw)
return resp
def session_request(self,
method,
url,
params=None,
data=None,
headers=None,
cookies=None,
files=None,
auth=None,
timeout=None,
allow_redirects=True,
proxies=None,
hooks=None,
stream=None,
verify=False,
cert=None,
json=None):
# Create the Request.
merged_cookies = merge_cookies(
merge_cookies(RequestsCookieJar(), self.cookies), cookies)
default_header = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36",
"Connection": "close"
}
req = Request(
method=method.upper(),
url=url,
headers=merge_setting(headers, default_header),
files=files,
data=data or {},
json=json,
params=params or {},
auth=auth,
cookies=merged_cookies,
hooks=hooks,
)
prep = self.prepare_request(req)
raw = ''
if prep.body:
raw = "{}\n{}\n\n{}".format(
prep.method + ' ' + prep.url,
'\n'.join('{}: {}'.format(k, v) for k, v in prep.headers.items()),
prep.body)
else:
raw = "{}\n{}".format(
prep.method + ' ' + prep.url,
'\n'.join('{}: {}'.format(k, v) for k, v in prep.headers.items()))
proxies = proxies or {}
if conf["proxy_config_bool"] and not proxies:
proxies = conf["proxy"]
# prep.url = prep.url.encode('utf-8', errors='ignore').decode('utf-8', errors='ignore')
# fix https://github.com/boy-hack/w13scan/issues/64
settings = self.merge_environment_settings(prep.url, proxies, stream,
verify, cert)
# Send the request.
send_kwargs = {
'timeout': timeout or conf["timeout"],
'allow_redirects': allow_redirects,
}
send_kwargs.update(settings)
resp = self.send(prep, **send_kwargs)
if resp.encoding == 'ISO-8859-1':
encodings = get_encodings_from_content(resp.text)
if encodings:
encoding = encodings[0]
else:
encoding = resp.apparent_encoding
resp.encoding = encoding
if not getattr(resp, 'raw', None):
resp.raw = raw
else:
tmp = resp.raw
resp.raw = raw
setattr(resp, 'raw2', tmp)
return resp
def on_incoming(self, msg):
if not msg.type == msg.CHANNEL:
return
# Catching all exceptions without alerting, as there is just so much crap that can go wrong with web stuff. Also, I'm lazy.
try:
urls = self.url_re.findall(msg.body)
for url in urls:
# Catch edge case where url is in brackets
while url.startswith('(') and url.endswith(')'):
url = url[1:-1]
head = requests.head(url, allow_redirects=True)
# work on the URL we were redirected to, if any
url = head.url
message = ""
content_type = head.headers['content-type']
# HTML websites
if 'text/html' in content_type:
# Set up any required request headers
req_headers = {}
# TODO: Accept-Language header from config
req = requests.get(url, headers=req_headers, timeout=5)
if 'charset' not in content_type:
# requests only looks at headers to detect the encoding, we must find the charset ourselves
# we can't use req.content because regex doesn't work on bytestrings apparently
encodings = get_encodings_from_content(req.text)
if encodings:
req.encoding = encodings[0]
soup = BeautifulSoup(req.text)
# Look for the <title> tag or an <h1>, whichever is first
title = soup.find(['title', 'h1'])
if title is None:
return
title = self.utils.tag_to_string(title)
title = ' '.join(title.split())
message = "Title: " + title
# Other resources
else:
content_length = head.headers.get('content-length', '')
if content_length.isdigit():
size = self.sizeof_fmt(int(content_length))
else:
size = "Unknown size"
# Searches for the last segment of the URL (the filename)
filename = re.search(r'/([^/]+)/?$', url).groups(1)[0]
message = "{}: {} ({})".format(filename, content_type, size)
self.bot.privmsg(msg.channel, message)
except Exception as exception:
print("Link Info Exception!")
print(type(exception), exception)