def stream_post(self, params, use_redirect_location=False, chunk_size=1024, show_bar=False):
if isinstance(params, str):
params = {'url': params}
if params.get('headers'):
h = params.pop('headers')
self.post_headers = dict(self.post_headers, **h)
para = {
'timeout': params.get('timeout', 5),
'headers': self.post_headers,
'stream': True,
}
# 使用 para 的值, 更新覆盖 req_params 的值
para = dict(params, **para)
content = ''
with requests.post(**para) as res:
if use_redirect_location:
return res.headers.get('location', '')
if 'content-length' not in res.headers:
return res.content
content_size = int(res.headers['content-length']) # 内容体总大小
if not show_bar:
for dat in res.iter_content(chunk_size=chunk_size, decode_unicode=True):
content += helper.to_str(dat)
return content
with click.progressbar(length=content_size, label='fetch') as bar:
for dat in res.iter_content(chunk_size=chunk_size, decode_unicode=True):
content += helper.to_str(dat)
bar.update(chunk_size)
return content
def aes_encrypt(self, plain, sec_key, enable_b64=True):
"""
使用 ``aes`` 加密数据, 并由 ``base64编码`` 加密后的数据
- ``sec_key`` 加密 ``msg``, 最后选择 ``是否由base64编码数据``
- msg长度为16位数, 不足则补 'ascii \\0'
.. warning::
msg长度为16位数, 不足则补 'ascii \\0'
:param plain:
:type plain: str
:param sec_key:
:type sec_key: str
:param enable_b64:
:type enable_b64: bool
:return:
:rtype:
"""
plain = helper.to_str(plain)
sec_key = helper.to_str(sec_key)
# 如果msg长度不为16倍数, 需要补位 '\0'
plain += '\0' * (self.bs - len(plain) % self.bs)
# 使用生成的 key, iv 加密
plain = helper.to_bytes(plain)
cipher = self.aes_obj(sec_key).encrypt(plain)
# 是否返回 base64 编码数据
cip = base64.b64encode(cipher) if enable_b64 else cipher
return helper.to_str(cip)
def parse_headers(self, use_cookies, raw):
"""
analyze headers from file or raw messages
:return: (url, dat)
:rtype:
"""
if not raw:
packet = helper.to_str(helper.read_file(self.fpth))
else:
packet = raw
dat = {}
pks = [x for x in packet.split('\n') if x.replace(' ', '')]
url = pks[0].split(' ')[1]
for i, cnt in enumerate(pks[1:]):
arr = cnt.split(':')
if len(arr) < 2:
continue
arr = [x.replace(' ', '') for x in arr]
_k, v = arr[0], ':'.join(arr[1:])
dat[_k] = v
if use_cookies:
try:
self.fmt_cookies(dat.pop('Cookie'))
except:
pass
self.headers = dat
self.url = 'https://{}{}'.format(self.headers.get('Host'), url)
return url, dat
def bs4get(self, url, use_cache=True, show_log=False, is_json=False):
raw = self.load(url, use_cache, show_log)
if not raw:
return
if is_json:
return json.loads(helper.to_str(raw))
return BS(raw, self.bs['parser'], from_encoding=self.bs['encoding'])
def search(self, name=''):
"""
name/ url
:param name:
:return:
"""
r = self.stream_post(
{'url': 'http://chromecj.com/handler/search/{}'.format(name)})
r = helper.to_str(r)
dat = []
for cj in r.split('|'):
info = cj.split('^')
# id, name, type, desc, img, date
dat.append({
'cid':
info[0],
'name':
info[1],
'desc':
info[3],
'url':
urljoin(
M['index'],
'/{}/{}/{}.html'.format(info[2], info[-1][:7], info[0]))
})
return dat
def parse_charles(self, use_cookies):
"""
analy plain info from charles packet
:return: (url, dat)
:rtype:
"""
packet = helper.to_str(helper.read_file(self.fpth))
dat = {}
pks = [x for x in packet.split('\n') if x.replace(' ', '')]
url = pks[0].split(' ')[1]
for i, cnt in enumerate(pks[1:]):
arr = cnt.split(':')
if len(arr) < 2:
continue
arr = [x.replace(' ', '') for x in arr]
_k, v = arr[0], ':'.join(arr[1:])
dat[_k] = v
if use_cookies:
self.fmt_cookies(dat.pop('Cookie'))
self.headers = dat
self.url = 'https://{}{}'.format(self.headers.get('Host'), url)
def rsa_base64_decrypt(self, cipher, b64=True):
"""
先base64 解码 再rsa 解密数据
"""
with open(self.key_file) as fp:
key_ = RSA.importKey(fp.read())
_cip = PKCS1_v1_5.new(key_)
cipher = base64.b64decode(cipher) if b64 else cipher
plain = _cip.decrypt(cipher, Random.new().read(15 + SHA.digest_size))
return helper.to_str(plain)
def _mock_input(self, target, content):
"""
mock human input
:param target: the element to input to
:param content: the content
:return:
"""
content = helper.to_str(content)
for w in content:
target.send_keys(w)
rand_block(0.01, 0.01)
def get_it_by_b64(url):
"""通过在 headers添加base64编码的数据实现
:param url:
:type url:
:return:
:rtype:
"""
log.debug('Try {}'.format(url))
tkn = b'roberts:123456'
tkn = helper.to_str(base64.b64encode(tkn))
sess.headers['Authorization'] = 'Basic {}'.format(tkn)
res = sess.get(url, verify=False)
return res
def rsa_base64_sign_str(self, plain, b64=True):
"""
对 msg rsa 签名, 然后使用 base64 encode 编码数据
"""
with open(self.key_file) as fp:
key_ = RSA.importKey(fp.read())
# h = SHA.new(plain if sys.version_info < (3, 0) else plain.encode('utf-8'))
plain = helper.to_bytes(plain)
# if hasattr(plain, 'encode'):
# plain = plain.encode()
h = SHA.new(plain)
cipher = pkcs.new(key_).sign(h)
cip = base64.b64encode(cipher) if b64 else cipher
return helper.to_str(cip)
def cl2num():
# 562, 552, 558, 556, 559, 554
y = [0, 1, 5, 8, 14, 17]
x = [2, 6, 7, 8, 9, 10, 11, 12, 13]
sc = [7, 9, 11, 12]
cnt = helper.to_str(helper.read_file('bj.txt'))
for i, l in enumerate(cnt.split('\n')):
l = [x for x in l.split(' ') if x]
if i not in y:
continue
v = l
s = '{}, 最高:{}, 最低:{}, 平均:{}, 人数:{}, 分数差:{}'.format(
v[2], v[7], v[9], v[11], v[6],
int(v[11]) - int(v[12]))
print(s)
def aes_decrypt(self, cipher, sec_key, enable_b64=True):
"""
由 ``base64解码数据``, 然后再使用 ``aes`` 解密数据
- 使用 ``sec_key`` 解密由 ``base64解码后的cipher`` 数据
- 先base64 decode, 再解密, 最后移除补位值 'ascii \\0'
:param cipher:
:type cipher: str
:param sec_key:
:type sec_key: str
:param enable_b64:
:type enable_b64: bool
:return:
:rtype:
"""
cipher = base64.b64decode(cipher) if enable_b64 else cipher
plain_ = self.aes_obj(sec_key).decrypt(cipher)
return helper.to_str(plain_).rstrip('\0')
def rsa_base64_encrypt(self, plain, b64=True):
"""
使用公钥加密 ``可见数据``
- 由于rsa公钥加密相对耗时, 多用来 ``加密数据量小`` 的数据
.. note::
1. 使用aes加密数据
2. 然后rsa用来加密aes加密数据时使用的key
:param plain:
:type plain:
:param b64:
:type b64:
:return:
:rtype:
"""
with open(self.key_file) as fp:
key_ = RSA.importKey(fp.read())
plain = helper.to_bytes(plain)
cipher = PKCS1_v1_5.new(key_).encrypt(plain)
cip = base64.b64encode(cipher) if b64 else cipher
return helper.to_str(cip)
def update_cache(self):
self.txt = helper.to_str(helper.read_file('awesome.md'))
self.md2dict()
def load_chapters(self):
if helper.is_file_ok(self.catalog_info['cache_file']):
self.da2017 = json.loads(
helper.to_str(helper.read_file(
self.catalog_info['cache_file'])))
