def async_post(self, method, url, kargs, charset):
if method == 'browser':
driver = WebDriver(**kargs)
driver.get(url)
print("got it ", url)
return ({
'data': driver.page,
'url': url,
'code': 200,
}, )
elif method == 'requests':
res = to(url, **kargs)
print("got it:", url)
if res.status_code == 200:
if charset:
encoding = charset
else:
if len(res.content) > 2000:
test_con = res.content[:2000]
else:
test_con = res.content
encoding = detect(test_con).get('encoding')
print(encoding)
return ({
'data': res.content.decode(encoding, 'ignore'),
'url': url,
'code': 200,
}, )
else:
return ({
'url': url,
'code': 400,
}, )
def post(self, key, **data):
link = None
form = None
method = 'post'
id = None
for i, f in enumerate(self.forms):
if f.action.find(key) != -1 and f.method.lower() == 'post':
link = f.action
form = f
id = i
break
if not link:
L.err("no search form action be found !", txt_color='red')
L.i('type search("xxx", key="[ here]")',
tag="only found 'key' :\n")
for i, v in enumerate(self.forms):
L.i(v.action, tag=i)
return None
data = self.form_check(id, data)
with show_pro('loging', link):
res = to(urljoin(self.url, link),
session=self.session,
data=data,
agent=True,
method=method)
return Analyze(res, session=self.session)
def to_msg(self, token, msg):
base = 'https://api.telegram.org/bot%s/' % token
url = up.urljoin(base, 'sendMessage')
chat = self.get_chat()
url += "?" + up.urlencode({'chat_id': chat['id'], 'text': msg})
res = to(url).json()
return res['ok']
def _link(self, url, encoding='utf8'):
try:
agent = random_choice(AGS)
res = to(url, headers={'user-agent': agent})
if res.status_code != 200:
raise Error404("not reachable", res.status_code)
if self.charset:
return url, self.parser(
res.content.decode(self.charset, 'ignore'))
return url, self.parser(res.content.decode(encoding, 'ignore'))
except HTTPError as e:
LogControl.fail(url, e.code)
return url, None
except Exception as e:
LogControl.wrn(url)
LogControl.err(url, e, txt_color='yellow')
return url, None
else:
LogControl.ok(url)
def update_msg(cls, token):
base = 'https://api.telegram.org/bot%s/' % token
url = up.urljoin(base, 'getUpdates')
logging.info(f"update url:{url}")
res = to(url).json()
if res['ok']:
for m in res['result']:
if not 'message' in m: continue
mm = m['message']
if not 'text' in mm: continue
print(mm['text'])
if not 'from' in mm: continue
mg = json.dumps(mm['chat'])
yield cls(msg_id=mm['message_id'],
msg_text=mm['text'],
to_chat=mg,
from_chat=json.dumps(mm['from']),
time=mm['date'],
update_id=m['update_id'])
def PSOT(self,url ,headers=None, **data):
if header:
res = to(url, data=data, method='post', headers=headers)
else:
res = to(url, data=data, method='post')
return res.status_code, res
def GET(self, url, headers=None, **kargs):
if headers:
res = to(url, headers=headers, **kargs)
else:
res = to(url **kargs)
return res.status_code, res
def __init__(self,
url,
show_process=True,
cookie=True,
session=None,
**kargs):
self.session = session
if show_process:
with show_pro("Geting", str(url) + " | " + str(kargs)):
if isinstance(url, requests.models.Response):
self.raw_response = url
else:
if cookie:
if not self.session:
self.session, self.raw_response = to(url,
cookie=True,
**kargs)
else:
_, self.raw_response = to(url,
cookie=True,
session=self.session,
**kargs)
else:
if self.session:
self.raw_response = to(url,
session=self.session,
**kargs)
else:
self.raw_response = to(url, **kargs)
else:
if isinstance(url, requests.models.Response):
self.raw_response = url
else:
if cookie:
self.session, self.raw_response = to(url,
cookie=True,
**kargs)
else:
if self.session:
self.raw_response = to(url,
session=self.session,
**kargs)
else:
self.raw_response = to(url, **kargs)
self.show_process = show_process
self.url = self.raw_response.url
self.encoding = self.raw_response.encoding.lower()
# L.err(self.encoding)
# check if encoding is iso-8859-1
self.encoding = ERR_ENCODING_2_RIGHT.get(self.encoding, self.encoding)
try:
self.content = self.raw_response.content.decode(self.encoding)
except UnicodeDecodeError as e:
L.err(e)
L.title(self.encoding + "error", "try another charset")
self._decode_raw()
self.content = re.sub(r'(\n)+', '\n', self.content)
self.Soup = BeautifulSoup(self.content, "html.parser")
self.Base = copy.copy(self.Soup)
def search(self, search_str, key='search'):
link = None
form = None
method = None
id = None
for i, f in enumerate(self.forms):
if not hasattr(f, 'method'):
f.method = 'get' # just for js's search form
if f.action.find(key) != -1:
link = f.action
form = f
id = i
break
# find GET form as search form
for i, f in enumerate(self.forms):
if f.method.lower() == 'get':
link = f.action
form = f
id = i
break
if not link:
L.err("no search form action be found !", txt_color='red')
L.i('type search("xxx", key="[ here]")',
tag="only found 'key' :\n")
for i, v in enumerate(self.forms):
L.i(v.action, tag=i)
return None
method = form.method.lower()
f_table = {}
for name in form.names():
i = form[name]['value']
if not i:
f_table[name] = search_str
else:
f_table[name] = i
if method == 'get':
link += "?%s" % urlencode(f_table)
return Analyze(urljoin(self.url, link),
show_process=self.show_process,
session=self.session,
method=method)
else:
res = to(link, method=method, session=self.session, data=f_table)
if res.headers['Content-Type'].find("json") != -1:
return res.json()
elif res.headers['Content-Type'].find('ml') != -1:
return Analyze(res,
session=self.session,
show_process=self.show_process)
else:
L.err("return res is not json or xml",
res.headers['Content-Type'])
return None
def get_my_ip():
res = to("http://ipecho.net/plain").text.strip()
return res
def query(data):
res = to(self.host, method='post', data=data)
return res.status_code, res.json()
def query(url):
res = to(url)
en = res.encoding
return res.status_code, res.content.decode(en)
def handle(self, tag, cmd, **kargs):
return tag, to(urljoin(self.target, cmd), **kargs).json()
def _cmd(self, data):
to(self.url, data=json.dumps(data), method='post')
def proxy_to(self, url, **kargs):
if self.proxy:
return to(url, proxy=self.proxy, **kargs)
else:
return to(url, **kargs)
def get_json(url):
return url, to(url).json()
def check(url):
show(url, end='')
global STA
try:
db_init = SqlEngine(database='%s/res_db/%s.db' % (ROOT, url))
db_init.create(
'cdn',
url=str,
sid=int,
avgtime=str,
srcip=str,
srcname=str,
isp=str,
name=str,
view=str,
)
except Exception as e:
pass
headers = {
'origin': 'https://www.17ce.com',
'referer': 'https://www.17ce.com/',
'content-type': 'application/x-www-form-urlencoded',
}
req, res = to("www.17ce.com", cookie=True, agent=True)
req.headers.update(headers)
req.cookies.update({'allSites': url})
verify = sha1(b'C^dLMi%r&JH7bkmdFCgGl8' + url.encode('utf8') +
b"1TnvST&D9LJ").hexdigest()
data = urlencode({
'rt': '1',
'nocache': '0',
'url': url,
'verify': verify,
'host': '',
'referer': '',
'cookie': '',
'agent': '',
'speed': '',
'postfield': '',
'pingcount': '',
'pingsize': '',
})
for i in range(4):
data += '&' + urlencode({'area[]': i})
for i in [0, 1, 2, 4, 6, 7, 8]:
data += '&' + urlencode({'isp[]': i})
show(" init ", end='')
res = req.post("https://www.17ce.com/site/ping", data=data).content
res = json.loads(res.decode('utf8', 'ignore'))
L(" ok")
time.sleep(1)
# show(res)
show("... from server getting data .. wait")
e = Exe(3)
d = urlencode({'tid': res['tid'], 'num': 0, 'ajax_over': 0})
STA[url] = True
e.done(reqq, recv, url, req, "https://www.17ce.com/site/ajaxfresh", d)
b = bar()
cc = 0
while STA[url]:
cc += 1
time.sleep(1)
show(next(b), end='\r')
if STA[url] == 're-try':
e.done(reqq, recv, url, req, "https://www.17ce.com/site/ajaxfresh",
d)
STA[url] = True