def async_post(self, method, url, kargs, charset): if method == 'browser': driver = WebDriver(**kargs) driver.get(url) print("got it ", url) return ({ 'data': driver.page, 'url': url, 'code': 200, }, ) elif method == 'requests': res = to(url, **kargs) print("got it:", url) if res.status_code == 200: if charset: encoding = charset else: if len(res.content) > 2000: test_con = res.content[:2000] else: test_con = res.content encoding = detect(test_con).get('encoding') print(encoding) return ({ 'data': res.content.decode(encoding, 'ignore'), 'url': url, 'code': 200, }, ) else: return ({ 'url': url, 'code': 400, }, )
def post(self, key, **data): link = None form = None method = 'post' id = None for i, f in enumerate(self.forms): if f.action.find(key) != -1 and f.method.lower() == 'post': link = f.action form = f id = i break if not link: L.err("no search form action be found !", txt_color='red') L.i('type search("xxx", key="[ here]")', tag="only found 'key' :\n") for i, v in enumerate(self.forms): L.i(v.action, tag=i) return None data = self.form_check(id, data) with show_pro('loging', link): res = to(urljoin(self.url, link), session=self.session, data=data, agent=True, method=method) return Analyze(res, session=self.session)
def to_msg(self, token, msg): base = 'https://api.telegram.org/bot%s/' % token url = up.urljoin(base, 'sendMessage') chat = self.get_chat() url += "?" + up.urlencode({'chat_id': chat['id'], 'text': msg}) res = to(url).json() return res['ok']
def _link(self, url, encoding='utf8'): try: agent = random_choice(AGS) res = to(url, headers={'user-agent': agent}) if res.status_code != 200: raise Error404("not reachable", res.status_code) if self.charset: return url, self.parser( res.content.decode(self.charset, 'ignore')) return url, self.parser(res.content.decode(encoding, 'ignore')) except HTTPError as e: LogControl.fail(url, e.code) return url, None except Exception as e: LogControl.wrn(url) LogControl.err(url, e, txt_color='yellow') return url, None else: LogControl.ok(url)
def update_msg(cls, token): base = 'https://api.telegram.org/bot%s/' % token url = up.urljoin(base, 'getUpdates') logging.info(f"update url:{url}") res = to(url).json() if res['ok']: for m in res['result']: if not 'message' in m: continue mm = m['message'] if not 'text' in mm: continue print(mm['text']) if not 'from' in mm: continue mg = json.dumps(mm['chat']) yield cls(msg_id=mm['message_id'], msg_text=mm['text'], to_chat=mg, from_chat=json.dumps(mm['from']), time=mm['date'], update_id=m['update_id'])
def PSOT(self,url ,headers=None, **data): if header: res = to(url, data=data, method='post', headers=headers) else: res = to(url, data=data, method='post') return res.status_code, res
def GET(self, url, headers=None, **kargs): if headers: res = to(url, headers=headers, **kargs) else: res = to(url **kargs) return res.status_code, res
def __init__(self, url, show_process=True, cookie=True, session=None, **kargs): self.session = session if show_process: with show_pro("Geting", str(url) + " | " + str(kargs)): if isinstance(url, requests.models.Response): self.raw_response = url else: if cookie: if not self.session: self.session, self.raw_response = to(url, cookie=True, **kargs) else: _, self.raw_response = to(url, cookie=True, session=self.session, **kargs) else: if self.session: self.raw_response = to(url, session=self.session, **kargs) else: self.raw_response = to(url, **kargs) else: if isinstance(url, requests.models.Response): self.raw_response = url else: if cookie: self.session, self.raw_response = to(url, cookie=True, **kargs) else: if self.session: self.raw_response = to(url, session=self.session, **kargs) else: self.raw_response = to(url, **kargs) self.show_process = show_process self.url = self.raw_response.url self.encoding = self.raw_response.encoding.lower() # L.err(self.encoding) # check if encoding is iso-8859-1 self.encoding = ERR_ENCODING_2_RIGHT.get(self.encoding, self.encoding) try: self.content = self.raw_response.content.decode(self.encoding) except UnicodeDecodeError as e: L.err(e) L.title(self.encoding + "error", "try another charset") self._decode_raw() self.content = re.sub(r'(\n)+', '\n', self.content) self.Soup = BeautifulSoup(self.content, "html.parser") self.Base = copy.copy(self.Soup)
def search(self, search_str, key='search'): link = None form = None method = None id = None for i, f in enumerate(self.forms): if not hasattr(f, 'method'): f.method = 'get' # just for js's search form if f.action.find(key) != -1: link = f.action form = f id = i break # find GET form as search form for i, f in enumerate(self.forms): if f.method.lower() == 'get': link = f.action form = f id = i break if not link: L.err("no search form action be found !", txt_color='red') L.i('type search("xxx", key="[ here]")', tag="only found 'key' :\n") for i, v in enumerate(self.forms): L.i(v.action, tag=i) return None method = form.method.lower() f_table = {} for name in form.names(): i = form[name]['value'] if not i: f_table[name] = search_str else: f_table[name] = i if method == 'get': link += "?%s" % urlencode(f_table) return Analyze(urljoin(self.url, link), show_process=self.show_process, session=self.session, method=method) else: res = to(link, method=method, session=self.session, data=f_table) if res.headers['Content-Type'].find("json") != -1: return res.json() elif res.headers['Content-Type'].find('ml') != -1: return Analyze(res, session=self.session, show_process=self.show_process) else: L.err("return res is not json or xml", res.headers['Content-Type']) return None
def get_my_ip(): res = to("http://ipecho.net/plain").text.strip() return res
def query(data): res = to(self.host, method='post', data=data) return res.status_code, res.json()
def query(url): res = to(url) en = res.encoding return res.status_code, res.content.decode(en)
def handle(self, tag, cmd, **kargs): return tag, to(urljoin(self.target, cmd), **kargs).json()
def _cmd(self, data): to(self.url, data=json.dumps(data), method='post')
def proxy_to(self, url, **kargs): if self.proxy: return to(url, proxy=self.proxy, **kargs) else: return to(url, **kargs)
def get_json(url): return url, to(url).json()
def check(url): show(url, end='') global STA try: db_init = SqlEngine(database='%s/res_db/%s.db' % (ROOT, url)) db_init.create( 'cdn', url=str, sid=int, avgtime=str, srcip=str, srcname=str, isp=str, name=str, view=str, ) except Exception as e: pass headers = { 'origin': 'https://www.17ce.com', 'referer': 'https://www.17ce.com/', 'content-type': 'application/x-www-form-urlencoded', } req, res = to("www.17ce.com", cookie=True, agent=True) req.headers.update(headers) req.cookies.update({'allSites': url}) verify = sha1(b'C^dLMi%r&JH7bkmdFCgGl8' + url.encode('utf8') + b"1TnvST&D9LJ").hexdigest() data = urlencode({ 'rt': '1', 'nocache': '0', 'url': url, 'verify': verify, 'host': '', 'referer': '', 'cookie': '', 'agent': '', 'speed': '', 'postfield': '', 'pingcount': '', 'pingsize': '', }) for i in range(4): data += '&' + urlencode({'area[]': i}) for i in [0, 1, 2, 4, 6, 7, 8]: data += '&' + urlencode({'isp[]': i}) show(" init ", end='') res = req.post("https://www.17ce.com/site/ping", data=data).content res = json.loads(res.decode('utf8', 'ignore')) L(" ok") time.sleep(1) # show(res) show("... from server getting data .. wait") e = Exe(3) d = urlencode({'tid': res['tid'], 'num': 0, 'ajax_over': 0}) STA[url] = True e.done(reqq, recv, url, req, "https://www.17ce.com/site/ajaxfresh", d) b = bar() cc = 0 while STA[url]: cc += 1 time.sleep(1) show(next(b), end='\r') if STA[url] == 're-try': e.done(reqq, recv, url, req, "https://www.17ce.com/site/ajaxfresh", d) STA[url] = True