def _get_profile(self) -> iter: try: url = 'https://outlook.live.com/owa/0/sessiondata.ashx?app=Mail' headers = """ Host: outlook.live.com Connection: keep-alive Content-Length: 0 Pragma: no-cache Cache-Control: no-cache Origin: https://outlook.live.com x-js-clienttype: 2 User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36 Accept: */* Sec-Fetch-Site: same-origin Sec-Fetch-Mode: cors Referer: https://outlook.live.com/ Accept-Encoding: gzip, deflate, br Accept-Language: zh-CN,zh;q=0.9 """ html = self._ha.getstring(url, req_data='', headers=headers) self._userid = helper_str.substring(html, '"UserEmailAddress":"', '"') res = PROFILE(self._clientid, self.task, self.task.apptype, self._userid) res.nickname = helper_str.substring(html, '"UserDisplayName":"', '"') yield res except Exception: self._logger.error('Got profile fail: {}'.format( traceback.format_exc()))
def _cookie_login(self): res = False try: if self.task.cookie: self._ha._managedCookie.add_cookies('.yahoo.com', self.task.cookie) url = 'https://mail.yahoo.com/' headers = """ accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 accept-encoding: gzip, deflate, br accept-language: zh-CN,zh;q=0.9 cache-control: no-cache pragma: no-cache referer: https://mail.yahoo.com/d/folders/1 upgrade-insecure-requests: 1 user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36""" html = self._ha.getstring(url, headers=headers) self.homepage = html self.max_id = substring(html, '"messages":[{"id":"', '"') self.mailWssid = substring(html, '"mailWssid":"', '"').encode().decode('unicode_escape') self.appid = substring(html, '"appId":"', '"') self.boxesid = substring(html, '"allMailboxes":[{"id":"', '"') self._userid = substring(html, '"email":"', '"') if not self._userid or not self.mailWssid or not self.boxesid or not self.appid: return res self._username = self._userid res = True except Exception: self._logger.error(f'cookielogin fail: {traceback.format_exc()}') return res
def _get_folders(self): # dic = {'1': 'Inbox', '2': 'Sent', '3': 'Draft', '4': 'Trash', '6': 'Bulk', '21': 'Archive'} try: soup = BeautifulSoup(self.homepage, 'lxml') lis = soup.select('[style="padding:0 0 0 0px"]') if not lis: self._logger.info('Got mail folder is none!') for li in lis: total = substring(str(li), 'data-test-total-count="', '"') if not total or not int(total): continue total = int(total) id = substring(str(li), '/d/folders/', '?') if not id: id = substring(str(li), '/d/folders/', '"') name = li.select_one('.rtlI_dz_sSg') if not id or not name: continue res = Folder() res.folderid = id res.name = name.get_text() res.mailcount = total yield res except Exception: self._logger.error('Got folder fail:{}'.format(traceback.format_exc()))
def _access_profile(self) -> Tuple[bool, str]: """access simple profile""" succ = False msg = None try: url = "https://www.facebook.com/" html = self._ha.getstring(url, headers=""" accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 accept-encoding: gzip, deflate accept-language: zh-CN,zh;q=0.9 cache-control: no-cache referer: https://www.facebook.com/ upgrade-insecure-requests: 1 user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36""" ) if html is None or html == "": raise Exception("Get profile page failed") # {"USER_ID":"100027859862248","ACCOUNT_ID":"100027859862248", # "NAME":"San Zhang","SHORT_NAME":"San Zhang","IS_MESSENGER_ON # LY_USER":false,"IS_DEACTIVATED_ALLOWED_ON_MESSENGER":false} succ, self._userid = helper_str.substringif( html, 'USER_ID":"', '"') if not succ or self._userid is None or self._userid == "": succ, self._userid = helper_str.substring( html, 'user_id:"', '"') if not succ or self._username is None or self._username == "": msg = "访问个人信息失败" return (succ, msg) succ, self._username = helper_str.substringif(html, 'NAME":"', '"') if not succ or self._username is None or self._username == "": succ, self._username = helper_str.substring( html, 'name:"', '"') if not succ or self._username is None or self._username == "": msg = "访问个人信息失败" return (succ, msg) succ, self.is_messenger_only_user = helper_str.substringif( html, 'IS_MESSENGER_ONLY_USER":'******',') if succ and not self.is_messenger_only_user is None: if self.is_messenger_only_user == "false": self.is_messenger_only_user = False else: self.is_messenger_only_user = True succ, self.is_deactived_allowed_on_messenger = helper_str.substringif( html, 'IS_DEACTIVATED_ALLOWED_ON_MESSENGER":', ',') if succ and not self.is_deactived_allowed_on_messenger is None: if self.is_deactived_allowed_on_messenger == "false": self.is_deactived_allowed_on_messenger = False else: self.is_deactived_allowed_on_messenger = True succ = True msg = "访问个人信息成功" except Exception: self._logger.error("Access profile error:%s" % traceback.format_exc()) succ = False msg = "访问个人信息失败" return (succ, msg)
def _order_detail(self, orderUrl): orderUrl = 'http:' + orderUrl headers = """ Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 Accept-Encoding: gzip, deflate Accept-Language: zh-CN,zh;q=0.9 Cache-Control: no-cache Connection: keep-alive Host: zuser.qyer.com Pragma: no-cache Referer: http://zuser.qyer.com/orderList Upgrade-Insecure-Requests: 1 User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36""" html = self._ha.getstring(orderUrl, headers=headers) ordertime = substring(html, '下单时间:', '<') data = substring(html, 'window.__INITIAL_STATE__=', ';(function(){var s') jsdata = json.loads(data) order = {} order['title'] = substring(data, '"p_title":"', '"').replace('\\u002F', '/') order['type'] = substring(data, '"p_type_name":"', '",') order['stutas'] = substring(data, '"st_name":"', '"') order['price'] = substring(data, '"total_price":"', '"') order['num'] = substring(data, '"num":"', '",') order['merchant_name'] = substring(data, '"merchant_name":"', '",') order['call'] = substring(data, '"call":"', '",') order['contactInfo'] = jsdata['renderData']['data']['contactInfo'] order['detail'] = jsdata['renderData']['data']['lastminute'] return ordertime, order
def _get_orders(self): try: page = 0 while True: page += 1 url = 'https://passport.baicheng.com/order/list?keyword=&orderState=0&page={}'.format( page) headers = """ accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 accept-encoding: gzip, deflate, br accept-language: zh-CN,zh;q=0.9 cache-control: no-cache pragma: no-cache referer: https://www.baicheng.com/ upgrade-insecure-requests: 1 user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36""" html = self._ha.getstring(url, headers=headers).replace( '\n', '').replace('\t', '').replace('\r', '') html = re.sub(r'\s{2,}', ' ', html) num = substring(html, '<span class="info">共', '页') if not num: break soup = BeautifulSoup(html, 'lxml') orderList = soup.select('.user_orderlistcon') if orderList: for o in orderList: try: orderid = substring(str(o), '订单号:', '<') res_one = ITRAVELORDER_ONE(self.task, self._appcfg._apptype, self.userid, orderid) ordertime = substring(html, '下单时间:', ' <') res_one.ordertime = ordertime order_sn = re.findall(r'order_sn="(.*?)"', str(o)) dic = {} info = [] if order_sn: for sn in order_sn: order = self._order_detail(sn) info.append(order) dic['order'] = info res_one.append_orders(dic) yield res_one except: pass if int(num) <= page: break except Exception: self._logger.error('{} get order fail: {}'.format( self.userid, traceback.format_exc()))
def _get_profile(self): try: url = "https://easternmiles.ceair.com/mpf/user/userinfo?locale=cn" headers = """ Accept: application/json, text/plain, */* Accept-Encoding: gzip, deflate, br Accept-Language: zh-CN,zh;q=0.9 Cache-Control: no-cache Connection: keep-alive Content-Length: 5 Content-Type: application/json;charset=UTF-8 Host: easternmiles.ceair.com Origin: https://easternmiles.ceair.com Pragma: no-cache Referer: https://easternmiles.ceair.com/mpf/ User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36 """ html = self._ha.getstring(url, headers=headers, req_data='zh_CN') jshtml = json.loads(html) res = PROFILE(self._clientid, self.task, self._appcfg._apptype, self.userid) res.phone = jshtml['phoneNumber'] res.email = jshtml['email'] res.account = jshtml['memberId'] gender = jshtml['sex'] if gender == 'M': res.gender = EGender.Male elif gender == 'F': res.gender = EGender.Female else: res.gender = EGender.Unknown res.birthday = jshtml['birthday'] countrycode = jshtml['homeCountryCode'] res.country = substring(html, '"{}","text":"'.format(countrycode), '"') provincecode = jshtml['homeProvinceCode'] res.region = substring(html, 'id":"{}","text":"'.format(provincecode), '"') res.adress = jshtml['homeCityCode'] + jshtml['homeAdress'] detail = {} detail['name'] = jshtml['firstCnName'] + jshtml['lastCnName'] detail['idcard'] = jshtml['idCard'] detail['homeZipCode'] = jshtml['homeZipCode'] res.append_details(detail) yield res except Exception: self._logger.error('{} get profile fail: {}'.format( self.userid, traceback.format_exc()))
def _cookie_login(self): try: url = 'https://passport.mafengwo.cn/setting/security/' headers = """ Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 Accept-Encoding: gzip, deflate, br Accept-Language: zh-CN,zh;q=0.9 Cache-Control: no-cache Connection: keep-alive Host: passport.mafengwo.cn Pragma: no-cache Referer: https://passport.mafengwo.cn/setting/ Upgrade-Insecure-Requests: 1 User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36""" html = self.ha.getstring(url, headers=headers) userid = substring(html, '"UID":', ',') if userid: self.userid = userid + '-mafengwo' return True else: return False except Exception: self._logger.error('Mafengwo cookie login error: {}'.format( traceback.format_exc())) return False
def _cookie_login(self): url = 'https://lvyou.baidu.com/user/edit/info' headers = """ Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 Accept-Encoding: gzip, deflate, br Accept-Language: zh-CN,zh;q=0.9 Cache-Control: no-cache Connection: keep-alive Host: lvyou.baidu.com Pragma: no-cache Referer: https://lvyou.baidu.com/user/order/ Upgrade-Insecure-Requests: 1 User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36""" try: data: bytes = self._ha.get_response_data(url, headers=headers) html = data.decode('utf-8') userid = substring(html, 'J_nav-channel-user-center user-center">', '<') if userid: self.userid = userid + '-baidutravel' return True else: return False except: return False
def _cookie_login(self): url = 'https://member.ly.com/information' headers = """ Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 Accept-Encoding: gzip, deflate, br Accept-Language: zh-CN,zh;q=0.9 Cache-Control: no-cache Connection: keep-alive Host: member.ly.com Pragma: no-cache Referer: https://member.ly.com/order Upgrade-Insecure-Requests: 1 User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36""" html = self.ha.getstring(url, headers=headers).replace('\r', '').replace( '\n', '').replace('\t', '') html = re.sub(r'\s{2,}', '', html) soup = BeautifulSoup(html, 'lxml') try: phone = soup.select_one("#tel").get_text('-') phone = phone.split('-')[1] userid = substring(self.cookie, 'userid=', '&') if userid: self.userid = userid + '-tongcheng' return True elif phone: self.userid = phone + '-tongcheng' return True else: return False except: return False
def _cookie_login(self) -> bool: try: url = 'http://mail.21cn.com/w2/logon/signOn.do' headers = """ Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 Accept-Encoding: gzip, deflate Accept-Language: zh-CN,zh;q=0.9 Cache-Control: no-cache Host: mail.21cn.com Pragma: no-cache Proxy-Connection: keep-alive Referer: http://mail.21cn.com/w2/template/inbox.jsp Upgrade-Insecure-Requests: 1 User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36""" html = self._ha.getstring(url, headers=headers) if "21CN个人邮箱" in html and "退出" in html and "收信" in html: strAccount = substring(html, "accountName\":\"", "\"") if not strAccount or strAccount == "-1": # self._log("Get account failed.") return False else: self.userid = strAccount + '-MAIL21cn' return True else: return False except Exception: self._logger.error('Cookie login fail: {}'.format( traceback.format_exc())) return False
def _get_living(self, profile: NetworkProfile): """住址信息""" try: # education # https://www.facebook.com/profile.php?id=100030846743121&sk=about§ion=overview&lst=100013325533097%3A100030846743121%3A1568790537 url: str = "https://www.facebook.com/profile.php?id={}&sk=about§ion=living&lst={}%3A{}%3A{}".format( profile._userid, self._userid, profile._userid, helper_time.ts_since_1970(10)) html = self._ha.getstring(url, headers=""" accept: */* accept-encoding: gzip, deflate accept-language: en-US,en;q=0.9 cache-control: no-cache content-type: application/x-www-form-urlencoded origin: https://www.facebook.com pragma: no-cache referer: {} sec-fetch-mode: cors sec-fetch-site: same-origin""".format(profile.url)) if html is None: return address = helper_str.substring( html, 'data-hovercard-prefer-more-content-show="1">', '<') if address: profile.address = address except Exception: self._logger.error( "Get education page failed: username:{} url:{}".format( profile._networkid, profile.url))
def _cookie_login(self): res = False try: url = "https://easternmiles.ceair.com/mpf/auth/loginCheck_CN?locale=cn" headers = """ Accept: application/json, text/plain, */* Accept-Encoding: gzip, deflate, br Accept-Language: zh-CN,zh;q=0.9 Cache-Control: no-cache Connection: keep-alive Content-Length: 5 Content-Type: application/json;charset=UTF-8 Host: easternmiles.ceair.com Origin: https://easternmiles.ceair.com Pragma: no-cache Referer: https://easternmiles.ceair.com/mpf/ User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36 """ html = self._ha.getstring(url, headers=headers, req_data='') userid = substring(html, '"memberId":"', '"') if userid: self.userid = userid + '-ceair' res = True return res except Exception: return res
def _cookie_login(self): url = 'http://www.qyer.com/qcross/home/ajax?action=loginstatus' postdata = 'timer={}'.format(int(datetime.now(pytz.timezone('Asia/Shanghai')).timestamp() * 1000)) headers = """ Accept: application/json, text/javascript, */*; q=0.01 Accept-Encoding: gzip, deflate Accept-Language: zh-CN,zh;q=0.9 Cache-Control: no-cache Content-Length: 19 Content-Type: application/x-www-form-urlencoded; charset=UTF-8 Host: www.qyer.com Origin: http://www.qyer.com Pragma: no-cache Proxy-Connection: keep-alive Referer: http://www.qyer.com/u/10552906/profile User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36 X-Requested-With: XMLHttpRequest""" try: html = self._ha.getstring(url, headers=headers, req_data=postdata) userid = substring(html, '"uid":"', '"') if userid: self.userid = userid + '-qyer' return True else: return False except Exception: return False
def _cookie_login(self): try: url = 'http://my.caissa.com.cn/ws/UserHandler.ashx?action=getinfo&charset=&callback=jQuery171017810140921028506_{}&_={}'.format( int( datetime.now(pytz.timezone('Asia/Shanghai')).timestamp() * 1000), int( datetime.now(pytz.timezone('Asia/Shanghai')).timestamp() * 1000)) headers = """ Accept: text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01 Accept-Encoding: gzip, deflate Accept-Language: zh-CN,zh;q=0.9 Cache-Control: no-cache Connection: keep-alive Host: my.caissa.com.cn Pragma: no-cache X-Requested-With: XMLHttpRequest Referer: http://my.caissa.com.cn/GetInfo/GetWebInfo?&nav= User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36""" html = self._ha.getstring(url, headers=headers) userid = substring(html, '"uid":"', '"') if userid: self.userid = userid + '-caissa' return True else: return False except: pass return False
def _cookie_login(self): try: csrfToken = substring(self.cookie, "csrfToken=", ";") if csrfToken is None or csrfToken == "": return False t = str( int( datetime.now(pytz.timezone("Asia/Shanghai")).timestamp() * 1000)) cookieurl = ( "http://user.qunar.com/index/basic?t={t}&csrfToken={csrfToken}" .format(t=t, csrfToken=csrfToken)) headers = """ Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 Accept-Encoding:gzip, deflate Accept-Language:zh-CN,zh;q=0.9 Cache-Control:max-age=0 Connection:keep-alive Host:user.qunar.com Upgrade-Insecure-Requests:1 User-Agent:Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36""" html = self._ha.getstring(cookieurl, headers=headers) jshtml = json.loads(html) self.userid = str(jshtml["data"]["uid"]) + "-qunar" return True except: return False
def _mail_detai(self, html, folder): if 'var' not in html: return None jshtml = re.findall(r"('id'.*?)'hmid'", html) if jshtml: for mail in jshtml: mailid = substring(mail, "'id':'", "'") res_one = EML(self._clientid, self.task, self.userid, mailid, folder, self.task.apptype) res_one.provider = substring(mail, "'from':'", "'") res_one.subject = substring(mail, "'subject':'", "'") sendtime = substring(mail, "'sentDate':new Date(", "),") if "'read':true" in mail: res_one.state = 1 else: res_one.state = 0 j = 1 a = [] for i in sendtime.split(','): if j == 2: i = str(int(i) + 1) if len(i) == 1: i = '0' + i a.append(i) j += 1 sendtime = f'{a[0]}-{a[1]}-{a[2]} {a[3]}:{a[4]}:{a[5]}' res_one.sendtime = datetime.datetime.strptime(sendtime, "%Y-%m-%d %H:%M:%S") # 下载邮件 url = f'http://webmail.vip.126.com/js6/read/readdata.jsp?sid={self.sid}&mid={mailid}&mode=download&l=read&action=download_eml' headers = f""" Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 Accept-Encoding: gzip, deflate Accept-Language: zh-CN,zh;q=0.9 Cache-Control: no-cache Connection: keep-alive Host: webmail.vip.126.com Pragma: no-cache Referer: http://webmail.vip.126.com/js6/main.jsp?sid={self.sid}&df=mailvip Upgrade-Insecure-Requests: 1 User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36""" # response = self._ha.get_response_stream(url, headers=headers) resp = self._ha.get_response(url, headers=headers) res_one.stream_length = resp.headers.get('Content-Length', 0) res_one.io_stream = ResponseIO(resp) # res_one.io_stream = response yield res_one
def _check_registration(self): """ 查询手机号是否注册了skype # 中国的手机号需要加上+86 :param account: :return: """ t = time.strftime('%Y-%m-%d %H:%M:%S') try: url = 'https://login.live.com/login.srf' headers = f""" Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 Accept-Encoding: gzip, deflate, br Accept-Language: zh-CN,zh;q=0.9 Cache-Control: no-cache Connection: keep-alive Host: login.live.com Pragma: no-cache Upgrade-Insecure-Requests: 1 User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36""" response = self._ha.getstring(url, headers=headers) uaid = substring(response, 'uaid=', '"') flowToken = substring(response, 'id="i0327" value="', '"') url = f'https://login.live.com/GetCredentialType.srf?vv=1600&mkt=ZH-CN&lc=2052&uaid={uaid}' headers = f""" Accept: application/json client-request-id: {uaid} Content-type: application/json; charset=UTF-8 hpgact: 0 hpgid: 33 Origin: https://login.live.com Referer: https://login.live.com/login.srf User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36""" postdata = '{"username":"******","uaid":"' + uaid + '","isOtherIdpSupported":false,"checkPhones":true,"isRemoteNGCSupported":true,"isCookieBannerShown":false,"isFidoSupported":false,"forceotclogin":false,"otclogindisallowed":true,"flowToken":"' + flowToken + '"}' html = self._ha.getstring(url, headers=headers, req_data=postdata) if '"IfExistsResult":0,' in html: self._write_task_back(ECommandStatus.Succeed, 'Registered', t, EBackResult.Registerd) else: self._write_task_back(ECommandStatus.Succeed, 'Not Registered', t, EBackResult.UnRegisterd) except Exception: self._logger.error('Check registration fail: {}'.format(traceback.format_exc())) self._write_task_back(ECommandStatus.Failed, 'Check registration fail', t, EBackResult.CheckRegisterdFail) return
def get_tweet(self, screenname): """ 拿推文 根据screen_name先拿userid然后根据userid去拿tweet """ sa = requests.Session() sa.proxies = self.luminati_proxy_dict search_url = "https://twitter.com/search" headers = { "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "accept-encoding": "gzip, deflate", "accept-language": "zh-CN,zh;q=0.9", "cache-control": "no-cache", "pragma": "no-cache", "sec-fetch-dest": "document", "sec-fetch-mode": "navigate", "sec-fetch-site": "none", "sec-fetch-user": "******", "upgrade-insecure-requests": "1", "user-agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36""",', } search_response = sa.get(search_url, headers=headers) html1 = search_response.text gt = substring(html1, 'decodeURIComponent("gt=', ";") userurl = f"https://api.twitter.com/graphql/esn6mjj-y68fNAj45x5IYA/UserByScreenName?variables=%7B%22screen_name%22%3A%22{screenname}%22%2C%22withHighlightedLabel%22%3Atrue%7D" headers1 = { "accept": "*/*", "accept-encoding": "gzip, deflate", "accept-language": "zh-CN,zh;q=0.9", "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA", "cache-control": "no-cache", "content-type": "application/json", "origin": "https://twitter.com", "pragma": "no-cache", "referer": "https://twitter.com/", "sec-fetch-dest": "empty", "sec-fetch-mode": "cors", "sec-fetch-site": "same-site", "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36", "x-guest-token": f"{gt}", "x-twitter-active-user": "******", "x-twitter-client-language": "zh-cn", } response = sa.get(userurl, headers=headers1) if response is None or response.status_code != 200: return jtext = response.text sj = json.loads(jtext) user = sj["data"]["user"] userid = user.get("rest_id") if userid is None or userid == "": print(f"Cant get userid by {screenname}, please check the username") tweeturl = f"https://api.twitter.com/2/timeline/media/{userid}.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_ext_alt_text=true&include_quote_count=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&simple_quoted_tweet=true&count=20&ext=mediaStats%2ChighlightedLabel" tweet_res = sa.get(url=tweeturl, headers=headers1) print(tweet_res.status_code) print(tweet_res.text)
def _get_sid(self): res = None sid = self._ha.cookies.get('Coremail') if not sid: return res if '%' in sid: sid = substring(sid, '%', '%') return sid
def _cookie_login(self) -> bool: res = False url = "https://mail.qq.com/cgi-bin/login" querystring = { "vt": "passport", "vm": "wpt", "ft": "loginpage", "target": "" } payload = "" headers = { 'accept': "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", 'accept-encoding': "gzip, deflate, br", 'accept-language': "zh-CN,zh;q=0.9,en;q=0.8", 'cache-control': "no-cache,no-cache", 'cookie': self.task.cookie, 'pragma': "no-cache", 'referer': "https://mail.qq.com/", 'upgrade-insecure-requests': "1", 'user-agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" } try: response = requests.request("GET", url, data=payload, headers=headers, params=querystring) index_txt = response.text if "设置" in index_txt and "退出" in index_txt and "关联其他QQ邮箱": self._html = index_txt re_sid = re.compile("sid\=(.+?)\&") sid = re_sid.search(self._html) self._sid = sid.group(1) self._userid = substring(index_txt, '关联其他QQ邮箱">', '<') res = True # 设置cookie self._ha._managedCookie.add_cookies('qq.com', self.task.cookie) except Exception: self._logger.error( f"Cookie login error, err:{traceback.format_exc()}") finally: return res
def _check_registration(self): """ 查询手机号是否注册了马蜂窝 :param account: :return: """ t = time.strftime('%Y-%m-%d %H:%M:%S') try: phone = self.task.phone url = 'https://passport.mafengwo.cn/regist.html' headers = """ Host: passport.mafengwo.cn Connection: keep-alive Upgrade-Insecure-Requests: 1 User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36 Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 Referer: http://www.mafengwo.cn/?mfw_chid=3546 Accept-Encoding: gzip, deflate, br Accept-Language: zh-CN,zh;q=0.9 """ html = self.ha.getstring(url, headers=headers) token = substring(html, 'name="token" value="', '"') url = 'https://passport.mafengwo.cn/regist' headers = """ Host: passport.mafengwo.cn Connection: keep-alive Content-Length: 59 Cache-Control: max-age=0 Origin: https://passport.mafengwo.cn Upgrade-Insecure-Requests: 1 Content-Type: application/x-www-form-urlencoded User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36 Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 Referer: https://passport.mafengwo.cn/regist.html Accept-Encoding: gzip, deflate, br Accept-Language: zh-CN,zh;q=0.9 """ data = f"token={token}&passport={phone}" html = self.ha.getstring(url, headers=headers, req_data=data) isreg = re.findall(r'<div class="alert alert-danger">', html) if isreg: self._write_task_back(ECommandStatus.Succeed, 'Registered', t, EBackResult.Registerd) else: self._write_task_back(ECommandStatus.Succeed, 'Not Registered', t, EBackResult.UnRegisterd) except Exception: self._logger.error('Check registration fail: {}'.format( traceback.format_exc())) self._write_task_back(ECommandStatus.Failed, 'Check registration fail', t, EBackResult.CheckRegisterdFail) return
def get_search_ids(self, key_word): """ 获取搜索的id """ sa = requests.Session() sa.proxies = self.luminati_proxy_dict search_url = "https://twitter.com/search" url1 = f"{search_url}?f=users&vertical=default&q={key_word}&src=typd" headers = { "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "accept-encoding": "gzip, deflate, br", "accept-language": "zh-CN,zh;q=0.9", "cache-control": "no-cache", "pragma": "no-cache", "sec-fetch-dest": "document", "sec-fetch-mode": "navigate", "sec-fetch-site": "none", "sec-fetch-user": "******", "upgrade-insecure-requests": "1", "user-agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36""",', } search_response = sa.get(search_url, headers=headers) html1 = search_response.text gt = substring(html1, 'decodeURIComponent("gt=', ";") url = f"https://api.twitter.com/2/search/adaptive.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_ext_alt_text=true&include_quote_count=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&simple_quoted_tweet=true&q={quote_plus(key_word)}&vertical=default&result_filter=user&count=20&query_source=typd&pc=1&spelling_corrections=1&ext=mediaStats%2ChighlightedLabel" headers = { "accept": "*/*", "accept-encoding": "gzip, deflate", "accept-language": "zh-CN,zh;q=0.9", "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA", "cache-control": "no-cache", "origin": "https://twitter.com", "pragma": "no-cache", "referer": "https://twitter.com/search?f=users&vertical=default&q=%E6%9D%8E%E5%AD%90%E6%9F%92&src=typd", "sec-fetch-dest": "empty", "sec-fetch-mode": "cors", "sec-fetch-site": "same-site", "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36", "x-guest-token": f"{gt}", "x-twitter-active-user": "******", "x-twitter-client-language": "zh-cn", } response = sa.get(url, headers=headers) if response is None or response.status_code != 200: return jtext = response.text sj = json.loads(jtext) users = sj["globalObjects"]["users"] if not users: self._logger.info("Search nothing in twitter!") return None for key, value in sj["globalObjects"]["users"].items(): data = value["screen_name"] print(data)
def _get_orders(self): url = 'http://www.super8.com.cn/MemInfo/GetMemCusOrder' pagesize = 5 pageindex = 0 resultCount = 0 orderCntGot = False while True: try: pageindex += 1 postdata = 'PageSize={}&PageIndex={}'.format( pagesize, pageindex) headers = """ Accept:*/* Accept-Encoding:gzip, deflate Accept-Language:zh-CN,zh;q=0.9 Cache-Control:no-cache Connection:keep-alive Content-Type:application/x-www-form-urlencoded; charset=UTF-8 Host:www.super8.com.cn Origin:http://www.super8.com.cn Pragma:no-cache Referer:http://www.super8.com.cn/MemInfo/MemCusOrder X-Requested-With:XMLHttpRequest""" html = self._ha.getstring(url, headers=headers, req_data=postdata) if not orderCntGot: # "rscount": 4, "pages": 1, "pageindex": 1} strResCnt = substring(html, '"rscount":', ',') if strResCnt: resultCount = int(strResCnt) orderCntGot = True jshtml = json.loads(html) arrdata = jshtml['data']['arrdata'] if arrdata: for order in arrdata: try: ordertime = order['CreateTime'] + ':00' orderid = order['OrderID'] res_one = ITRAVELORDER_ONE(self.task, self._appcfg._apptype, self.userid, orderid) res_one.host = 'http://www.super8.com.cn' res_one.ordertime = ordertime res_one.append_orders(order) yield res_one except: pass if resultCount <= pagesize * pageindex: break except Exception: self._logger.error('{} get order fail: {}'.format( self.userid, traceback.format_exc()))
def _get_profile(self) -> iter: try: csrf, _root_token_ = self._get_csrf() url = 'https://mail.aliyun.com/alimail/getStartupInfo?tpl=v5&_csrf_token_={}'.format(csrf) headers = """ Accept: */* Accept-Encoding: gzip, deflate, br Accept-Language: zh-CN,zh;q=0.9 Cache-Control: no-cache Connection: keep-alive Host: mail.aliyun.com Pragma: no-cache Referer: https://mail.aliyun.com/alimail/ User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36""" html = self._ha.getstring(url, headers=headers) res = PROFILE(self._clientid, self.task, self.task.apptype, self.userid) res.email = substring(html, '"email":"', '"') res.nickname = substring(html, '"displayName":"', '"') yield res except Exception: self._logger.error('{} got profile fail: {}'.format(self.userid, traceback.format_exc()))
def _get_profile(self): try: url = 'http://mail.21cn.com/w2/logon/signOn.do' html = self._ha.getstring(url, headers=""" Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 Accept-Encoding: gzip, deflate Accept-Language: zh-CN,zh;q=0.9 Cache-Control: no-cache Host: mail.21cn.com Pragma: no-cache Proxy-Connection: keep-alive Upgrade-Insecure-Requests: 1 User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36""" ) detail = {} detail['userid'] = substring(html, '"uuserId":"', '"') res = PROFILE(self._clientid, self.task, self.task.apptype, self.userid) res.nickname = substring(html, '"userName":"******"') res.account = substring(html, 'accountName":"', '"') photourl = substring(html, 'src="..', '"') if photourl: photourl = 'http://mail.21cn.com/w2' + photourl profilepic: RESOURCES = RESOURCES(self._clientid, self.task, photourl, EResourceType.Picture, self.task.apptype) resp_stream: ResponseIO = self._ha.get_response_stream( photourl) profilepic.io_stream = resp_stream profilepic.filename = photourl.rsplit('/', 1)[-1] profilepic.sign = ESign.PicUrl res.append_resource(profilepic) yield profilepic yield res except Exception: self._logger.error('{} got profile fail: {}'.format( self.userid, traceback.format_exc()))
def _get_profile(self): try: url = 'https://lvyou.baidu.com/user/edit/info' headers = """ Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 Accept-Encoding: gzip, deflate, br Accept-Language: zh-CN,zh;q=0.9 Cache-Control: no-cache Connection: keep-alive Host: lvyou.baidu.com Pragma: no-cache Referer: https://lvyou.baidu.com/user/order/ Upgrade-Insecure-Requests: 1 User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36""" data: bytes = self._ha.get_response_data(url, headers=headers) html = data.decode('utf-8') res = PROFILE(self._clientid, self.task, self._appcfg._apptype, self.userid) res.nickname = substring( html, 'J_nav-channel-user-center user-center">', '<') photourl = substring(html, 'class="" src="', '"') if photourl: photourl = 'https:' + photourl profilepic: RESOURCES = RESOURCES(self._clientid, self.task, photourl, EResourceType.Picture, self._appcfg._apptype) resp_stream: ResponseIO = self._ha.get_response_stream( photourl) profilepic.io_stream = resp_stream profilepic.filename = photourl.rsplit('/', 1)[-1] profilepic.sign = ESign.PicUrl res.append_resource(profilepic) yield profilepic yield res except Exception: self._logger.error('{} got profile fail: {}'.format( self.userid, traceback.format_exc()))
def order_Detail(self, detailUrl): headers = """ Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 Accept-Encoding: text Accept-Language: zh-CN,zh;q=0.9 Cache-Control: no-cache Connection: keep-alive Host: lvyou.baidu.com Pragma: no-cache Referer: https://lvyou.baidu.com/user/order/ Upgrade-Insecure-Requests: 1 User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36""" data: bytes = self._ha.get_response_data(detailUrl, headers=headers) html = data.decode('utf-8') order = {} ordertime = substring(html, '>下单日期:', '<') + ' ' + '00:00:00' soup = BeautifulSoup(html, 'lxml') orderid = substring(html, "order_id: '", "'") ord = soup.select_one('.order-body').get_text(' ').replace( '\t', '').replace('\r', '').replace('\n', '') order['detail'] = ord return ordertime, order
def __get_token_sign(self): try: url = 'https://pan.baidu.com/disk/home?errno=0&errmsg=Auth%20Login%20Sucess&&bduss=&ssnerror=0&traceid=' html = self._ha.getstring(url, headers=""" Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 Accept-Encoding: gzip, deflate, br Accept-Language: zh-CN,zh;q=0.9 Cache-Control: no-cache Connection: keep-alive Host: pan.baidu.com Pragma: no-cache Referer: https://pan.baidu.com/disk/home?errno=0&errmsg=Auth%20Login%20Sucess&&bduss=&ssnerror=0&traceid= Upgrade-Insecure-Requests: 1 User-Agent: netdisk;6.7.1.9;PC;PC-Windows;10.0.17763;WindowsBaiduYunGuanJia""") global token, timestamp, sign token = substring(html, "initPrefetch('", "'") timestamp = substring(html, '"timestamp":', ',') sign1 = substring(html, '"sign1":"', '"') sign3 = substring(html, '"sign3":"', '"') sign = self.__get_sign(sign1, sign3) if not token or not sign: self._logger.info('Got sign or token is None') except Exception: self._logger.error('Got sign and token fail: {}'.format(traceback.format_exc()))
def _cookie_login(self) -> bool: try: csrf, _root_token_ = self._get_csrf() url = 'https://mail.aliyun.com/alimail/getStartupInfo?tpl=v5&_csrf_token_={}'.format(csrf) headers = """ Accept: */* Accept-Encoding: gzip, deflate, br Accept-Language: zh-CN,zh;q=0.9 Cache-Control: no-cache Connection: keep-alive Host: mail.aliyun.com Pragma: no-cache Referer: https://mail.aliyun.com/alimail/ User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36""" html = self._ha.getstring(url, headers=headers) _root_token_ = substring(html, '"browserSessionKey":"', '"') if "browserSessionKey" in html: self.owner = substring(html, '"email":"', '"') self.userid = substring(html, '"email":"', '"') + '-aliyun' return True else: return False except: return False