def _captchaAutoCheck(self, results): params = { 'answer': results, 'login_site': 'E', 'rand': 'sjrand', } jsonRet = EasyHttp.send(autoVerifyUrls['check_url'], params=params) # print('captchaCheck: %s' % jsonRet) def verify(response): return Captcha.__REPONSE_NORMAL_CDOE_SUCCESSFUL == response[ 'result_code'] if 'result_code' in response else False return verify(jsonRet)
def freeProxySixth(): """ 讯代理 http://www.xdaili.cn/ :return: """ url = 'http://www.xdaili.cn/ipagent/freeip/getFreeIps?page=1&rows=10' try: res = EasyHttp.get(url, timeout=10).json() if not res or not res['RESULT'] or not res['RESULT']['rows']: Log.w('http://www.goubanjia.com无效') return [] for row in res['RESULT']['rows']: yield '{}:{}'.format(row['ip'], row['port']) except Exception as e: pass
def check_login(): response = EasyHttp.post_custom(loginUrls['normal']['conf']) if not response or not response.json(): Log.d('登录状态检查失败,重新请求') status, login = do_login() if not status: return False resp = response.json() login_status = resp.get('data').get('is_login') Log.d('登录状态:%s' % login_status) if 'Y' != login_status: Log.d('登录状态已过期,重新请求') status, login = do_login() if not status: return False return True
def _submitOrderRequest(self, tourFlag='dc'): formData = { 'secretStr': TrainUtils.undecodeSecretStr(self.__ticket.secretStr), # 'secretStr': self.__ticket.secretStr, # 'secretStr': urllib.parse.unquote(self.__ticket.secretStr), 'train_date': Utils.formatDate(self.__ticket.startDate), # 2018-01-04 'back_train_date': time.strftime("%Y-%m-%d", time.localtime()), # query date:2017-12-31 'tour_flag': tourFlag, 'purpose_codes': self.__ticket.passengerType, 'query_from_station_name': self.__ticket.fromStation, 'query_to_station_name': self.__ticket.toStation, 'undefined': '', } jsonRet = EasyHttp.send(self._urlInfo['submitOrderRequest'], data=formData) # print('submitOrderRequest %s' % jsonRet) return jsonRet['status'], jsonRet['messages']
def freeProxyWallFirst(): """ 墙外网站 cn-proxy :return: """ urls = ['http://cn-proxy.com/', 'http://cn-proxy.com/archives/218'] for url in urls: r = EasyHttp.get(url, timeout=10) if not r: Log.w('http://cn-proxy.com无效') return [] proxies = re.findall( r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\w\W]<td>(\d+)</td>', ) for proxy in proxies: yield ':'.join(proxy)
def freeProxyNinth(): """ 码农代理 https://proxy.coderbusy.com/ :return: """ urls = ['https://proxy.coderbusy.com/classical/country/cn.aspx?page=1'] for url in urls: r = EasyHttp.get(url, timeout=10) if not r: Log.w('http://proxy.coderbusy.com无效') return [] proxies = re.findall( 'data-ip="(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})".+?>(\d+)</td>', r) for proxy in proxies: yield ':'.join(proxy)
def _login_init(self): url_info = copy.deepcopy(self._urlInfo["getDevicesId"]) url_info['url'] = self._urlInfo["getDevicesId"]['url'] + str(int(time.time()*1000)) devices_id_rsp = EasyHttp.get_custom(url_info) if devices_id_rsp: callback = devices_id_rsp.text.replace("callbackFunction('", '').replace("')", '') text = json.loads(callback) devices_id = text.get('dfp') exp = text.get('exp') EasyHttp.setCookies(RAIL_DEVICEID=devices_id, RAIL_EXPIRATION=exp) # Log.d('设备Id:%s'%devices_id) return True, '获取设备指纹成功' EasyHttp.send(self._urlInfo['index']) EasyHttp.send(self._urlInfo['loginInit']) return False,'获取设备指纹失败'
def query(trainDate, fromStation, toStation, passengerType=PASSENGER_TYPE_ADULT): params = { r"leftTicketDTO.train_date": trainDate, r"leftTicketDTO.from_station": city2code(fromStation), r"leftTicketDTO.to_station": city2code(toStation), r"purpose_codes": passengerType, } jsonRet = EasyHttp.send(queryUrls["query"], params=params) try: if jsonRet: return Query.__decode(jsonRet["data"]["result"], passengerType) except Exception as e: Log.e(e) return []
def query(trainDate, fromStation, toStation, passengerType=PASSENGER_TYPE_ADULT): params = { r'leftTicketDTO.train_date': trainDate, r'leftTicketDTO.from_station': city2code(fromStation), r'leftTicketDTO.to_station': city2code(toStation), r'purpose_codes': passengerType } jsonRet = EasyHttp.send(queryUrls['query'], params=params) try: if jsonRet: return Query.__decode(jsonRet['data']['result'], passengerType) except Exception as e: Log.e(e) return []
def _getExtraInfo(self): def getRepeatSubmitToken(html): repeatSubmitToken = re.findall(r"var globalRepeatSubmitToken = '(.*)'", html)[0] # print('RepeatSubmitToken = %s' % repeatSubmitToken) return repeatSubmitToken html = EasyHttp.send(self._urlInfo['getExtraInfo']) if not Utils.check(html, 'getExtraInfoUrl: failed to visit %s' % self._urlInfo['getExtraInfo']['url']): return False self.__ticket.repeatSubmitToken = getRepeatSubmitToken(html) def decodeTicketInfoForPassengerForm(html): ticketInfoForPassengerForm = re.findall(r'var ticketInfoForPassengerForm=(.*);', html)[0] return json.loads(ticketInfoForPassengerForm.replace("'", "\"")) # print('=======================') self.__ticket.ticketInfoForPassengerForm = decodeTicketInfoForPassengerForm(html) return True
def freeProxyTen(): """ 云代理 http://www.ip3366.net/free/ :return: """ urls = ['http://www.ip3366.net/free/'] for url in urls: r = EasyHttp.get(url, timeout=10) if not r: Log.w('http://www.ip3366.com无效') return [] proxies = re.findall( r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td>(\d+)</td>', r) for proxy in proxies: yield ":".join(proxy)
def _captchaCheck(self, results): data = { 'answer': results, 'login_site': 'E', 'rand': 'sjrand', '_': int(time.time() * 1000) } jsonRet = EasyHttp.send(loginUrls['normal']['captchaCheck'], params=data) # print('captchaCheck: %s' % jsonRet) def verify(response): return Captcha.__REPONSE_NORMAL_CDOE_SUCCESSFUL == response[ 'result_code'] if 'result_code' in response else False return verify(jsonRet)
def freeProxyEight(): """ 秘密代理 http://www.mimiip.com """ url_gngao = ['http://www.mimiip.com/gngao/%s' % n for n in range(1, 2)] # 国内高匿 url_gnpu = ['http://www.mimiip.com/gnpu/%s' % n for n in range(1, 2)] # 国内普匿 url_gntou = ['http://www.mimiip.com/gntou/%s' % n for n in range(1, 2)] # 国内透明 url_list = url_gngao + url_gnpu + url_gntou for url in url_list: r = EasyHttp.get(url, timeout=10) if not r: Log.w('http://www.mimiip.com无效') return [] proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\w\W].*<td>(\d+)</td>', r) for proxy in proxies: yield ':'.join(proxy)
def _queryOrderWaitTime(self): params = { "random": "%10d" % (time.time() * 1000), "tourFlag": self.__ticket.ticketInfoForPassengerForm["tour_flag"] or "dc", "_json_att": "", "REPEAT_SUBMIT_TOKEN": self.__ticket.repeatSubmitToken, } jsonRet = EasyHttp.send(self._urlInfo["queryOrderWaitTime"], params=params) print("queryOrderWaitTime: %s" % jsonRet) return ( jsonRet["status"], jsonRet["messages"], jsonRet["data"]["waitTime"], jsonRet["data"]["orderId"], jsonRet["data"]["msg"] if "msg" in jsonRet["data"] else None, )
def freeProxyTwelve(page_count=2): """ guobanjia http://ip.jiangxianli.com/?page= 免费代理库 超多量 :return: """ for i in range(1, page_count + 1): url = 'http://ip.jiangxianli.com/?page={}'.format(i) html_tree = EasyHttp.getHtmlTree(url) if not html_tree: Log.w('http://ip.jiangxianli.com无效') return [] tr_list = html_tree.xpath("/html/body/div[1]/div/div[1]/div[2]/table/tbody/tr") if len(tr_list) == 0: continue for tr in tr_list: yield tr.xpath("./td[2]/text()")[0] + ":" + tr.xpath("./td[3]/text()")[0]
def freeProxySeventh(): """ 快代理 https://www.kuaidaili.com """ url_list = [ 'https://www.kuaidaili.com/free/inha/{page}/', 'https://www.kuaidaili.com/free/intr/{page}/' ] for url in url_list: for page in range(1, 2): page_url = url.format(page=page) tree = EasyHttp.getHtmlTree(page_url) if tree is None: Log.w('http://www.kuaidaili.com无效') return [] proxy_list = tree.xpath('.//table//tr') for tr in proxy_list[1:]: yield ':'.join(tr.xpath('./td/text()')[0:2])
def _getQueueCount(self): formData = { # Thu+Jan+04+2018+00:00:00+GMT+0800 # 'train_date': datetime.strptime( # self.__ticket.ticketInfoForPassengerForm['queryLeftTicketRequestDTO']['train_date'], '%Y%m%d').strftime( # '%b+%a+%d+%Y+00:00:00+GMT+0800'), # Mon Jan 08 2018 00:00:00 GMT+0800 (中国标准时间) "train_date": datetime.strptime( self.__ticket.ticketInfoForPassengerForm[ "queryLeftTicketRequestDTO"]["train_date"], "%Y%m%d", ).strftime("%b %a %d %Y 00:00:00 GMT+0800") + " (中国标准时间)", "train_no": self.__ticket.ticketInfoForPassengerForm[ "queryLeftTicketRequestDTO"]["train_no"], "stationTrainCode": self.__ticket.trainNo, "seatType": self.__ticket.seatType, "fromStationTelecode": self.__ticket.fromStationCode, "toStationTelecode": self.__ticket.toStationCode, "leftTicket": self.__ticket.ticketInfoForPassengerForm["leftTicketStr"], "purpose_codes": self.__ticket.ticketInfoForPassengerForm["purpose_codes"], "train_location": self.__ticket.ticketInfoForPassengerForm["train_location"], "_json_att": "", "REPEAT_SUBMIT_TOKEN": self.__ticket.repeatSubmitToken, } jsonRet = EasyHttp.send(self._urlInfo["getQueueCount"], data=formData) return ( jsonRet["status"], jsonRet["messages"], jsonRet["data"]["ticket"] if "data" in jsonRet and "ticket" in jsonRet["data"] else -1, jsonRet["data"]["count"] if "data" in jsonRet and "count" in jsonRet["data"] else -1, )
def freeProxyWallSecond(): """ https://proxy-list.org/english/index.php :return: """ urls = [ 'https://proxy-list.org/english/index.php?p=%s' % n for n in range(1, 10) ] import base64 for url in urls: r = EasyHttp.get(url, timeout=10) if not r: Log.w('http://proxy-list.org/english/index.php无效') return [] proxies = re.findall(r"Proxy\('(.*?)'\)", r) for proxy in proxies: yield base64.b64decode(proxy).decode()
def _loginNormal(self, userName, userPwd, autoCheck=2, type=TYPE_LOGIN_NORMAL_WAY): status, msg = self._login_init() if not status: return status, msg self._uamtk_static() if autoCheck == CAPTCHA_CHECK_METHOD_THREE: results, verify = Captcha().verifyCodeAuto() elif autoCheck == CAPTCHA_CHECK_METHOD_HAND: results, verify = Captcha().verifyCaptchaByHand(type=type) else: results, verify = Captcha().verifyCodeAutoByMyself(type=type) if not verify: return False, '验证码识别错误!' Log.v('验证码识别成功') payload = OrderedDict() payload['username'] = userName payload['password'] = userPwd payload['appid'] = 'otn' payload['answer'] = results response = EasyHttp.post_custom(self._urlInfo['login'], data=payload) def isLoginSuccess(responseJson): return 0 == responseJson['result_code'] if responseJson and 'result_code' in responseJson else False, \ responseJson[ 'result_message'] if responseJson and 'result_message' in responseJson else '登录失败' if response.status_code != requests.codes.ok: return False, "登录请求被强制重定向,准备重试..." result, msg = isLoginSuccess(response.json()) if not result: return False, msg self._userLogin() self._passportRedirect() result, msg, apptk = self._uamtk() if not Utils.check(result, msg): return False, 'uamtk failed' return self._uamauthclient(apptk)
def _submitOrderRequest(self, tourFlag="dc"): formData = { "secretStr": TrainUtils.undecodeSecretStr(self.__ticket.secretStr), # 'secretStr': self.__ticket.secretStr, # 'secretStr': urllib.parse.unquote(self.__ticket.secretStr), "train_date": Utils.formatDate(self.__ticket.startDate), # 2018-01-04 "back_train_date": time.strftime("%Y-%m-%d", time.localtime()), # query date:2017-12-31 "tour_flag": tourFlag, "purpose_codes": self.__ticket.passengerType, "query_from_station_name": self.__ticket.fromStation, "query_to_station_name": self.__ticket.toStation, "undefined": "", } jsonRet = EasyHttp.send(self._urlInfo["submitOrderRequest"], data=formData) print("submitOrderRequest %s" % jsonRet) return jsonRet["status"], jsonRet["messages"]
def _checkOrderInfo(self, passengersDetails, seatType, ticketTypeCodes=1): formData = { 'cancel_flag': self.__ticket.ticketInfoForPassengerForm['orderRequestDTO']['cancel_flag'] or '2', 'bed_level_order_num': self.__ticket.ticketInfoForPassengerForm['orderRequestDTO'][ 'bed_level_order_num'] or '000000000000000000000000000000', 'passengerTicketStr': TrainUtils.passengerTicketStrs(seatType, passengersDetails, ticketTypeCodes), 'oldPassengerStr': TrainUtils.oldPassengerStrs(passengersDetails), 'tour_flag': self.__ticket.ticketInfoForPassengerForm['tour_flag'] or 'dc', 'randCode': '', 'whatsSelect': '1', '_json_att': '', 'REPEAT_SUBMIT_TOKEN': self.__ticket.repeatSubmitToken, } jsonRet = EasyHttp.send(self._urlInfo['checkOrderInfo'], data=formData) submitStatus = '' errMsg = '' if jsonRet['data']: submitStatus = jsonRet['data']['submitStatus'] errMsg = jsonRet['data']['errMsg'] if 'errMsg' in jsonRet['data'] else 'submit falied' return jsonRet['status'], jsonRet['messages'], submitStatus, errMsg
def freeProxyEleven(): """ IP海 http://www.iphai.com/free/ng :return: """ urls = [ 'http://www.iphai.com/free/ng', 'http://www.iphai.com/free/np', 'http://www.iphai.com/free/wg', 'http://www.iphai.com/free/wp' ] for url in urls: r = EasyHttp.get(url, timeout=10) if not r: Log.w('http://www.iphai.com无效') return [] proxies = re.findall( r'<td>\s*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s*?</td>[\s\S]*?<td>\s*?(\d+)\s*?</td>', r) for proxy in proxies: yield ":".join(proxy)
def _loginAsyncSuggest(self, userName, userPwd): self._init() results, verify = Captcha().verifyCaptchaByHand( type=TYPE_LOGIN_OTHER_WAY) if not verify: return False, '验证码识别错误!' formData = { 'loginUserDTO.user_name': userName, 'userDTO.password': userPwd, 'randCode': results, } jsonRet = EasyHttp.send(self._urlInfo['login'], data=formData) print('loginAsyncSuggest: %s' % jsonRet) def isSuccess(response): return response['status'] and response['data']['loginCheck'] == 'Y' if 'data' in response else False, \ response['data']['otherMsg'] if 'data' in response else response['messages'] loginSuccess, otherMsg = isSuccess(jsonRet) return loginSuccess, '%s:%s' % (userName, otherMsg or '登录成功!')
def _confirmSingleOrGoForQueue(self, passengersDetails): formData = { "passengerTicketStr": TrainUtils.passengerTicketStrs(self.__ticket.seatType, passengersDetails, self.__ticket.ticketTypeCodes), "oldPassengerStr": TrainUtils.oldPassengerStrs(passengersDetails), "randCode": "", "purpose_codes": self.__ticket.ticketInfoForPassengerForm["purpose_codes"], "key_check_isChange": self.__ticket.ticketInfoForPassengerForm["key_check_isChange"], "leftTicketStr": self.__ticket.ticketInfoForPassengerForm["leftTicketStr"], "train_location": self.__ticket.ticketInfoForPassengerForm["train_location"], "choose_seats": "".join(CHOOSE_SEATS) or "", "seatDetailType": "000", # todo::make clear 000 comes from "whatsSelect": "1", "roomType": "00", # todo::make clear this value comes from "dwAll": "N", "_json_att": "", "REPEAT_SUBMIT_TOKEN": self.__ticket.repeatSubmitToken, } jsonRet = EasyHttp.send(self._urlInfo["confirmForQueue"], data=formData) return ( jsonRet["status"], jsonRet["messages"], jsonRet["data"]["submitStatus"], jsonRet["data"]["errMsg"] if "errMsg" in jsonRet["data"] else None, )
def freeProxySecond(area=33, page=1): """ 代理66 http://www.66ip.cn/ :param area: 抓取代理页数,page=1北京代理页,page=2上海代理页...... :param page: 翻页 :return: """ area = 33 if area > 33 else area for area_index in range(1, area + 1): for i in range(1, page + 1): url = "http://www.66ip.cn/areaindex_{}/{}.html".format(area_index, i) html_tree = EasyHttp.getHtmlTree(url) if not html_tree: Log.w('http://www.66ip.cn无效') return [] tr_list = html_tree.xpath("//*[@id='footer']/div/table/tr[position()>1]") if len(tr_list) == 0: continue for tr in tr_list: yield tr.xpath("./td[1]/text()")[0] + ":" + tr.xpath("./td[2]/text()")[0] break
def _confirmSingleOrGoForQueue(self, passengersDetails,choose_seat): formData = { 'passengerTicketStr': TrainUtils.passengerTicketStrs(self.__ticket.seatType, passengersDetails, self.__ticket.ticketTypeCodes), 'oldPassengerStr': TrainUtils.oldPassengerStrs(passengersDetails), 'randCode': '', 'purpose_codes': self.__ticket.ticketInfoForPassengerForm['purpose_codes'], 'key_check_isChange': self.__ticket.ticketInfoForPassengerForm['key_check_isChange'], 'leftTicketStr': self.__ticket.ticketInfoForPassengerForm['leftTicketStr'], 'train_location': self.__ticket.ticketInfoForPassengerForm['train_location'], 'choose_seats': ''.join(choose_seat) or '', 'seatDetailType': '000', # todo::make clear 000 comes from 'whatsSelect': '1', 'roomType': '00', # todo::make clear this value comes from 'dwAll': 'N', '_json_att': '', 'REPEAT_SUBMIT_TOKEN': self.__ticket.repeatSubmitToken, } jsonRet = EasyHttp.send(self._urlInfo['confirmForQueue'], data=formData) return jsonRet['status'], jsonRet['messages'], jsonRet['data']['submitStatus'], jsonRet['data'][ 'errMsg'] if 'errMsg' in jsonRet['data'] else None
def _getQueueCount(self): formData = { # Thu+Jan+04+2018+00:00:00+GMT+0800 # 'train_date': datetime.strptime( # self.__ticket.ticketInfoForPassengerForm['queryLeftTicketRequestDTO']['train_date'], '%Y%m%d').strftime( # '%b+%a+%d+%Y+00:00:00+GMT+0800'), # Mon Jan 08 2018 00:00:00 GMT+0800 (中国标准时间) 'train_date': datetime.strptime( self.__ticket.ticketInfoForPassengerForm[ 'queryLeftTicketRequestDTO']['train_date'], '%Y%m%d').strftime('%b %a %d %Y 00:00:00 GMT+0800') + ' (中国标准时间)', 'train_no': self.__ticket.ticketInfoForPassengerForm[ 'queryLeftTicketRequestDTO']['train_no'], 'stationTrainCode': self.__ticket.trainNo, 'seatType': self.__ticket.seatType, 'fromStationTelecode': self.__ticket.fromStationCode, 'toStationTelecode': self.__ticket.toStationCode, 'leftTicket': self.__ticket.ticketInfoForPassengerForm['leftTicketStr'], 'purpose_codes': self.__ticket.ticketInfoForPassengerForm['purpose_codes'], 'train_location': self.__ticket.ticketInfoForPassengerForm['train_location'], '_json_att': '', 'REPEAT_SUBMIT_TOKEN': self.__ticket.repeatSubmitToken, } jsonRet = EasyHttp.send(self._urlInfo['getQueueCount'], data=formData) return jsonRet['status'], jsonRet['messages'], \ jsonRet['data']['ticket'] if 'data' in jsonRet and 'ticket' in jsonRet['data'] else -1, \ jsonRet['data']['count'] if 'data' in jsonRet and 'count' in jsonRet['data'] else -1
def freeProxyFourth(page_count=2): """ 西刺代理 http://www.xicidaili.com :return: """ url_list = [ 'http://www.xicidaili.com/nn/', # 高匿 'http://www.xicidaili.com/nt/', # 透明 ] for each_url in url_list: for i in range(1, page_count + 1): page_url = each_url + str(i) tree = EasyHttp.getHtmlTree(page_url) if not tree: Log.w('http://www.xicidaili.com无效') return [] proxy_list = tree.xpath('.//table[@id="ip_list"]//tr[position()>1]') for proxy in proxy_list: try: yield ':'.join(proxy.xpath('./td/text()')[0:2]) except Exception as e: pass
def _loginNormal(self, userName, userPwd, autoCheck=2): status, msg = self._login_init() if not status: return status, msg self._uamtk_static() if autoCheck == CAPTCHA_CHECK_METHOD_THREE: results, verify = Captcha().verifyCodeAuto() elif autoCheck == CAPTCHA_CHECK_METHOD_HAND: results, verify = Captcha().verifyCaptchaByHand() else: results, verify = Captcha().verifyCodeAutoByMyself() if not verify: return False, '验证码识别错误!' Log.v('验证码识别成功') payload = OrderedDict() payload['username'] = userName payload['password'] = userPwd payload['appid'] = 'otn' payload['answer'] = results jsonRet = EasyHttp.send(self._urlInfo['login'], data=payload) def isLoginSuccess(responseJson): return 0 == responseJson['result_code'] if responseJson and 'result_code' in responseJson else False, \ responseJson[ 'result_message'] if responseJson and 'result_message' in responseJson else '登录失败' result, msg = isLoginSuccess(jsonRet) if not result: return False, msg self._userLogin() self._passportRedirect() result, msg, apptk = self._uamtk() if not Utils.check(result, msg): return False, 'uamtk failed' return self._uamauthclient(apptk)
def _getExtraInfo(self): def getRepeatSubmitToken(html): repeatSubmitToken = re.findall( r"var globalRepeatSubmitToken = '(.*)'", html)[0] print("RepeatSubmitToken = %s" % repeatSubmitToken) return repeatSubmitToken html = EasyHttp.send(self._urlInfo["getExtraInfo"]) if not Utils.check( html, "getExtraInfoUrl: failed to visit %s" % self._urlInfo["getExtraInfo"]["url"], ): return False self.__ticket.repeatSubmitToken = getRepeatSubmitToken(html) def decodeTicketInfoForPassengerForm(html): ticketInfoForPassengerForm = re.findall( r"var ticketInfoForPassengerForm=(.*);", html)[0] return json.loads(ticketInfoForPassengerForm.replace("'", '"')) self.__ticket.ticketInfoForPassengerForm = decodeTicketInfoForPassengerForm( html) return True