def start_requests(self): print 'scrapy start requests' if self.mobile_number: mobile_number_array = [] if str(self.mobile_number).find(",") == -1: mobile_number_array.append(self.mobile_number) else: mobile_number_array = str(self.mobile_number).split(',') for t_mobile_number in mobile_number_array: request = scrapy.Request( url=crypt.get_posturl(), method='POST', body=crypt.get_poststr(t_mobile_number), headers={ 'X-CLIENT-PFM': '20', 'X-CLIENT-VCODE': '81', 'X-CLIENT-PID': '8888888', 'Content-Type': 'application/json; charset=utf-8', 'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE', 'Accept-Encoding': 'gzip', } ) request.meta['mobile'] = t_mobile_number request.meta['msk'] = crypt.sk request.meta['mtk'] = crypt.tk request.meta['muid'] = crypt.uid yield request
def start_sequence_requests(self): print 'start_requests' parent_dir = os.path.dirname(__file__) # 获取当前文件夹的绝对路径 file_path = os.path.join(parent_dir, 'mobilecode.txt') with open(file_path, 'r') as f: line = f.readline() while line: print 'fetching mobile code : ' + line.strip() seed = int(line.strip()) * 10000 for i in xrange(10000): mobile_number = seed + i request = scrapy.Request( url=crypt.get_posturl(), method='POST', body=crypt.get_poststr(str(mobile_number)), headers={ 'X-CLIENT-PFM': '20', 'X-CLIENT-VCODE': '81', 'X-CLIENT-PID': '8888888', 'Content-Type': 'application/json; charset=utf-8', 'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE', 'Accept-Encoding': 'gzip', } ) request.meta['mobile'] = str(mobile_number) request.meta['msk'] = crypt.sk request.meta['mtk'] = crypt.tk request.meta['muid'] = crypt.uid yield request line = f.readline()
def fetch_data(self, mobile_number): request = scrapy.Request( url=crypt.get_posturl(), method='POST', body=crypt.get_poststr(mobile_number), headers={ 'X-CLIENT-PFM': '20', 'X-CLIENT-VCODE': '81', 'X-CLIENT-PID': '8888888', 'Content-Type': 'application/json; charset=utf-8', 'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE', 'Accept-Encoding': 'gzip', } ) request.meta['mobile'] = mobile_number request.meta['msk'] = crypt.sk request.meta['mtk'] = crypt.tk request.meta['muid'] = crypt.uid settings = get_project_settings() downloader = HTTP11DownloadHandler(settings) deferred = downloader.download_request(request) deferred.addCallback(self.parse_response, request) deferred.addErrback(self.parse_error) reactor.run()
def parse(self, response): try: json_obj = json.loads(response.body) if str(json_obj['resultCode'] == 0): json_str = crypt.decrypt_mobile_sk( json_obj['data'], str(response.request.meta['msk'])) print json_str data = json.loads(json_str) # if str(data['n']) != '': item = MobileItem() item['jsonstr'] = json_str item['name'] = str(data['n']) item['mobile'] = str(data['p']) item['timestamp'] = int(time.time() * 300) item['_id'] = crypt.get_md5(str(data['p'])) yield item elif str(json_obj['resultCode']) == '-1': print 'result code : -1' print 'response body: ' + str(response.body) print 'uid: ' + str(crypt.uid) + ' tk: ' + str( crypt.tk) + ' sk: ' + str(crypt.sk) else: if response.request.meta['sk'] == crypt.sk: # sendEmail("change auth") if mutex.acquire(10): if not crypt.is_changing: print 'changing auth' if crypt.change_auth(): print 'change auth success' else: print 'change auth failure' else: while crypt.is_changing: time.sleep(1) mutex.release() request = scrapy.Request( url=crypt.get_posturl(), method='POST', body=crypt.get_poststr(response.request.meta['mobile']), headers={ 'X-CLIENT-PFM': '20', 'X-CLIENT-VCODE': '81', 'X-CLIENT-PID': '8888888', 'Content-Type': 'application/json; charset=utf-8', 'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE', 'Accept-Encoding': 'gzip', }) request.meta['mobile'] = response.request.meta['mobile'] request.meta['msk'] = crypt.sk request.meta['mtk'] = crypt.tk request.meta['muid'] = crypt.uid yield request # '{"resultCode":1400,"errorMsg":"req invalid"}' print json_obj['resultCode'] print json_obj['errorMsg'] except Exception, e: print e.message print traceback.format_exc()
def start_sequence_requests(self): print 'start_requests' parent_dir = os.path.dirname(__file__) # 获取当前文件夹的绝对路径 file_path = os.path.join(parent_dir, 'mobilecode.txt') with open(file_path, 'r') as f: line = f.readline() while line: print 'fetching mobile code : ' + line.strip() seed = int(line.strip()) * 10000 for i in xrange(10000): mobile_number = seed + i request = scrapy.Request( url=crypt.get_posturl(), method='POST', body=crypt.get_poststr(str(mobile_number)), headers={ 'X-CLIENT-PFM': '20', 'X-CLIENT-VCODE': '81', 'X-CLIENT-PID': '8888888', 'Content-Type': 'application/json; charset=utf-8', 'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE', 'Accept-Encoding': 'gzip', }) request.meta['mobile'] = str(mobile_number) request.meta['msk'] = crypt.sk request.meta['mtk'] = crypt.tk request.meta['muid'] = crypt.uid yield request line = f.readline()
def parse(self, response): try: json_obj = json.loads(response.body) if str(json_obj['resultCode'] == 0): json_str = crypt.decrypt_mobile_sk(json_obj['data'], str(response.request.meta['msk'])) print json_str data = json.loads(json_str) # if str(data['n']) != '': item = MobileItem() item['jsonstr'] = json_str item['name'] = str(data['n']) item['mobile'] = str(data['p']) item['timestamp'] = int(time.time() * 300) item['_id'] = crypt.get_md5(str(data['p'])) yield item elif str(json_obj['resultCode']) == '-1': print 'result code : -1' print 'response body: ' + str(response.body) print 'uid: ' + str(crypt.uid) + ' tk: ' + str(crypt.tk) + ' sk: ' + str(crypt.sk) else: if response.request.meta['sk'] == crypt.sk: # sendEmail("change auth") if mutex.acquire(10): if not crypt.is_changing: print 'changing auth' if crypt.change_auth(): print 'change auth success' else: print 'change auth failure' else: while crypt.is_changing: time.sleep(1) mutex.release() request = scrapy.Request( url=crypt.get_posturl(), method='POST', body=crypt.get_poststr(response.request.meta['mobile']), headers={ 'X-CLIENT-PFM': '20', 'X-CLIENT-VCODE': '81', 'X-CLIENT-PID': '8888888', 'Content-Type': 'application/json; charset=utf-8', 'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE', 'Accept-Encoding': 'gzip', } ) request.meta['mobile'] = response.request.meta['mobile'] request.meta['msk'] = crypt.sk request.meta['mtk'] = crypt.tk request.meta['muid'] = crypt.uid yield request # '{"resultCode":1400,"errorMsg":"req invalid"}' print json_obj['resultCode'] print json_obj['errorMsg'] except Exception, e: print e.message print traceback.format_exc()
def fetch_mobile_number_by_proxyip(mobile_number): session = Session() url = crypt.get_posturl() data = crypt.get_poststr(mobile_number) headers = { 'X-CLIENT-PFM': '20', 'X-CLIENT-VCODE': '81', 'X-CLIENT-PID': '8888888', 'Content-Type': 'application/json; charset=utf-8', 'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE', 'Accept-Encoding': 'gzip', } req = Request('POST', url, data=data, headers=headers) prepare_request = req.prepare() # do something with prepareReq.body # prepareReq.body = 'No, I want exactly this as the body.' # do something with prepareReq.headers # del prepareReq.headers['Content-Type'] # resp = s.send(prepareReq, # stream=stream, # verify=verify, # proxies=proxies, # cert=cert, # timeout=timeout # ) # proxies = ["http://45.32.108.74:8080"] # proxies = { # 'http': 'http://45.32.108.74:8080', # 'http': 'http://61.231.188.203:8080', # 'http': 'http://101.254.188.198:8080', # 'http': 'http://61.160.212.74:3128', # 'http': 'http://60.206.148.135:3128', # } for i in xrange(10): try: proxies = selenium_crawl_proxy_id() resp = session.send(prepare_request, proxies=proxies) print resp except requests.exceptions.ProxyError, pe: print pe.message
def fetch_mobile_number(self, mobile_number): request = scrapy.Request( url=crypt.get_posturl(), method='POST', body=crypt.get_poststr(mobile_number), headers={ 'X-CLIENT-PFM': '20', 'X-CLIENT-VCODE': '81', 'X-CLIENT-PID': '8888888', 'Content-Type': 'application/json; charset=utf-8', 'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE', 'Accept-Encoding': 'gzip', } ) request.meta['mobile'] = mobile_number request.meta['msk'] = crypt.sk request.meta['mtk'] = crypt.tk request.meta['muid'] = crypt.uid
def pushQueue(mobile_number): request = scrapy.Request( url=crypt.get_posturl(), method='POST', body=crypt.get_poststr(mobile_number), headers={ 'X-CLIENT-PFM': '20', 'X-CLIENT-VCODE': '81', 'X-CLIENT-PID': '8888888', 'Content-Type': 'application/json; charset=utf-8', 'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE', 'Accept-Encoding': 'gzip', } ) request.meta['mobile'] = mobile_number request.meta['msk'] = crypt.sk request.meta['mtk'] = crypt.tk request.meta['muid'] = crypt.uid redis_client = redis.Redis(host='192.168.0.21', port=6379) redis_client.zadd('mobile_online_spider:requests', _encode_request(request), 1.0)
def fetch_mobile_number(mobile_number): print 'fetch mobile number: ' + mobile_number url = crypt.get_posturl() data = crypt.get_poststr(mobile_number) headers = { 'X-CLIENT-PFM': '20', 'X-CLIENT-VCODE': '81', 'X-CLIENT-PID': '8888888', 'Content-Type': 'application/json; charset=utf-8', 'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE', 'Accept-Encoding': 'gzip', } meta = { "mobile": mobile_number, "msk": crypt.sk, "mtk": crypt.tk, "muid": crypt.uid } response = requests.post(url, data, headers=headers) print response print response.content
def pushQueue(mobile_number): request = scrapy.Request( url=crypt.get_posturl(), method='POST', body=crypt.get_poststr(mobile_number), headers={ 'X-CLIENT-PFM': '20', 'X-CLIENT-VCODE': '81', 'X-CLIENT-PID': '8888888', 'Content-Type': 'application/json; charset=utf-8', 'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE', 'Accept-Encoding': 'gzip', }) request.meta['mobile'] = mobile_number request.meta['msk'] = crypt.sk request.meta['mtk'] = crypt.tk request.meta['muid'] = crypt.uid redis_client = redis.Redis(host='192.168.0.21', port=6379) redis_client.zadd('mobile_online_spider:requests', _encode_request(request), 1.0)
def start_requests(self): print 'start_requests' if self.mobile_number: mobile_number_array = [] if str(self.mobile_number).find(",") == -1: mobile_number_array.append(self.mobile_number) else: mobile_number_array = str(self.mobile_number).split(',') for t_mobile_number in mobile_number_array: request = scrapy.Request( url=crypt.get_posturl(), method='POST', body=crypt.get_poststr(t_mobile_number), headers={ 'X-CLIENT-PFM': '20', 'X-CLIENT-VCODE': '81', 'X-CLIENT-PID': '8888888', 'Content-Type': 'application/json; charset=utf-8', 'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE', 'Accept-Encoding': 'gzip', } ) request.meta['mobile'] = t_mobile_number request.meta['msk'] = crypt.sk request.meta['mtk'] = crypt.tk request.meta['muid'] = crypt.uid yield request else: parent_dir = os.path.dirname(__file__) # 获取当前文件夹的绝对路径 file_path = os.path.join(parent_dir, 'mobilecode.txt') mobile_number_segments = [] with open(file_path, 'r') as f: line = f.readline() while line: mobile_number_segments.append(line.strip()) line = f.readline() for mobile_number_segments_index in xrange(len(mobile_number_segments)): mobile_number_segment = random.choice(mobile_number_segments) print 'fetching mobile code : ' + mobile_number_segment seed = int(mobile_number_segment) * 10000 random_num = random.randint(1, 9) for i in xrange(10000): mobile_number = seed + random_num * 1111 - i request = scrapy.Request( url=crypt.get_posturl(), method='POST', body=crypt.get_poststr(str(mobile_number)), headers={ 'X-CLIENT-PFM': '20', 'X-CLIENT-VCODE': '81', 'X-CLIENT-PID': '8888888', 'Content-Type': 'application/json; charset=utf-8', 'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE', 'Accept-Encoding': 'gzip', } ) request.meta['mobile'] = str(mobile_number) request.meta['msk'] = crypt.sk request.meta['mtk'] = crypt.tk request.meta['muid'] = crypt.uid yield request
def start_requests(self): print 'start_requests' if self.mobile_number: mobile_number_array = [] if str(self.mobile_number).find(",") == -1: mobile_number_array.append(self.mobile_number) else: mobile_number_array = str(self.mobile_number).split(',') for t_mobile_number in mobile_number_array: request = scrapy.Request( url=crypt.get_posturl(), method='POST', body=crypt.get_poststr(t_mobile_number), headers={ 'X-CLIENT-PFM': '20', 'X-CLIENT-VCODE': '81', 'X-CLIENT-PID': '8888888', 'Content-Type': 'application/json; charset=utf-8', 'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE', 'Accept-Encoding': 'gzip', }) request.meta['mobile'] = t_mobile_number request.meta['msk'] = crypt.sk request.meta['mtk'] = crypt.tk request.meta['muid'] = crypt.uid yield request else: parent_dir = os.path.dirname(__file__) # 获取当前文件夹的绝对路径 file_path = os.path.join(parent_dir, 'mobilecode.txt') mobile_number_segments = [] with open(file_path, 'r') as f: line = f.readline() while line: mobile_number_segments.append(line.strip()) line = f.readline() for mobile_number_segments_index in xrange( len(mobile_number_segments)): mobile_number_segment = random.choice(mobile_number_segments) print 'fetching mobile code : ' + mobile_number_segment seed = int(mobile_number_segment) * 10000 random_num = random.randint(1, 9) for i in xrange(10000): mobile_number = seed + random_num * 1111 - i request = scrapy.Request( url=crypt.get_posturl(), method='POST', body=crypt.get_poststr(str(mobile_number)), headers={ 'X-CLIENT-PFM': '20', 'X-CLIENT-VCODE': '81', 'X-CLIENT-PID': '8888888', 'Content-Type': 'application/json; charset=utf-8', 'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE', 'Accept-Encoding': 'gzip', }) request.meta['mobile'] = str(mobile_number) request.meta['msk'] = crypt.sk request.meta['mtk'] = crypt.tk request.meta['muid'] = crypt.uid yield request