def start_requests(self):
print 'scrapy start requests'
if self.mobile_number:
mobile_number_array = []
if str(self.mobile_number).find(",") == -1:
mobile_number_array.append(self.mobile_number)
else:
mobile_number_array = str(self.mobile_number).split(',')
for t_mobile_number in mobile_number_array:
request = scrapy.Request(
url=crypt.get_posturl(),
method='POST',
body=crypt.get_poststr(t_mobile_number),
headers={
'X-CLIENT-PFM': '20',
'X-CLIENT-VCODE': '81',
'X-CLIENT-PID': '8888888',
'Content-Type': 'application/json; charset=utf-8',
'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE',
'Accept-Encoding': 'gzip',
}
)
request.meta['mobile'] = t_mobile_number
request.meta['msk'] = crypt.sk
request.meta['mtk'] = crypt.tk
request.meta['muid'] = crypt.uid
yield request
def start_sequence_requests(self):
print 'start_requests'
parent_dir = os.path.dirname(__file__) # 获取当前文件夹的绝对路径
file_path = os.path.join(parent_dir, 'mobilecode.txt')
with open(file_path, 'r') as f:
line = f.readline()
while line:
print 'fetching mobile code : ' + line.strip()
seed = int(line.strip()) * 10000
for i in xrange(10000):
mobile_number = seed + i
request = scrapy.Request(
url=crypt.get_posturl(),
method='POST',
body=crypt.get_poststr(str(mobile_number)),
headers={
'X-CLIENT-PFM': '20',
'X-CLIENT-VCODE': '81',
'X-CLIENT-PID': '8888888',
'Content-Type': 'application/json; charset=utf-8',
'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE',
'Accept-Encoding': 'gzip',
}
)
request.meta['mobile'] = str(mobile_number)
request.meta['msk'] = crypt.sk
request.meta['mtk'] = crypt.tk
request.meta['muid'] = crypt.uid
yield request
line = f.readline()
def fetch_data(self, mobile_number):
request = scrapy.Request(
url=crypt.get_posturl(),
method='POST',
body=crypt.get_poststr(mobile_number),
headers={
'X-CLIENT-PFM': '20',
'X-CLIENT-VCODE': '81',
'X-CLIENT-PID': '8888888',
'Content-Type': 'application/json; charset=utf-8',
'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE',
'Accept-Encoding': 'gzip',
}
)
request.meta['mobile'] = mobile_number
request.meta['msk'] = crypt.sk
request.meta['mtk'] = crypt.tk
request.meta['muid'] = crypt.uid
settings = get_project_settings()
downloader = HTTP11DownloadHandler(settings)
deferred = downloader.download_request(request)
deferred.addCallback(self.parse_response, request)
deferred.addErrback(self.parse_error)
reactor.run()
def parse(self, response):
try:
json_obj = json.loads(response.body)
if str(json_obj['resultCode'] == 0):
json_str = crypt.decrypt_mobile_sk(
json_obj['data'], str(response.request.meta['msk']))
print json_str
data = json.loads(json_str)
# if str(data['n']) != '':
item = MobileItem()
item['jsonstr'] = json_str
item['name'] = str(data['n'])
item['mobile'] = str(data['p'])
item['timestamp'] = int(time.time() * 300)
item['_id'] = crypt.get_md5(str(data['p']))
yield item
elif str(json_obj['resultCode']) == '-1':
print 'result code : -1'
print 'response body: ' + str(response.body)
print 'uid: ' + str(crypt.uid) + ' tk: ' + str(
crypt.tk) + ' sk: ' + str(crypt.sk)
else:
if response.request.meta['sk'] == crypt.sk:
# sendEmail("change auth")
if mutex.acquire(10):
if not crypt.is_changing:
print 'changing auth'
if crypt.change_auth():
print 'change auth success'
else:
print 'change auth failure'
else:
while crypt.is_changing:
time.sleep(1)
mutex.release()
request = scrapy.Request(
url=crypt.get_posturl(),
method='POST',
body=crypt.get_poststr(response.request.meta['mobile']),
headers={
'X-CLIENT-PFM': '20',
'X-CLIENT-VCODE': '81',
'X-CLIENT-PID': '8888888',
'Content-Type': 'application/json; charset=utf-8',
'User-Agent':
'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE',
'Accept-Encoding': 'gzip',
})
request.meta['mobile'] = response.request.meta['mobile']
request.meta['msk'] = crypt.sk
request.meta['mtk'] = crypt.tk
request.meta['muid'] = crypt.uid
yield request
# '{"resultCode":1400,"errorMsg":"req invalid"}'
print json_obj['resultCode']
print json_obj['errorMsg']
except Exception, e:
print e.message
print traceback.format_exc()
def start_sequence_requests(self):
print 'start_requests'
parent_dir = os.path.dirname(__file__) # 获取当前文件夹的绝对路径
file_path = os.path.join(parent_dir, 'mobilecode.txt')
with open(file_path, 'r') as f:
line = f.readline()
while line:
print 'fetching mobile code : ' + line.strip()
seed = int(line.strip()) * 10000
for i in xrange(10000):
mobile_number = seed + i
request = scrapy.Request(
url=crypt.get_posturl(),
method='POST',
body=crypt.get_poststr(str(mobile_number)),
headers={
'X-CLIENT-PFM': '20',
'X-CLIENT-VCODE': '81',
'X-CLIENT-PID': '8888888',
'Content-Type': 'application/json; charset=utf-8',
'User-Agent':
'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE',
'Accept-Encoding': 'gzip',
})
request.meta['mobile'] = str(mobile_number)
request.meta['msk'] = crypt.sk
request.meta['mtk'] = crypt.tk
request.meta['muid'] = crypt.uid
yield request
line = f.readline()
def parse(self, response):
try:
json_obj = json.loads(response.body)
if str(json_obj['resultCode'] == 0):
json_str = crypt.decrypt_mobile_sk(json_obj['data'], str(response.request.meta['msk']))
print json_str
data = json.loads(json_str)
# if str(data['n']) != '':
item = MobileItem()
item['jsonstr'] = json_str
item['name'] = str(data['n'])
item['mobile'] = str(data['p'])
item['timestamp'] = int(time.time() * 300)
item['_id'] = crypt.get_md5(str(data['p']))
yield item
elif str(json_obj['resultCode']) == '-1':
print 'result code : -1'
print 'response body: ' + str(response.body)
print 'uid: ' + str(crypt.uid) + ' tk: ' + str(crypt.tk) + ' sk: ' + str(crypt.sk)
else:
if response.request.meta['sk'] == crypt.sk:
# sendEmail("change auth")
if mutex.acquire(10):
if not crypt.is_changing:
print 'changing auth'
if crypt.change_auth():
print 'change auth success'
else:
print 'change auth failure'
else:
while crypt.is_changing:
time.sleep(1)
mutex.release()
request = scrapy.Request(
url=crypt.get_posturl(),
method='POST',
body=crypt.get_poststr(response.request.meta['mobile']),
headers={
'X-CLIENT-PFM': '20',
'X-CLIENT-VCODE': '81',
'X-CLIENT-PID': '8888888',
'Content-Type': 'application/json; charset=utf-8',
'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE',
'Accept-Encoding': 'gzip',
}
)
request.meta['mobile'] = response.request.meta['mobile']
request.meta['msk'] = crypt.sk
request.meta['mtk'] = crypt.tk
request.meta['muid'] = crypt.uid
yield request
# '{"resultCode":1400,"errorMsg":"req invalid"}'
print json_obj['resultCode']
print json_obj['errorMsg']
except Exception, e:
print e.message
print traceback.format_exc()
def fetch_mobile_number_by_proxyip(mobile_number):
session = Session()
url = crypt.get_posturl()
data = crypt.get_poststr(mobile_number)
headers = {
'X-CLIENT-PFM': '20',
'X-CLIENT-VCODE': '81',
'X-CLIENT-PID': '8888888',
'Content-Type': 'application/json; charset=utf-8',
'User-Agent':
'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE',
'Accept-Encoding': 'gzip',
}
req = Request('POST', url, data=data, headers=headers)
prepare_request = req.prepare()
# do something with prepareReq.body
# prepareReq.body = 'No, I want exactly this as the body.'
# do something with prepareReq.headers
# del prepareReq.headers['Content-Type']
# resp = s.send(prepareReq,
# stream=stream,
# verify=verify,
# proxies=proxies,
# cert=cert,
# timeout=timeout
# )
# proxies = ["http://45.32.108.74:8080"]
# proxies = {
# 'http': 'http://45.32.108.74:8080',
# 'http': 'http://61.231.188.203:8080',
# 'http': 'http://101.254.188.198:8080',
# 'http': 'http://61.160.212.74:3128',
# 'http': 'http://60.206.148.135:3128',
# }
for i in xrange(10):
try:
proxies = selenium_crawl_proxy_id()
resp = session.send(prepare_request, proxies=proxies)
print resp
except requests.exceptions.ProxyError, pe:
print pe.message
def fetch_mobile_number(self, mobile_number):
request = scrapy.Request(
url=crypt.get_posturl(),
method='POST',
body=crypt.get_poststr(mobile_number),
headers={
'X-CLIENT-PFM': '20',
'X-CLIENT-VCODE': '81',
'X-CLIENT-PID': '8888888',
'Content-Type': 'application/json; charset=utf-8',
'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE',
'Accept-Encoding': 'gzip',
}
)
request.meta['mobile'] = mobile_number
request.meta['msk'] = crypt.sk
request.meta['mtk'] = crypt.tk
request.meta['muid'] = crypt.uid
def pushQueue(mobile_number):
request = scrapy.Request(
url=crypt.get_posturl(),
method='POST',
body=crypt.get_poststr(mobile_number),
headers={
'X-CLIENT-PFM': '20',
'X-CLIENT-VCODE': '81',
'X-CLIENT-PID': '8888888',
'Content-Type': 'application/json; charset=utf-8',
'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE',
'Accept-Encoding': 'gzip',
}
)
request.meta['mobile'] = mobile_number
request.meta['msk'] = crypt.sk
request.meta['mtk'] = crypt.tk
request.meta['muid'] = crypt.uid
redis_client = redis.Redis(host='192.168.0.21', port=6379)
redis_client.zadd('mobile_online_spider:requests', _encode_request(request), 1.0)
def fetch_mobile_number(mobile_number):
print 'fetch mobile number: ' + mobile_number
url = crypt.get_posturl()
data = crypt.get_poststr(mobile_number)
headers = {
'X-CLIENT-PFM': '20',
'X-CLIENT-VCODE': '81',
'X-CLIENT-PID': '8888888',
'Content-Type': 'application/json; charset=utf-8',
'User-Agent':
'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE',
'Accept-Encoding': 'gzip',
}
meta = {
"mobile": mobile_number,
"msk": crypt.sk,
"mtk": crypt.tk,
"muid": crypt.uid
}
response = requests.post(url, data, headers=headers)
print response
print response.content
def pushQueue(mobile_number):
request = scrapy.Request(
url=crypt.get_posturl(),
method='POST',
body=crypt.get_poststr(mobile_number),
headers={
'X-CLIENT-PFM': '20',
'X-CLIENT-VCODE': '81',
'X-CLIENT-PID': '8888888',
'Content-Type': 'application/json; charset=utf-8',
'User-Agent':
'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE',
'Accept-Encoding': 'gzip',
})
request.meta['mobile'] = mobile_number
request.meta['msk'] = crypt.sk
request.meta['mtk'] = crypt.tk
request.meta['muid'] = crypt.uid
redis_client = redis.Redis(host='192.168.0.21', port=6379)
redis_client.zadd('mobile_online_spider:requests',
_encode_request(request), 1.0)
def start_requests(self):
print 'start_requests'
if self.mobile_number:
mobile_number_array = []
if str(self.mobile_number).find(",") == -1:
mobile_number_array.append(self.mobile_number)
else:
mobile_number_array = str(self.mobile_number).split(',')
for t_mobile_number in mobile_number_array:
request = scrapy.Request(
url=crypt.get_posturl(),
method='POST',
body=crypt.get_poststr(t_mobile_number),
headers={
'X-CLIENT-PFM': '20',
'X-CLIENT-VCODE': '81',
'X-CLIENT-PID': '8888888',
'Content-Type': 'application/json; charset=utf-8',
'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE',
'Accept-Encoding': 'gzip',
}
)
request.meta['mobile'] = t_mobile_number
request.meta['msk'] = crypt.sk
request.meta['mtk'] = crypt.tk
request.meta['muid'] = crypt.uid
yield request
else:
parent_dir = os.path.dirname(__file__) # 获取当前文件夹的绝对路径
file_path = os.path.join(parent_dir, 'mobilecode.txt')
mobile_number_segments = []
with open(file_path, 'r') as f:
line = f.readline()
while line:
mobile_number_segments.append(line.strip())
line = f.readline()
for mobile_number_segments_index in xrange(len(mobile_number_segments)):
mobile_number_segment = random.choice(mobile_number_segments)
print 'fetching mobile code : ' + mobile_number_segment
seed = int(mobile_number_segment) * 10000
random_num = random.randint(1, 9)
for i in xrange(10000):
mobile_number = seed + random_num * 1111 - i
request = scrapy.Request(
url=crypt.get_posturl(),
method='POST',
body=crypt.get_poststr(str(mobile_number)),
headers={
'X-CLIENT-PFM': '20',
'X-CLIENT-VCODE': '81',
'X-CLIENT-PID': '8888888',
'Content-Type': 'application/json; charset=utf-8',
'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE',
'Accept-Encoding': 'gzip',
}
)
request.meta['mobile'] = str(mobile_number)
request.meta['msk'] = crypt.sk
request.meta['mtk'] = crypt.tk
request.meta['muid'] = crypt.uid
yield request
def start_requests(self):
print 'start_requests'
if self.mobile_number:
mobile_number_array = []
if str(self.mobile_number).find(",") == -1:
mobile_number_array.append(self.mobile_number)
else:
mobile_number_array = str(self.mobile_number).split(',')
for t_mobile_number in mobile_number_array:
request = scrapy.Request(
url=crypt.get_posturl(),
method='POST',
body=crypt.get_poststr(t_mobile_number),
headers={
'X-CLIENT-PFM': '20',
'X-CLIENT-VCODE': '81',
'X-CLIENT-PID': '8888888',
'Content-Type': 'application/json; charset=utf-8',
'User-Agent':
'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE',
'Accept-Encoding': 'gzip',
})
request.meta['mobile'] = t_mobile_number
request.meta['msk'] = crypt.sk
request.meta['mtk'] = crypt.tk
request.meta['muid'] = crypt.uid
yield request
else:
parent_dir = os.path.dirname(__file__) # 获取当前文件夹的绝对路径
file_path = os.path.join(parent_dir, 'mobilecode.txt')
mobile_number_segments = []
with open(file_path, 'r') as f:
line = f.readline()
while line:
mobile_number_segments.append(line.strip())
line = f.readline()
for mobile_number_segments_index in xrange(
len(mobile_number_segments)):
mobile_number_segment = random.choice(mobile_number_segments)
print 'fetching mobile code : ' + mobile_number_segment
seed = int(mobile_number_segment) * 10000
random_num = random.randint(1, 9)
for i in xrange(10000):
mobile_number = seed + random_num * 1111 - i
request = scrapy.Request(
url=crypt.get_posturl(),
method='POST',
body=crypt.get_poststr(str(mobile_number)),
headers={
'X-CLIENT-PFM': '20',
'X-CLIENT-VCODE': '81',
'X-CLIENT-PID': '8888888',
'Content-Type': 'application/json; charset=utf-8',
'User-Agent':
'Dalvik/2.1.0 (Linux; U; Android 5.0.2; Redmi Note 2 MIUI/V7.5.5.0.LHMCNDE',
'Accept-Encoding': 'gzip',
})
request.meta['mobile'] = str(mobile_number)
request.meta['msk'] = crypt.sk
request.meta['mtk'] = crypt.tk
request.meta['muid'] = crypt.uid
yield request
