def add_new() -> None:
question, answer = None, None
while not question:
question = input('Question:\n')
while not answer:
answer = input('Answer:\n')
DBWorker.add(question, answer)
def edit(row):
print('current question:', row.question)
question = input('please write a new question:\n')
if not question:
question = row.question
print('current answer:', row.answer)
answer = input('please write a new answer:\n')
if not answer:
answer = row.answer
DBWorker.edit(row, question, answer)
def change(row):
allowed = ('d', 'e')
input_text = 'press "d" to delete the flashcard:\npress "e" to edit the flashcard:\n'
user_choice = check_input(input_text,
proper_values=allowed,
message='is not an option')
if user_choice == 'e':
Flashcards.edit(row)
if user_choice == 'd':
DBWorker.delete(row)
def learning(row):
allowed = ('y', 'n')
input_text = 'press "y" if your answer is correct:\npress "n" if your answer is wrong:\n'
user_choice = check_input(input_text,
proper_values=allowed,
message='is not an option')
if user_choice == 'y':
DBWorker.session(row, row.session + 1)
else:
DBWorker.session(row, 1)
def practice() -> None:
rows = DBWorker.get_all()
if not rows:
print('There is no flashcard to practice!')
return
for row in rows:
allowed = ('y', 'n', 'u')
input_text = 'press "y" to see the answer:\npress "n" to skip:\npress "u" to update:\n'
print('Question:', row.question)
user_choice = check_input(input_text,
proper_values=allowed,
message='is not an option')
if user_choice == 'u':
Flashcards.change(row)
continue
elif user_choice == 'y':
print('Answer:', row.answer, '\n')
Flashcards.learning(row)
class Spider:
url = 'https://renrendai.com/'
header = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'
}
session = requests.session()
dbWorker = DBWorker()
def __init__(self):
self.__make_sure_login()
@staticmethod
def parse_ids(ids) -> [str]:
return [str(each) for each in ids]
def get_html(self, base_url: str = url, header=None) -> requests.Response:
if header is None:
header = self.header
return self.session.get(base_url, headers=header, timeout=30)
def post_html(self, url, data=None, header=None):
if header is None:
header = self.header
return self.session.post(url, headers=header, data=data)
def step0(self):
for i in range(50):
result = self.get_html(
'https://renrendai.com/loan/list/loanList?startNum=%s&limit=10'
% i)
if result.status_code == 200:
response = result.json()
for each in response['data']['list']:
if r.sismember('done', each['loanId']):
return
r.sadd('yet', each['loanId'])
time.sleep(1)
def step1(self, loan_id: str):
response = self.get_html(base_url='https://renrendai.com/' + loan_id)
if response.status_code == 302:
print(302)
return 302
elif response.status_code == 200:
result = re.findall(r"var info = '(.*)';",
response.content.decode())
if not result:
print('error info')
return 1
else:
res = json.loads(result[0].replace('\\u0022', '"').replace(
'\\u005C', '\\'))
self.step1_save(res)
pass
return 200
else:
print(response.status_code)
return response.status_code
def step1_save(self, result):
loan = result['loan']
user = result['borrower']
record = result['userLoanRecord']
try:
ready_time = time.localtime(int(loan['readyTime'] / 1000))
except:
ready_time = None
loan_data = Loan(
loanId=loan['loanId'],
userId=user['userId'],
title=loan['title'],
amount=loan['amount'],
interest=loan['interest'],
months=loan['months'],
readyTime=ready_time,
interestPerShare=loan['interestPerShare'],
creditLevel=user['creditLevel'],
repayType=loan['repayType'],
repaySource=result['repaySource'],
leftMonths=loan['leftMonths'],
status=loan['status'],
)
try:
if self.dbWorker.search(Loan.loanId == loan['loanId']):
return -1
except:
return -1
self.dbWorker.insert(loan_data)
borrower = []
try:
borrower.append(str(user['userId']))
if self.dbWorker.search(Borrower.userId == user['userId']):
return
except:
borrower.append('')
try:
borrower.append(user['nickName'])
except:
borrower.append('')
try:
borrower.append(user['realName'])
except:
borrower.append('')
try:
borrower.append(user['idNo'])
except:
borrower.append('')
try:
borrower.append(user['gender'])
except:
borrower.append('')
try:
borrower.append(user['mobile'])
except:
borrower.append('')
try:
borrower.append(user['birthDay'])
except:
borrower.append('')
try:
borrower.append(user['graduation'])
except:
borrower.append('')
try:
borrower.append(str(user['marriage']))
except:
borrower.append('')
try:
borrower.append(user['salary'])
except:
borrower.append('')
try:
borrower.append(str(user['hasHouse']))
except:
borrower.append('')
try:
borrower.append(str(user['hasCar']))
except:
borrower.append('')
try:
borrower.append(str(user['houseLoan']))
except:
borrower.append('')
try:
borrower.append(str(user['carLoan']))
except:
borrower.append('')
try:
borrower.append(user['officeDomain'])
except:
borrower.append('')
try:
borrower.append(user['city'])
except:
borrower.append('')
try:
borrower.append(user['workYears'])
except:
borrower.append('')
try:
borrower.append(user['auditItems'])
except:
borrower.append('')
try:
borrower.append(str(record['totalCount']))
except:
borrower.append('')
try:
borrower.append(str(record['successCount']))
except:
borrower.append('')
try:
borrower.append(str(record['alreadyPayCount']))
except:
borrower.append('')
try:
borrower.append(str(record['borrowAmount']))
except:
borrower.append('')
try:
borrower.append(str(record['notPayPrincipal']))
except:
borrower.append('')
try:
borrower.append(str(record['notPayTotalAmount']))
except:
borrower.append('')
try:
borrower.append(str(record['overdueAmount']))
except:
borrower.append('')
try:
borrower.append(str(record['overdueCount']))
except:
borrower.append('')
try:
borrower.append(str(record['overdueTotalAmount']))
except:
borrower.append('')
borrower_data = Borrower(
userId=borrower[0],
nickName=borrower[1],
realName=borrower[2],
idNo=borrower[3],
gender=borrower[4],
mobile=borrower[5],
birthDay=borrower[6],
graduation=borrower[7],
marriage=borrower[8],
salary=borrower[9],
hasHouse=borrower[10],
hasCar=borrower[11],
houseLoan=borrower[12],
carLoan=borrower[13],
officeDomain=borrower[14],
city=borrower[15],
workYears=borrower[16],
auditItems=borrower[17],
totalCount=borrower[18],
successCount=borrower[19],
alreadyPayCount=borrower[20],
borrowAmount=borrower[21],
notPayPrincipal=borrower[22],
notPayTotalAmount=borrower[23],
overdueAmount=borrower[24],
overdueCount=borrower[25],
overdueTotalAmount=borrower[26],
)
self.dbWorker.insert(borrower_data)
# with open('loan.csv', 'ab+') as f:
# # f.write('loanId,userId,title,amount,interest,months,readyTime,interestPerShare,creditLevel,repayType,repaySource,leftMonths,status\n'.encode())
# f.write(','.join(loan_data).encode())
# f.write('\n'.encode())
# with open('borrower.csv', 'ab+') as f:
# # f.write('userId,nickName,realName,idNo,gender,mobile,birthDay,graduation,marriage,salary,hasHouse,hasCar,houseLoan,carLoan,officeDomain,city,workYears,auditItems\n'.encode())
# f.write(','.join(borrower).encode())
# f.write('\n'.encode())
def step2(self, loan_id):
while True:
response = self.get_html(
'https://renrendai.com/transfer/detail/loanInvestment?loanId='
+ loan_id)
if response.status_code != 200:
# {"status":1001,"message":"用户未登录, 或者登录状态已过期"}
print(response.status_code)
return
res = json.loads(response.content.decode())
if res['status'] != 0:
if res['status'] == 1001:
self.__make_sure_login()
continue
else:
print(res)
exit(2)
self.step2_save(res)
return
def step2_save(self, result):
# with open('loanInvestment.csv', 'ab+') as f:
# # # f.write('loanId,userId,userNickName,amount,lendTime\n'.encode())
# # for each in result['data']['list']:
# # f.write(','.join([str(each['loanId']), str(each['userId']), each['userNickName'], str(each['amount']), time.strftime('%Y-%m-%d %H:%M', time.localtime(int(each['lendTime'] / 1000)))]).encode())
# # f.write('\n'.encode())
investments = []
for each in result['data']['list']:
investments.append(
Investment(
loanId=each['loanId'],
userId=each['userId'],
userNickName=each['userNickName'],
amount=each['amount'],
lendTime=time.localtime(int(each['lendTime'] / 1000)),
))
self.dbWorker.insert_all(investments)
def step3(self, loan_id):
while True:
response = self.get_html(
'https://renrendai.com/transfer/detail/loanTransferred?loanId='
+ loan_id)
if response.status_code != 200:
print(response.status_code)
return
res = json.loads(response.content.decode())
if res['status'] != 0:
if res['status'] == 1001:
self.__make_sure_login()
continue
else:
print(res)
exit(3)
self.step3_save(loan_id, res)
return
def step3_save(self, loan_id, result):
if result['status'] != 0:
print(result['message'])
exit(3)
loanrepayments = []
for each in result['data']['list']:
if each['actualRepayTime']:
temp = time.localtime(int(each['actualRepayTime'] / 1000))
else:
temp = None
loanrepayments.append(
Loanrepayment(
loanId=int(loan_id),
repayTime=time.localtime(int(each['repayTime'] / 1000)),
repayType=each['repayType'],
unRepaidAmount=each['unRepaidAmount'],
repaidFee=each['repaidFee'],
actualRepayTime=temp,
))
self.dbWorker.insert_all(loanrepayments)
# with open('loanTransferred.csv', 'ab+') as f:
# # f.write('loan_id,repayTime,repayType,unRepaidAmount,repaidFee,actualRepayTime\n'.encode())
# for each in result['data']['list']:
# temp = [str(loan_id), time.strftime('%Y-%m-%d', time.localtime(int(each['repayTime'] / 1000))), each['repayType'], str(each['unRepaidAmount']), str(each['repaidFee'])]
# if each['actualRepayTime']:
# temp.append(time.strftime('%Y-%m-%d', time.localtime(int(each['actualRepaidTime'] / 1000))))
# else:
# temp.append('')
# f.write(','.join(temp).encode())
# f.write('\n'.encode())
def step4(self, loan_id):
while True:
html = self.get_html(
'https://renrendai.com/transfer/detail/loanRepayment?loanId=%s'
% loan_id)
if html.status_code != 200:
print(html.status_code)
return
res = json.loads(html.content.decode())
if res['status'] != 0:
if res['status'] == 1001:
self.__make_sure_login()
continue
else:
print(res)
exit(4)
self.step4_save(loan_id, res)
return
def step4_save(self, loan_id, result):
transfers = []
for each in result['data']['list']:
transfers.append(
Transfer(
transfer_id=int(each['transferId']),
loanId=loan_id,
toUserId=each['toUserId'],
fromUserId=each['fromUserId'],
price=float(each['price']),
createTime=time.localtime(int(each['createTime'] / 1000)),
))
self.dbWorker.insert_all(transfers)
def run(self):
while True:
each = r.spop('yet')
# each = r.rpop('old')
if not each:
print('done')
return
# print('sleep for 2h')
# time.sleep(2*60*60)
# continue
else:
each = each.decode()
print(each)
if self.step1('loan-' + each + '.html') == -1:
continue
self.step2(each)
self.step3(each)
r.sadd('done', int(each))
# r.zadd('old-done', {each: int(each)})
time.sleep(1)
def __make_sure_login(self):
login = self.post_html(
url='https://renrendai.com/p2p/bond/isShowBondOffDialog').json()
if login['message'] == 'not login':
print('log in failed')
self.__login()
self.__make_sure_login()
else:
print('log in success')
def __login(self):
html = self.get_html('https://renrendai.com/passport/index/captcha')
base64_data = base64.b64encode(html.content)
captcha = 'data:image/jpeg;base64,' + base64_data.decode()
data_captcha = {
'v_pic': captcha,
'v_type': 'n4',
}
res = requests.post(
'http://apigateway.jianjiaoshuju.com/api/v_1/yzm.html',
headers=header_captcha,
data=data_captcha)
if res.json()['errCode'] != 0:
print(res.json())
exit(1)
data = {
'j_username': user['username'],
'j_password': user['password'],
'j_code': res.json()['v_code'],
'rememberme': 'on',
'targetUrl': '',
'returnUrl': '/user/account/p2p/index',
}
h = self.post_html(url='https://renrendai.com/passport/index/doLogin',
data=data)
res = h.json()
if res['status'] == 0:
return
else:
print(res['message'])
raise Exception("Bad arguments")
_id = request.args.get("id")
result = db_worker.get_full_mobile_info_by_id(_id)
if len(result) != 0:
return json.dumps(result[0], ensure_ascii=False)
raise Exception("No such mobile, id={0}".format(_id))
except Exception as e:
return make_response(str(e), 400, {'olol': 'ololol'})
@app.route("/get_all_mobiles")
def get_all_mobiles():
try:
if not check_arguments(['page'], request.args):
raise Exception("Bad arguments")
page = request.args.get('page')
count_per_page = 2
return json.dumps(db_worker.get_all_mobiles(page, count_per_page))
except Exception as e:
return make_response(str(e), 400, {'olol': 'ololol'})
if __name__ == "__main__":
db_worker = DBWorker()
app.run(debug=True, port=65013)
class Spider:
header = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36',
}
header2 = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36',
'Content-Type': 'application/json;charset=UTF-8',
}
session = requests.Session()
dbWorker = DBWorker()
flag = True
def __init__(self):
self.flag = True
self.login()
pass
def get_html(self, url, header=None):
if not header:
header = self.header
res = self.session.get(url, headers=header, timeout=30)
if res.status_code != 200:
print(res.content.decode())
exit(res.status_code)
return res
def post_html(self, url, data=None, header=None):
if not header:
header = self.header
try:
res = self.session.post(url, data=data, headers=header)
except Exception as e:
print(e)
time.sleep(30)
return self.post_html(url, data, header)
if res.status_code != 200:
if res.status_code == 502 or res.status_code == 500:
print(res.status_code)
time.sleep(180)
return self.post_html(url, data, header)
print(res.status_code)
print(res.content.decode())
exit(res.status_code)
return res
def confirm_login(self, token):
return self.session.post(
'https://tz.ppdai.com/api/raptor/h5api/userDetailInfoV3',
headers=self.header2,
data=json.dumps({'token': token}))
def login(self):
self.session = requests.session()
html = self.get_html('https://ac.ppdai.com/ValidateCode/Image')
base64_data = base64.b64encode(html.content)
captcha = 'data:image/jpeg;base64,' + base64_data.decode()
data_captcha = {
'v_pic': captcha,
'v_type': 'n4',
}
res = requests.post(
'http://apigateway.jianjiaoshuju.com/api/v_1/yzm.html',
headers=header_captcha,
data=data_captcha)
if res.json()['errCode'] != 0:
if res.json()['errCode'] == 102:
time.sleep(10)
return self.login()
print(res.json())
exit(1)
url = 'https://ac.ppdai.com/User/Login'
data = {
'IsAsync': 'true',
'Redirect': '',
'UserName': user['username'],
'Password': user['password'],
'ValidateCode': res.json()['v_code'],
'RememberMe': 'true',
}
html = self.post_html(url, data).json()
if html['Code'] == 1:
print(html)
return
else:
if html['Message'] == '验证码输入有误' or html['Message'] == '验证码错误':
print('验证码输入有误')
self.login()
else:
if html['Message'] == '您的帐号登录频繁，请稍后再试':
print('您的帐号登录频繁，请稍后再试')
time.sleep(600)
self.login()
else:
print(html)
exit(-1)
# ck_dict = requests.utils.dict_from_cookiejar(self.session.cookies)
# res = self.confirm_login(ck_dict['token'])
# print(res.content.decode())
def step0(self):
ip = Ip()
data = {
'authInfo': "",
'authenticated': False,
'availableBalance': 0,
'creditCodes': "",
'dataList': [],
'didIBid': "0",
'maxAmount': 0,
'minAmount': 0,
'months': "",
'needTotalCount': True,
'pageCount': 0,
'pageIndex': 1,
'pageSize': 10,
'rates': "",
'riskLevelCategory': 0,
'sort': 0,
'source': 1,
'successLoanNum': "",
'totalCount': 0,
}
while True:
if r_ip.dbsize() < 3:
ip.get_ips()
print(
str(data['riskLevelCategory']) + '---------' +
str(data['pageIndex']))
# 代理服务器
proxy_host = r_ip.randomkey().decode()
proxy_port = r_ip.get(proxy_host).decode()
proxy_meta = "http://%(host)s:%(port)s" % {
"host": proxy_host,
"port": proxy_port,
}
proxies = {
"http": proxy_meta,
"https": proxy_meta,
}
# res = json.loads(self.post_html('https://invest.ppdai.com/api/invapi/ListingListAuthService/listingPagerAuth', data=json.dumps(data), header=header).content.decode())
res = json.loads(
requests.post(
'https://invest.ppdai.com/api/invapi/ListingListNoAuthService/listingPagerNoAuth',
data=json.dumps(data),
headers=self.header2,
proxies=proxies).content.decode())
if res['result'] != 1:
print(res)
exit(res['result'])
if not res['resultContent']['dataList']:
if data['riskLevelCategory'] == 2:
return
data['riskLevelCategory'] += 1
data['pageIndex'] = 1
continue
for each in res['resultContent']['dataList']:
listing_id = each['listingId']
print(listing_id)
if r.sismember('done', listing_id):
continue
else:
r.sadd('yet', listing_id)
data['pageIndex'] += 1
time.sleep(3)
def step1(self, listing_id):
if self.dbWorker.search(Loan.loan_id == listing_id):
return False
while True:
url = 'https://invest.ppdai.com/api/invapi/LoanDetailPcService/showListingBaseInfo'
data = {
'listingId': str(listing_id),
'source': 1,
}
res = json.loads(
self.post_html(url, header=self.header2,
data=json.dumps(data)).content.decode())
if res['result'] != 1:
if res['result'] == -1:
print(res)
time.sleep(300)
continue
if res['result'] == 404:
print(res)
return False
if res['result'] == 5066:
print(res)
time.sleep(30)
self.flag = False
return False
if res['result'] == 5067:
return False
if res['result'] == 1012:
print(res)
time.sleep(300)
continue
print(res)
exit(-1)
return self.step1_save(res['resultContent'])
def step1_save(self, result):
loan_id = result['listing']['listingId']
title = result['listing']['title']
stamps = []
if result['listing']['ispay']:
stamps.append('赔')
if result['listing']['vouch'] == 168:
stamps.append('安')
if result['listing']['listType'] == 15:
stamps.append('助农')
if result['listing']['listType'] == 16:
stamps.append('富')
if result['listing']['listType'] == 18:
stamps.append('车')
if result['listing']['listType'] == 25:
stamps.append('助学')
if result['listing']['isSevenDaysRecede']:
stamps.append('7天')
credit = result['listing']['creditCode']
user_name = result['userInfo']['userName']
amount = result['listing']['amount']
current_rate = result['listing']['currentRate']
months = result['listing']['months']
if result['repaymentMethod'] == 1:
repayment_method = '一次还本付息'
elif result['listing']['listType'] == 16:
repayment_method = '一次还本付息'
elif result['listing']['listType'] == 15:
repayment_method = '月还息，季还1/4本金'
elif result['listing']['listType'] == 88:
repayment_method = ''
else:
repayment_method = '等额本息（按月还款）'
try:
progress = int(
(result['listing']['funding'] / result['listing']['amount']) *
100)
except:
progress = 0
bid_users = result['totalBidUsers']
list_end_date = time.strptime(result['listing']['listEndDate'],
'%Y-%m-%d %H:%M:%S')
try:
bid_status = {
0: "正在预审中",
2: "借款审批中",
4: "借款成功",
12: "借款成功",
20: "资金募集中"
}[result['listing']['statusId']]
except:
if result['listing']['statusId'] != 1 or result['listingSaveBid']:
bid_status = '投标已结束'
else:
bid_status = '投标中'
loan_use = result['loanUse']
user_id = result['userInfo']['id']
self.dbWorker.insert(
Loan(
loan_id=loan_id,
title=title,
stamps='，'.join(stamps),
credit=credit,
user_name=user_name,
amount=amount,
current_rate=current_rate,
months=months,
repayment_method=repayment_method,
progress=progress,
bid_users=bid_users,
list_end_date=list_end_date,
bid_status=bid_status,
loan_use=loan_use,
user_id=user_id,
))
return user_id
def step2(self, user_id, listing_id):
while True:
url = 'https://invest.ppdai.com/api/invapi/LoanDetailPcService/showBorrowerInfo'
data = {
'listingId': str(listing_id),
'source': 1,
}
res = json.loads(
self.post_html(url=url,
data=json.dumps(data),
header=self.header2).content.decode())
if res['result'] != 1:
if res['result'] == -1:
print(res)
time.sleep(300)
continue
if res['result'] == 5066:
print(res)
time.sleep(30)
self.flag = False
return False
if res['result'] == 5067:
return False
if res['result'] == 1012:
print(res)
time.sleep(300)
continue
print(res)
exit(-2)
self.step2_save(user_id, res['resultContent'])
return
def step2_save(self, user_id, result):
if self.dbWorker.search(User.user_id == user_id):
return
real_name = result['realName']
id_number = result['idNumber']
gender = result['gender']
age = result['age']
register_date = time.strptime(result['registerDateStr'], '%Y-%m-%d')
degree = '暂未提供'
graduation = '暂未提供'
study_style = '暂未提供'
if result['educationInfo']:
degree = result['educationInfo']['educationDegree']
graduation = result['educationInfo']['graduate']
study_style = result['educationInfo']['studyStyle']
if result['overdueStatus'] == 1:
overdue_status = '有逾期'
elif result['overdueStatus'] == 2:
overdue_status = '无逾期'
else:
overdue_status = '暂未提供'
overdue_types = None
if result['overdueStatus'] == 1 and result['overdueTyps']:
overdue_types = '，'.join(result['overdueTyps'])
repayment_source = result['repaymentSourceType']
work_info = result['workInfo']
if result['income']:
income = result['income']
else:
income = '暂未提供'
if result['balAmount']:
bal_amount = result['balAmount']
else:
bal_amount = '暂未提供'
if result['industry']:
industry = result['industry']
else:
industry = '暂未提供'
auths = ''
if result['userAuthsList']:
auths = '，'.join(
[each['name'] for each in result['userAuthsList']])
self.dbWorker.insert(
User(
user_id=user_id,
real_name=real_name,
id_number=id_number,
gender=gender,
age=age,
register_date=register_date,
degree=degree,
graduation=graduation,
study_style=study_style,
overdue_status=overdue_status,
overdue_types=overdue_types,
repayment_source=repayment_source,
work_info=work_info,
income=income,
bal_amount=bal_amount,
industry=industry,
auths=auths,
))
def step3(self, user_id, listing_id):
while True:
url = 'https://invest.ppdai.com/api/invapi/LoanDetailPcService/showBorrowerStatistics'
data = {
'listingId': str(listing_id),
'source': 1,
}
res = json.loads(
self.post_html(url=url,
data=json.dumps(data),
header=self.header2).content.decode())
if res['result'] != 1:
if res['result'] == -1:
print(res)
time.sleep(300)
continue
if res['result'] == 5066:
print(res)
time.sleep(30)
self.flag = False
return False
if res['result'] == 5067:
return False
if res['result'] == 1012:
print(res)
time.sleep(300)
continue
print(res)
exit(-3)
self.step3_save(user_id, listing_id, res['resultContent'])
return
def step3_save(self, user_id, loan_id, result):
if self.dbWorker.search(Statistic.loan_id == loan_id):
return
if result['loanerStatistics']:
success_borrow_num = result['loanerStatistics']['listingStatics'][
'successNum']
try:
first_success_borrow_date = time.strptime(
result['loanerStatistics']['listingStatics']
['firstSuccessDate'], '%Y-%m-%d %H:%M:%S')
except:
first_success_borrow_date = None
history = str(result['loanerStatistics']['listingStatics']
['wasteNum']) + "次流标，" + str(
result['loanerStatistics']['listingStatics']
['cancelNum']) + "次撤标，" + str(
result['loanerStatistics']['listingStatics']
['failNum']) + "次失败"
success_pay_num = result['loanerStatistics']['successNum']
normal_num = result['loanerStatistics']['normalNum']
overdue_less_num = result['loanerStatistics']['overdueLessNum']
overdue_more_num = result['loanerStatistics']['overdueMoreNum']
owing_amount_map = result['loanerStatistics']['owingAmountMap']
overdue_day_map = result['loanerStatistics']['overdueDayMap']
total_principal = result['loanerStatistics']['totalPrincipal']
owing_amount = result['loanerStatistics']['owingAmount']
loan_amount_max = result['loanerStatistics']['loanAmountMax']
debt_amount_max = result['loanerStatistics']['debtAmountMax']
debt_amount_map = result['loanerStatistics']['debtAmountMap']
else:
success_borrow_num = None
first_success_borrow_date = None
history = None
success_pay_num = None
normal_num = None
overdue_less_num = None
overdue_more_num = None
owing_amount_map = None
overdue_day_map = None
total_principal = None
owing_amount = None
loan_amount_max = None
debt_amount_max = None
debt_amount_map = None
if result['otherProjectInfo']:
business_and_financial_info = result['otherProjectInfo'][
'businessAndFinancialInfo']
repayment_power_change = result['otherProjectInfo'][
'repaymentPowerChange']
bad_info = result['otherProjectInfo']['badInfo']
administrative_penalty = result['otherProjectInfo'][
'administrativePenalty']
else:
business_and_financial_info = None
repayment_power_change = None
bad_info = None
administrative_penalty = None
self.dbWorker.insert(
Statistic(
loan_id=loan_id,
success_borrow_num=success_borrow_num,
first_success_borrow_date=first_success_borrow_date,
history=history,
success_pay_num=success_pay_num,
normal_num=normal_num,
overdue_less_num=overdue_less_num,
overdue_more_num=overdue_more_num,
owing_amount_map=owing_amount_map,
overdue_day_map=overdue_day_map,
total_principal=total_principal,
owing_amount=owing_amount,
loan_amount_max=loan_amount_max,
debt_amount_max=debt_amount_max,
debt_amount_map=debt_amount_map,
business_and_financial_info=business_and_financial_info,
repayment_power_change=repayment_power_change,
bad_info=bad_info,
administrative_penalty=administrative_penalty,
))
if result['loanerStatistics'] and result['loanerStatistics'][
'previousListings']:
previous_listing = []
for each in result['loanerStatistics']['previousListings']:
try:
status = {
0: "待审中",
1: "正在进行中",
2: "待批准",
3: "流标",
4: "成功",
5: "批准失败",
6: "借出者付款",
7: "借入者收到借款",
8: "借入者还款",
9: "借出者收到还款",
10: "已撤回",
11: "草稿",
12: "已还完"
}[each['statusId']]
except:
status = ''
previous_listing.append(
PreviousListing(
user_id=user_id,
title=each['title'],
rate=each['showRate'],
months=each['months'],
amount=each['amount'],
creation_date=time.strptime(each['creationDate'],
'%Y-%m-%d %H:%M:%S'),
status=status,
loan_id=loan_id,
))
self.dbWorker.insert_all(previous_listing)
def step4(self, listing_id):
while True:
url = 'https://invest.ppdai.com/api/invapi/LoanDetailPcService/showBidRecord'
data = {
'listingId': str(listing_id),
'source': 1,
}
res = json.loads(
self.post_html(url=url,
data=json.dumps(data),
header=self.header2).content.decode())
if res['result'] != 1:
if res['result'] == -1:
print(res)
time.sleep(300)
continue
if res['result'] == 5066:
print(res)
time.sleep(30)
self.flag = False
return False
if res['result'] == 5067:
return False
if res['result'] == 1012:
print(res)
time.sleep(300)
continue
print(res)
exit(-4)
self.step4_save(listing_id, res['resultContent'])
return
def step4_save(self, loan_id, result):
if not result['bidRecordList']:
return
records = []
for each in result['bidRecordList']:
try:
source = {
3: "APP投标",
4: "自动投标",
5: "快投",
7: "OpenAPi投标",
8: "项目",
9: "自动投标",
10: "一键投标",
12: "一键投标",
13: "自动投标",
14: "项目",
16: "策略",
18: "项目",
19: "极速投标"
}[each['source']]
except:
source = ''
records.append(
Record(
loan_id=loan_id,
lender_id=each['lenderId'],
lender_name=each['lenderName'],
source=source,
rate=each['bidRate'],
participation_amount=each['participationAmount'],
creation_date=time.strptime(each['creationDate'],
'%Y-%m-%d %H:%M:%S'),
))
self.dbWorker.insert_all(records)
def step5(self, listing_id):
while True:
url = 'https://invest.ppdai.com/api/invapi/LoanDetailPcService/showDebtRecord'
data = {
'listingId': str(listing_id),
'source': 1,
}
res = json.loads(
self.post_html(url=url,
data=json.dumps(data),
header=self.header2).content.decode())
if res['result'] != 1:
if res['result'] == -1:
print(res)
time.sleep(300)
continue
if res['result'] == 5066:
print(res)
time.sleep(30)
self.flag = False
return False
if res['result'] == 5067:
return False
if res['result'] == 1012:
print(res)
time.sleep(300)
continue
print(res)
exit(-5)
self.step5_save(listing_id, res['resultContent'])
return
def step5_save(self, listing_id, result):
if not result['debtRecordList']:
return
debt_record = []
for each in result['debtRecordList']:
try:
buy_source_type = {
1: "PC投标",
2: "APP投标",
3: "自动投标",
4: "OpenAPi投标",
9: "PC一键投标",
10: "APP一键投标",
}[each['buySourceType']]
except:
buy_source_type = '项目'
debt_record.append(
DebtRecord(
loan_id=listing_id,
lender_id=each['lenderId'],
lender_name=each['lenderName'],
owing_principal=each['owingPrincipal'],
price_for_sell=each['priceForSell'],
buy_source_type=buy_source_type,
close_bid_date=time.strptime(each['closeBidDate'],
'%Y-%m-%d %H:%M:%S'),
debt_deal_id=str(each['debtDealId']),
buyer_user_name=each['buyerUserName'],
))
self.dbWorker.insert_all(debt_record)
def run(self):
while self.flag:
ld = r.spop('yet')
if not ld:
print('sleep for 2h')
time.sleep(2 * 60 * 60)
continue
ld = int(ld.decode())
print(ld)
r.sadd('done', ld)
user_id = self.step1(ld)
if not user_id:
continue
time.sleep(3)
self.step2(user_id, ld)
time.sleep(3)
self.step3(user_id, ld)
time.sleep(3)
self.step4(ld)
time.sleep(3)
self.step5(ld)
time.sleep(3)